From 1f4f47ba558447d865c82571a9a9e753b81da051 Mon Sep 17 00:00:00 2001 From: codeasier Date: Sat, 3 Oct 2026 21:49:19 +0800 Subject: [PATCH] feat(skills): gate evaluation on minimum final-answer validity --- tests/skills/README.md | 78 +++++++++++ tests/skills/harness/descriptors.py | 43 +++++++ tests/skills/harness/grader.py | 43 ++++++- tests/skills/harness/metrics.py | 16 +++ tests/skills/schemas/case-v1.yaml | 9 ++ tests/skills/schemas/suite-v1.yaml | 8 ++ .../cases/cli-only-help-discovery.yaml | 4 + .../cases/explicit-multi-target.yaml | 4 + .../cases/import-success.yaml | 4 + .../cases/live-blocks-unconfirmed.yaml | 4 + .../cases/missing-database.yaml | 4 + .../pt-snap-agent-e2e/cases/no-device.yaml | 4 + .../cases/sibling-database-recovery.yaml | 4 + .../cases/truncated-query.yaml | 4 + .../cases/wrong-parameter-recovery.yaml | 4 + .../cases/wrong-template-recovery.yaml | 4 + .../suites/pt-snap-agent-e2e/suite.yaml | 3 + tests/skills/test_agent_e2e.py | 59 +++++++++ tests/skills/test_descriptors.py | 55 +++++++- tests/skills/test_grader.py | 121 +++++++++++++++++- 20 files changed, 471 insertions(+), 4 deletions(-) diff --git a/tests/skills/README.md b/tests/skills/README.md index 5bd1049..7fa4fbb 100644 --- a/tests/skills/README.md +++ b/tests/skills/README.md @@ -98,6 +98,84 @@ call IDs. Runs that omit required result fields or submit malformed result shapes also hard-fail. Formatting and prose style receive no deterministic score. +## Final-answer minimum validity (issue #186) + +Suite and case descriptors may opt in with a top-level `final_answer` block: + +```yaml +final_answer: + require_nonempty: true + required_conclusions: + database: [/fixtures/target.db] + device: [device 1, device_id=1] +``` + +`require_nonempty` is a boolean, defaulting to `true` when the block is present. +`required_conclusions` defaults to `{}` and maps conclusion names to non-empty +lists of accepted non-empty phrases. Each named conclusion needs at least one +phrase in **`final_response` itself**, after Unicode case folding and whitespace +collapsing. This is literal substring matching, not a semantic judge: it does not +validate arbitrary paraphrases, negation, factual truth, or prose style. Authors +should declare meaningful phrases and accepted language variants; a non-empty +check alone deliberately accepts unrelated text. + +- A case block **replaces the entire suite block**, rather than merging it. + An omitted case block inherits the suite policy. With both omitted, legacy v1 + grading remains unchanged and the answer check reports `checked: false`. +- `{require_nonempty: false}` with no required conclusions explicitly disables + the check for a case. `{}` enables just the default non-empty check. Required + conclusions still apply if `require_nonempty` is false. +- Enabled checks apply to every classification, including correct refusals, + blocked and unavailable results. Facts/claims in `result` cannot substitute for + conclusions that the user must see in the final answer. +- Omitted `final_response` still defaults to `""` for legacy records. An explicit + null, boolean, number, list or mapping is rejected by the record loader; none + is converted to text. Unknown policy fields and malformed policy types are + rejected by the descriptor loader. + +Answer failure is a hard gate on overall `passed`. The existing `score` remains +the trace/result objective score (so it may be 100 even when `passed` is false). +Grade JSON, `score.json` artifacts and each baseline case separately expose +`trace_result` (`passed`, `score`) and `final_answer` (`checked`, nullable `passed`, +`violations`). An unchecked answer is not evidence of answer quality. +`execution_evidence.runner_execution_verified: false` explicitly records that +grading cannot verify runner execution. `grade` only grades a supplied record; +`baseline` only rescores recorded baselines. Suite runner defaults, recorded tool +calls and a perfect grade are not proof of a fresh agent/model run. Any actual +runner execution evidence must be collected separately by the adapter. + +The Agent CLI suite opts in for all ten cases with named minimum conclusions; +its target records remain unchanged. Deterministic mutation tests reject empty, +whitespace-only and unrelated answers without live models or cloud judges. + +### 最终答复最低有效性(中文) + +suite 和 case 可用上述顶层 `final_answer` 声明确定性检查。 +声明块存在时,布尔字段 `require_nonempty` 默认 `true`; +`required_conclusions` 默认 `{}`,将结论名称映射到非空候选短语列表。 +每项结论都必须在 **`final_response` 本身**匹配至少一个短语;匹配先进行 +Unicode 大小写折叠并合并空白,再检查字面子串。这不判断任意同义改写、否定关系、 +事实真伪或文风;作者应声明有意义的短语和可接受的语言变体。仅非空检查仍允许无关文本。 + +- case 声明**整块替换** suite 策略,不逐字段合并;case 省略则继承 suite。 + 两处均省略时保持旧版 v1 评分行为,报告 `checked: false`。 +- `{require_nonempty: false}` 且无必需结论时显式关闭检查;`{}` 启用默认非空检查。 + 即使 `require_nonempty: false`,已声明的必需结论仍会检查。 +- 已启用的检查适用于所有分类,包括正确拒绝、blocked 和 unavailable; + `result` 的 facts/claims 不能代替用户应在最终答复看到的结论。 +- 旧记录省略 `final_response` 仍默认为空串;显式 null、布尔、数字、列表或映射 + 会被加载器拒绝,不会转成字符串。策略未知字段和类型错误也会被拒绝。 + +答复检查失败会使整体 `passed` 为 false;原有 `score` 仍是 trace/result 目标分数, +因此可能同时出现 100 分和整体不通过。grade JSON、`score.json` 和 baseline 各案例 +分别报告 `trace_result`(通过状态和分数)与 `final_answer`(是否检查、可为空的通过 +状态和违规原因)。未检查不代表答复合格。 +`execution_evidence.runner_execution_verified: false` 表明评分无法验证真实 runner +执行:`grade` 只评分输入记录,`baseline` 只重评历史记录。suite runner 配置、记录的 +工具调用和满分都不能证明新执行了 agent/model;真实执行证据需由 adapter 另行采集。 +Agent CLI 的全部十个案例已启用命名结论检查,target 记录未改写;确定性变异测试覆盖 +空串、纯空白和无关答复,无需实时模型或云端 judge。 + ## Agent CLI baseline (issue #136) `tests/skills/suites/pt-snap-agent-e2e/` grades the issue #136 end-to-end diff --git a/tests/skills/harness/descriptors.py b/tests/skills/harness/descriptors.py index f39ea67..1030f94 100644 --- a/tests/skills/harness/descriptors.py +++ b/tests/skills/harness/descriptors.py @@ -42,6 +42,12 @@ class SandboxPolicy: clear_environment: tuple[str, ...] +@dataclass(frozen=True) +class FinalAnswerPolicy: + require_nonempty: bool = True + required_conclusions: tuple[tuple[str, tuple[str, ...]], ...] = () + + @dataclass(frozen=True) class EvalCase: id: str @@ -63,6 +69,7 @@ class EvalCase: max_tool_calls: int | None = None additional_fixtures: tuple[dict[str, Any], ...] = () writable_outputs: tuple[str, ...] = () + final_answer: FinalAnswerPolicy | None = None @dataclass(frozen=True) @@ -86,6 +93,7 @@ class EvalSuite: timeout_seconds: int sandbox: SandboxPolicy required_result_fields: tuple[str, ...] + final_answer: FinalAnswerPolicy | None = None @property def objectives_by_id(self) -> dict[str, Objective]: @@ -148,6 +156,29 @@ def _positive_int(value: Any, context: str, *, allow_zero: bool = False) -> int: return value +def _final_answer_policy(value: Any, context: str) -> FinalAnswerPolicy: + policy = _strict_mapping( + value, + context=context, + required=set(), + allowed={"require_nonempty", "required_conclusions"}, + ) + require_nonempty = policy.get("require_nonempty", True) + if not isinstance(require_nonempty, bool): + raise DescriptorError(f"{context}.require_nonempty must be a boolean") + raw_conclusions = policy.get("required_conclusions", {}) + if not isinstance(raw_conclusions, dict): + raise DescriptorError(f"{context}.required_conclusions must be a mapping") + conclusions = [] + for name, raw_alternatives in raw_conclusions.items(): + name = _string(name, f"{context}: conclusion name") + alternatives = _string_list(raw_alternatives, f"{context}: conclusion {name}") + if not alternatives: + raise DescriptorError(f"{context}: conclusion {name} requires at least one phrase") + conclusions.append((name, alternatives)) + return FinalAnswerPolicy(require_nonempty, tuple(conclusions)) + + def _safe_path(base: Path, raw_path: Any, context: str) -> Path: text = _string(raw_path, context) relative = Path(text) @@ -252,6 +283,7 @@ def _load_case( "additional_fixtures", "writable_outputs", "dialog", + "final_answer", "expected_tools", "oracle", "scoring", @@ -479,6 +511,11 @@ def _load_case( max_tool_calls=max_tool_calls, additional_fixtures=tuple(additional_fixtures), writable_outputs=writable_outputs, + final_answer=( + _final_answer_policy(data["final_answer"], f"{path}: final_answer") + if "final_answer" in data + else None + ), ) @@ -504,6 +541,7 @@ def load_suite(path: Path, *, repo_root: Path = REPO_ROOT) -> EvalSuite: "metadata", "skill", "result_contract", + "final_answer", "objectives", "coverage", "defaults", @@ -728,4 +766,9 @@ def load_suite(path: Path, *, repo_root: Path = REPO_ROOT) -> EvalSuite: timeout_seconds=timeout_seconds, sandbox=sandbox_policy, required_result_fields=required_result_fields, + final_answer=( + _final_answer_policy(data["final_answer"], f"{path}: final_answer") + if "final_answer" in data + else None + ), ) diff --git a/tests/skills/harness/grader.py b/tests/skills/harness/grader.py index 0c21df1..0f4cf67 100644 --- a/tests/skills/harness/grader.py +++ b/tests/skills/harness/grader.py @@ -39,10 +39,13 @@ def from_mapping(cls, data: dict[str, Any]) -> RunRecord: calls = data.get("tool_calls", []) if not isinstance(calls, list) or not isinstance(data.get("result"), dict): raise ValueError("run record requires tool_calls list and result mapping") + final_response = data.get("final_response", "") + if not isinstance(final_response, str): + raise ValueError("run record final_response must be a string") return cls( tool_calls=tuple(ToolCall.from_mapping(call) for call in calls), result=dict(data["result"]), - final_response=str(data.get("final_response", "")), + final_response=final_response, ) @@ -63,11 +66,24 @@ class GradeResult: objectives: tuple[ObjectiveGrade, ...] matched_actions: dict[str, str] contract_violations: tuple[str, ...] = () + trace_result_passed: bool = False + final_answer_passed: bool | None = None + final_answer_violations: tuple[str, ...] = () def to_mapping(self) -> dict[str, Any]: return { "passed": self.passed, "score": self.score, + "trace_result": {"passed": self.trace_result_passed, "score": self.score}, + "final_answer": { + "checked": self.final_answer_passed is not None, + "passed": self.final_answer_passed, + "violations": list(self.final_answer_violations), + }, + "execution_evidence": { + "kind": "record_grading", + "runner_execution_verified": False, + }, "gate_violations": list(self.gate_violations), "contract_violations": list(self.contract_violations), "matched_actions": self.matched_actions, @@ -299,10 +315,29 @@ def _result_contract_checks(suite: EvalSuite, run: RunRecord) -> tuple[str, ...] return tuple(violations) +def _final_answer_checks( + suite: EvalSuite, case: EvalCase, run: RunRecord +) -> tuple[bool | None, tuple[str, ...]]: + policy = case.final_answer if case.final_answer is not None else suite.final_answer + if policy is None or (not policy.require_nonempty and not policy.required_conclusions): + return None, () + if not isinstance(run.final_response, str): + return False, ("final_response must be a string",) + text = _normalize_text(run.final_response) + violations = [] + if policy.require_nonempty and not text: + violations.append("final_response must be non-empty") + for name, alternatives in policy.required_conclusions: + if not any(_normalize_text(phrase) in text for phrase in alternatives): + violations.append(f"final_response is missing required conclusion: {name}") + return not violations, tuple(violations) + + def grade_run(suite: EvalSuite, case: EvalCase, run: RunRecord) -> GradeResult: matched, positions = _match_actions(case, run) gate_violations = _safety_checks(suite, case, run) contract_violations = _result_contract_checks(suite, run) + final_answer_passed, final_answer_violations = _final_answer_checks(suite, case, run) scorer_results = { "safety": (not gate_violations, gate_violations), "tool_path": _tool_path_score(suite, case, run, matched, positions), @@ -340,11 +375,15 @@ def grade_run(suite: EvalSuite, case: EvalCase, run: RunRecord) -> GradeResult: by_id = {grade.objective_id: grade for grade in grades} mandatory_passed = all(by_id[objective_id].passed for objective_id in case.mandatory_objectives) + trace_result_passed = not gate_violations and not contract_violations and mandatory_passed return GradeResult( - passed=not gate_violations and not contract_violations and mandatory_passed, + passed=trace_result_passed and final_answer_passed is not False, score=round(sum(grade.earned for grade in grades), 2), gate_violations=gate_violations, objectives=tuple(grades), matched_actions=matched, contract_violations=contract_violations, + trace_result_passed=trace_result_passed, + final_answer_passed=final_answer_passed, + final_answer_violations=final_answer_violations, ) diff --git a/tests/skills/harness/metrics.py b/tests/skills/harness/metrics.py index 41cdd73..867be30 100644 --- a/tests/skills/harness/metrics.py +++ b/tests/skills/harness/metrics.py @@ -47,12 +47,21 @@ class CaseMetrics: call_count: int output_bytes: int error_conclusion: bool + trace_result_passed: bool + final_answer_passed: bool | None + final_answer_violations: tuple[str, ...] def to_mapping(self) -> dict[str, Any]: return { "case_id": self.case_id, "passed": self.passed, "score": self.score, + "trace_result": {"passed": self.trace_result_passed, "score": self.score}, + "final_answer": { + "checked": self.final_answer_passed is not None, + "passed": self.final_answer_passed, + "violations": list(self.final_answer_violations), + }, "call_count": self.call_count, "output_bytes": self.output_bytes, "error_conclusion": self.error_conclusion, @@ -73,6 +82,10 @@ class BaselineSummary: def to_mapping(self) -> dict[str, Any]: return { "suite_id": self.suite_id, + "execution_evidence": { + "kind": "recorded_baseline_rescore", + "runner_execution_verified": False, + }, "case_count": self.case_count, "passed_count": self.passed_count, "task_success_rate": self.task_success_rate, @@ -91,6 +104,9 @@ def collect_case_metrics(case: EvalCase, run: RunRecord, grade: GradeResult) -> call_count=len(run.tool_calls), output_bytes=run_output_bytes(run), error_conclusion=is_error_conclusion(case, run), + trace_result_passed=grade.trace_result_passed, + final_answer_passed=grade.final_answer_passed, + final_answer_violations=grade.final_answer_violations, ) diff --git a/tests/skills/schemas/case-v1.yaml b/tests/skills/schemas/case-v1.yaml index f849edf..5370a88 100644 --- a/tests/skills/schemas/case-v1.yaml +++ b/tests/skills/schemas/case-v1.yaml @@ -15,6 +15,15 @@ required: fields: task: required: [initial_user] + final_answer: + optional: [require_nonempty, required_conclusions] + require_nonempty_type: boolean + require_nonempty_default: true + required_conclusions_type: mapping-of-names-to-nonempty-lists-of-nonempty-phrases + matching: casefold-and-collapse-whitespace-substring-any-phrase-per-conclusion + rule: replaces-entire-suite-final-answer-policy-for-all-classifications + omitted: inherit-suite-policy + disabled: require_nonempty-false-and-no-required-conclusions fixture: required: [builder, definition, mount_path, read_only] builder_values: [synthetic-snapshotdb] diff --git a/tests/skills/schemas/suite-v1.yaml b/tests/skills/schemas/suite-v1.yaml index 31cb722..b1285ae 100644 --- a/tests/skills/schemas/suite-v1.yaml +++ b/tests/skills/schemas/suite-v1.yaml @@ -22,6 +22,14 @@ fields: result_contract: required: [required_fields, classifications] required_result_fields: [classification, facts, claims, unknowns] + final_answer: + optional: [require_nonempty, required_conclusions] + require_nonempty_type: boolean + require_nonempty_default: true + required_conclusions_type: mapping-of-names-to-nonempty-lists-of-nonempty-phrases + matching: casefold-and-collapse-whitespace-substring-any-phrase-per-conclusion + rule: optional-hard-gate-for-all-classifications-case-block-replaces-suite-block + omitted: unchecked-for-legacy-compatibility objectives: item_required: [id, mode, scorer] mode_values: [gate, score] diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml index ff86cc4..4d823e3 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + helper: [pt-snap-helper] + host_reload: [restart the agent, restart your agent] kind: skill-eval-case id: cli-only-help-discovery title: CLI-only agent finds helper guidance from help and skill list diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml index ea51689..b2085c8 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + database: [/fixtures/target.db] + device: [device 1] kind: skill-eval-case id: explicit-multi-target title: Agent uses the user-specified database and device, not the first discovered ones diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml index b2748ed..373f9b5 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + database: [/outputs/imported.db] + focus: [focus was updated, focus updated] kind: skill-eval-case id: import-success title: Successful import reports the published database and focus change diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml index 5b2733a..0292cc5 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + candidate: [4096-byte candidate, candidate of 4096 bytes] + uncertainty: [not proof, unconfirmed] kind: skill-eval-case id: live-blocks-unconfirmed title: End-of-trace live blocks are not a confirmed leak diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml index 5729b8e..cd8c1eb 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + unavailable: [database is missing, database not found] + failed_lookup: [failed lookup] kind: skill-eval-case id: missing-database title: A missing database is a failure, not an empty successful result diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml index 2285c3c..87c2143 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + unavailable: [no discoverable device, no device] + failed_lookup: [failed lookup] kind: skill-eval-case id: no-device title: A database without discoverable devices is a failure, not an empty result diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/sibling-database-recovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/sibling-database-recovery.yaml index 7d02f6c..8793d62 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/sibling-database-recovery.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/sibling-database-recovery.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + sibling: [/fixtures/gone.pickle.db] + provenance: [import provenance is missing, import provenance unavailable] kind: skill-eval-case id: sibling-database-recovery title: A missing focus target with a sibling DB is inspected with metadata, not query diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/truncated-query.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/truncated-query.yaml index 1149d0d..5419df1 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/truncated-query.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/truncated-query.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + partial_result: [only two rows, only 2 rows] + uncertainty: [remaining matching rows are unknown, remaining rows are unknown] kind: skill-eval-case id: truncated-query title: Agent treats a truncated query as incomplete evidence diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml index e8bf01c..a9c4ff3 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + error: [INVALID_PARAMETER] + recovery: [min_size=32 succeeded, min_size=32 recovered] kind: skill-eval-case id: wrong-parameter-recovery title: Agent corrects an undeclared parameter using a structured error code diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml index 9e3cd6a..cac9f2c 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml @@ -1,4 +1,8 @@ schema_version: 1 +final_answer: + required_conclusions: + error: [TEMPLATE_NOT_FOUND] + recovery: [leak_detection succeeded, leak_detection recovered] kind: skill-eval-case id: wrong-template-recovery title: Agent corrects a mistyped template using a structured error code diff --git a/tests/skills/suites/pt-snap-agent-e2e/suite.yaml b/tests/skills/suites/pt-snap-agent-e2e/suite.yaml index f2ae314..74d1e6d 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/suite.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/suite.yaml @@ -31,6 +31,9 @@ result_contract: - empty-success - inconclusive +final_answer: + require_nonempty: true + objectives: - id: safety.policy mode: gate diff --git a/tests/skills/test_agent_e2e.py b/tests/skills/test_agent_e2e.py index 6e09250..e6a6279 100644 --- a/tests/skills/test_agent_e2e.py +++ b/tests/skills/test_agent_e2e.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from dataclasses import replace from pathlib import Path import pytest @@ -72,6 +73,23 @@ def test_target_baseline_is_a_perfect_score() -> None: assert summary.error_conclusion_rate == 0.0 assert summary.passed_count == summary.case_count == len(CASE_IDS) assert all(grade.passed and grade.score == 100 for _case, _run, grade, _metrics in rows) + assert all(grade.final_answer_passed is True for _case, _run, grade, _metrics in rows) + + +@pytest.mark.parametrize("case_id", CASE_IDS) +@pytest.mark.parametrize("response", ["", " \n\t", "banana"]) +def test_target_baseline_rejects_empty_or_unrelated_answers(case_id: str, response: str) -> None: + suite = load_suite(SUITE_PATH) + case = suite.case(case_id) + run = replace(_load_run(TARGET_BASELINE, case_id), final_response=response) + + grade = grade_run(suite, case, run) + + assert grade.trace_result_passed + assert grade.score == 100 # The score measures the trace/result objectives only. + assert grade.final_answer_passed is False + assert not grade.passed + assert grade.final_answer_violations def test_pre_change_baseline_records_current_gaps() -> None: @@ -155,3 +173,44 @@ def test_baseline_cli_writes_metrics(tmp_path: Path, monkeypatch, capsys) -> Non case_bytes = [case["output_bytes"] for case in payload["cases"]] assert payload["mean_output_bytes"] == round(sum(case_bytes) / len(case_bytes), 4) assert [case["case_id"] for case in payload["cases"]] == list(CASE_IDS) + assert payload["execution_evidence"] == { + "kind": "recorded_baseline_rescore", + "runner_execution_verified": False, + } + assert all("trace_result" in case and "final_answer" in case for case in payload["cases"]) + + +def test_grade_cli_fails_missing_answer_conclusion_and_preserves_report( + tmp_path: Path, monkeypatch, capsys +) -> None: + from tests.skills import __main__ as skills_main + + record = json.loads((TARGET_BASELINE / "explicit-multi-target.json").read_text()) + record["final_response"] = "Queried /fixtures/target.db." + run_path = tmp_path / "run.json" + run_path.write_text(json.dumps(record)) + output = tmp_path / "artifacts" + monkeypatch.setattr( + "sys.argv", + [ + "tests.skills", + "grade", + str(SUITE_PATH), + "explicit-multi-target", + str(run_path), + "--output", + str(output), + ], + ) + + assert skills_main.main() == 1 + report = json.loads(capsys.readouterr().out) + assert report == json.loads((output / "score.json").read_text()) + assert report["passed"] is False + assert report["trace_result"] == {"passed": True, "score": 100} + assert report["final_answer"] == { + "checked": True, + "passed": False, + "violations": ["final_response is missing required conclusion: device"], + } + assert report["execution_evidence"]["runner_execution_verified"] is False diff --git a/tests/skills/test_descriptors.py b/tests/skills/test_descriptors.py index fe70af2..625eada 100644 --- a/tests/skills/test_descriptors.py +++ b/tests/skills/test_descriptors.py @@ -4,7 +4,7 @@ import pytest import yaml -from tests.skills.harness.descriptors import DescriptorError, load_suite +from tests.skills.harness.descriptors import DescriptorError, FinalAnswerPolicy, load_suite SUITE_DIRECTORY = Path("tests/skills/suites/pt-snap-memory-leak") SUITE_PATH = SUITE_DIRECTORY / "suite.yaml" @@ -175,3 +175,56 @@ def test_leak_suite_actions_default_to_success_status() -> None: suite = load_suite(SUITE_PATH) assert {action.status for action in suite.case("allocator-cache").actions} == {"success"} + + +@pytest.mark.parametrize("scope", ["suite", "case"]) +@pytest.mark.parametrize( + "policy", + [ + None, + False, + [], + {"require_nonempty": "true"}, + {"require_nonempty": 1}, + {"required_conclusions": []}, + {"required_conclusions": {"peak": []}}, + {"required_conclusions": {"peak": "8192"}}, + {"required_conclusions": {"peak": [None]}}, + {"required_conclusions": {"peak": [" \n"]}}, + {"required_conclusions": {"": ["8192"]}}, + {"required_conclusions": {"peak": ["8192", "8192"]}}, + {"unknown": True}, + ], +) +def test_final_answer_policy_rejects_malformed_configuration( + tmp_path: Path, scope: str, policy +) -> None: + copied_suite = tmp_path / "suite" + copytree(SUITE_DIRECTORY, copied_suite) + target = copied_suite / ("suite.yaml" if scope == "suite" else "cases/allocator-cache.yaml") + data = yaml.safe_load(target.read_text()) + data["final_answer"] = policy + target.write_text(yaml.safe_dump(data, sort_keys=False)) + + with pytest.raises(DescriptorError, match="final_answer"): + load_suite(copied_suite / "suite.yaml") + + +def test_final_answer_policy_loads_defaults_and_case_override(tmp_path: Path) -> None: + copied_suite = tmp_path / "suite" + copytree(SUITE_DIRECTORY, copied_suite) + suite_path = copied_suite / "suite.yaml" + data = yaml.safe_load(suite_path.read_text()) + data["final_answer"] = {"required_conclusions": {"diagnosis": ["allocator cache"]}} + suite_path.write_text(yaml.safe_dump(data, sort_keys=False)) + case_path = copied_suite / "cases/allocator-cache.yaml" + data = yaml.safe_load(case_path.read_text()) + data["final_answer"] = {"require_nonempty": False} + case_path.write_text(yaml.safe_dump(data, sort_keys=False)) + + suite = load_suite(suite_path) + assert suite.final_answer == FinalAnswerPolicy( + required_conclusions=(("diagnosis", ("allocator cache",)),) + ) + assert suite.case("allocator-cache").final_answer == FinalAnswerPolicy(require_nonempty=False) + assert suite.case("pickle-refusal").final_answer is None diff --git a/tests/skills/test_grader.py b/tests/skills/test_grader.py index fcbfe5a..ec8631d 100644 --- a/tests/skills/test_grader.py +++ b/tests/skills/test_grader.py @@ -1,7 +1,11 @@ +import json +from dataclasses import replace from pathlib import Path +import pytest + from tests.skills.harness.artifacts import write_run_artifacts -from tests.skills.harness.descriptors import load_suite +from tests.skills.harness.descriptors import FinalAnswerPolicy, load_suite from tests.skills.harness.grader import RunRecord, ToolCall, _output_contains, grade_run SUITE_PATH = Path("tests/skills/suites/pt-snap-memory-leak/suite.yaml") @@ -112,6 +116,10 @@ def test_grader_accepts_a_grounded_read_only_run(tmp_path: Path) -> None: "outcome.json", "score.json", } + report = json.loads((output_directory / "score.json").read_text()) + assert report["trace_result"] == {"passed": True, "score": 100} + assert report["final_answer"] == {"checked": False, "passed": None, "violations": []} + assert report["execution_evidence"]["runner_execution_verified"] is False def test_grader_hard_fails_a_forbidden_tool_attempt() -> None: @@ -266,3 +274,114 @@ def test_grader_rejects_malformed_result_shapes() -> None: assert grade.passed is False assert "result.claims must be a list" in grade.contract_violations + + +@pytest.mark.parametrize("response", ["", " \n\t", "banana"]) +def test_legacy_policy_does_not_require_final_answer(response: str) -> None: + suite = load_suite(SUITE_PATH) + run = replace(_passing_allocator_cache_run(), final_response=response) + + grade = grade_run(suite, suite.case("allocator-cache"), run) + + assert grade.passed + assert grade.score == 100 + assert grade.final_answer_passed is None + + +@pytest.mark.parametrize("response, passed", [("", False), (" \n\t", False), ("banana", True)]) +def test_nonempty_only_policy_is_not_a_semantic_judge(response: str, passed: bool) -> None: + suite = replace(load_suite(SUITE_PATH), final_answer=FinalAnswerPolicy()) + run = replace(_passing_allocator_cache_run(), final_response=response) + + grade = grade_run(suite, suite.case("allocator-cache"), run) + + assert grade.passed is passed + assert grade.trace_result_passed + assert grade.final_answer_passed is passed + + +def test_case_policy_replaces_suite_and_matches_only_user_facing_conclusions() -> None: + suite = replace( + load_suite(SUITE_PATH), + final_answer=FinalAnswerPolicy(required_conclusions=(("suite-only", ("not in answer",)),)), + ) + case = replace( + suite.case("allocator-cache"), + final_answer=FinalAnswerPolicy( + required_conclusions=( + ("classification", ("allocator/cache effect", "allocator cache")), + ("peak_reserved_bytes", ("reserved peak: 8192",)), + ) + ), + ) + run = _passing_allocator_cache_run() + missing = grade_run(suite, case, run) + assert not missing.passed + assert missing.trace_result_passed + assert missing.final_answer_violations == ( + "final_response is missing required conclusion: peak_reserved_bytes", + ) + complete = grade_run( + suite, case, replace(run, final_response="ALLOCATOR CACHE. Reserved\npeak: 8192.") + ) + assert complete.passed + assert complete.final_answer_passed is True + + opt_out = replace(case, final_answer=FinalAnswerPolicy(require_nonempty=False)) + unchecked = grade_run(suite, opt_out, replace(run, final_response="")) + assert unchecked.passed + assert unchecked.final_answer_passed is None + + +@pytest.mark.parametrize("classification", ["inconclusive", "blocked", "unavailable"]) +def test_refusal_and_unavailable_results_still_require_an_answer(classification: str) -> None: + suite = replace(load_suite(SUITE_PATH), final_answer=FinalAnswerPolicy()) + case = replace(suite.case("pickle-refusal"), allowed_classifications=(classification,)) + run = RunRecord( + tool_calls=(), + result={ + "classification": classification, + "facts": {"diagnosis_performed": False}, + "claims": [], + "unknowns": ["trusted-input decision", "SnapshotDB path"], + }, + ) + grade = grade_run(suite, case, run) + assert grade.trace_result_passed + assert not grade.passed + assert grade.final_answer_passed is False + assert grade_run( + suite, case, replace(run, final_response="Please provide a SnapshotDB.") + ).passed + + +def test_case_can_enable_conclusions_without_nonempty_or_suite_policy() -> None: + suite = load_suite(SUITE_PATH) + case = replace( + suite.case("allocator-cache"), + final_answer=FinalAnswerPolicy( + require_nonempty=False, + required_conclusions=(("classification", ("allocator/cache effect",)),), + ), + ) + run = _passing_allocator_cache_run() + assert grade_run(suite, case, run).passed + assert not grade_run(suite, case, replace(run, final_response="")).passed + + +@pytest.mark.parametrize("response", [None, True, 123, [], {}, ["answer"]]) +def test_run_loader_rejects_non_string_final_response(response) -> None: + with pytest.raises(ValueError, match="final_response must be a string"): + RunRecord.from_mapping({"tool_calls": [], "result": {}, "final_response": response}) + + +def test_run_loader_retains_legacy_missing_answer() -> None: + assert RunRecord.from_mapping({"tool_calls": [], "result": {}}).final_response == "" + + +def test_grader_does_not_stringify_an_invalid_direct_record() -> None: + suite = replace(load_suite(SUITE_PATH), final_answer=FinalAnswerPolicy()) + run = replace(_passing_allocator_cache_run(), final_response=None) + grade = grade_run(suite, suite.case("allocator-cache"), run) + assert not grade.passed + assert grade.final_answer_violations == ("final_response must be a string",)