From 6c5a01bff3af692f7fa09dba49676b74efa660d5 Mon Sep 17 00:00:00 2001 From: Oikantik Basu <140976895+Golden007-prog@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:22:04 +0530 Subject: [PATCH] fix(benchmark): bound HTML report history Signed-off-by: Oikantik Basu <140976895+Golden007-prog@users.noreply.github.com> --- .../benchmarks/authority/VALIDATION_REPORT.md | 6 ++ safeai/benchmark/report.py | 10 +++- tests/test_authority_benchmark.py | 55 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/docs/benchmarks/authority/VALIDATION_REPORT.md b/docs/benchmarks/authority/VALIDATION_REPORT.md index 313de36..fcb503d 100644 --- a/docs/benchmarks/authority/VALIDATION_REPORT.md +++ b/docs/benchmarks/authority/VALIDATION_REPORT.md @@ -62,6 +62,12 @@ write_html(evaluation, "authority-benchmark.html", history=history, label="current-revision") ``` +Supply at most 20 historical evaluations per HTML report. A larger history +raises `ValueError` before rendering or opening the output file; an existing +report remains unchanged. Select a smaller history explicitly rather than +relying on silent truncation. This bounds the number of embedded historical +briefs, not the size of an individual evaluation. + The current evaluation is appended after the supplied history. The trend table shows revision labels, corpus case counts, Lane-A status, end-to-end exercised authority attribution coverage (including matched diff --git a/safeai/benchmark/report.py b/safeai/benchmark/report.py index d619aa7..ade4680 100644 --- a/safeai/benchmark/report.py +++ b/safeai/benchmark/report.py @@ -500,6 +500,9 @@ def _history_html(snapshots: Sequence[tuple[str, dict]]) -> str: return "\n".join(out) +MAX_HTML_HISTORY = 20 + + def render_html(aggregate: dict, *, history: Sequence[tuple[str, dict]] = (), label: str = "Current") -> str: """Render an offline HTML brief with optional, explicitly ordered history. @@ -508,7 +511,11 @@ def render_html(aggregate: dict, *, history: Sequence[tuple[str, dict]] = (), precedes the current aggregate; nothing is sorted, scanned or fetched. The current brief includes its exact Markdown source for comparison; historical briefs retain their full text in expandable sections. + At most 20 historical evaluations are accepted; excess history raises + ValueError before any brief is rendered rather than silently truncating it. """ + if len(history) > MAX_HTML_HISTORY: + raise ValueError(f"HTML history accepts at most {MAX_HTML_HISTORY} evaluations") markdown = render_markdown(aggregate) out = ['', '', '', @@ -533,6 +540,7 @@ def write_html(aggregate: dict, path: str | os.PathLike, *, history: Sequence[tuple[str, dict]] = (), label: str = "Current") -> str | os.PathLike: """Write deterministic UTF-8 HTML with LF newlines on every platform.""" + document = render_html(aggregate, history=history, label=label) with open(path, "w", encoding="utf-8", newline="\n") as fh: - fh.write(render_html(aggregate, history=history, label=label)) + fh.write(document) return path diff --git a/tests/test_authority_benchmark.py b/tests/test_authority_benchmark.py index 03af847..e59c5ec 100644 --- a/tests/test_authority_benchmark.py +++ b/tests/test_authority_benchmark.py @@ -10,6 +10,8 @@ import re from html.parser import HTMLParser +import pytest + from safeai.benchmark import compare, metrics, report, runner REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -550,3 +552,56 @@ def test_html_history_keeps_explicit_order_counts_and_corpus_caveat(): assert "mix or truth do not prove scanner improvement" in html assert "Historical brief: z-old" in html assert "".join(page.markdown) == report.render_markdown(current) + + +def test_html_history_limit_preserves_order_at_the_boundary(): + history = [(f"revision-{index}", {}) for index in range(20)] + page = _BenchmarkHTML(report.render_html({}, history=history)) + assert [row[0] for row in page.rows if row[0].startswith("revision-")] == [ + label for label, _ in history] + + +@pytest.mark.parametrize("existing", [False, True]) +def test_html_excess_history_is_rejected_before_rendering_or_writing(tmp_path, monkeypatch, existing): + destination = tmp_path / "report.html" + if existing: + destination.write_bytes(b"previous report") + + def unexpected_render(_): + pytest.fail("excess history must be rejected before rendering any brief") + + monkeypatch.setattr(report, "render_markdown", unexpected_render) + history = [("revision", {})] * 21 + with pytest.raises(ValueError, match="at most 20"): + report.render_html({}, history=history) + with pytest.raises(ValueError, match="at most 20"): + report.write_html({}, destination, history=history) + if existing: + assert destination.read_bytes() == b"previous report" + else: + assert not destination.exists() + + +@pytest.mark.parametrize("payload", [ + '', + '', + '\">', + '', + 'click', + '<script>alert(1)</script>', +]) +def test_html_history_keeps_adversarial_labels_and_metrics_inert(payload): + aggregate = _committed_evaluation() + aggregate["digest"] = payload + aggregate["cases"] = [{"id": payload, "passed": False, "failures": [payload]}] + aggregate["metrics"]["attribution"]["end_to_end"] = payload + aggregate["metrics"]["canonical"]["false_escalation_rate"] = { + "status": "measured", "value": payload} + before = copy.deepcopy(aggregate) + page = _BenchmarkHTML(report.render_html(aggregate, history=[(payload, aggregate)], label=payload)) + assert not {tag for tag, _ in page.tags} & {"script", "svg", "img", "iframe", "a", "link"} + assert not any(key.startswith("on") for _, attrs in page.tags for key in attrs) + assert all(re.fullmatch(r"0(?:\.\d+)?|1(?:\.0+)?", m["value"]) for m in page.meters) + assert any(row[0] == payload for row in page.rows) + assert "".join(page.markdown) == report.render_markdown(aggregate) + assert aggregate == before