From fe17df909af6d500a7995a6af30177ac7acc2a22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 05:55:28 +0900 Subject: [PATCH 01/59] test(noema): add observed defect false-negative corpus --- ...ema_observed_defect_corpus_current_main.py | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tests/test_noema_observed_defect_corpus_current_main.py diff --git a/tests/test_noema_observed_defect_corpus_current_main.py b/tests/test_noema_observed_defect_corpus_current_main.py new file mode 100644 index 0000000000..da20fcb405 --- /dev/null +++ b/tests/test_noema_observed_defect_corpus_current_main.py @@ -0,0 +1,163 @@ +"""Executable regressions for observed Noema review false-negative shapes. + +These cases are grounded in externally demonstrated review findings rather than +claims of benchmark superiority. They keep the trusted review admission layer +honest about exact source coordinates and require the model prompt/validator to +attack more than one high-value defect class on material changes. +""" + +from __future__ import annotations + +import json + +import pytest + +from scripts.ci import noema_review_gate as noema + + +DIFF = """diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++new = 1 +""" + + +def _source_ref() -> dict[str, object]: + return {"path": "src/tool.py", "line": 1, "side": "RIGHT"} + + +def _class_evidence(kind: str) -> dict[str, dict[str, object]]: + return {field: _source_ref() for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind]} + + +def _probe(kind: str, *, hypothesis: str) -> dict[str, object]: + return { + **_source_ref(), + "probe_kind": kind, + "class_evidence": _class_evidence(kind), + "hypothesis": hypothesis, + "attack_or_counterexample": f"Attack {kind} at the exact changed line.", + "evidence": f"Observed source-bound evidence for {kind}.", + "outcome": "falsified", + } + + +def _verdict() -> dict[str, object]: + return { + "decision": "approve", + "summary": "Two independently classified defect shapes were attacked.", + "findings": [], + "reviewed_lines": [{**_source_ref(), "analysis": "Reviewed the exact changed line."}], + "adversarial_validation": { + "status": "passed", + "residual_risk": "No runtime integration exercise was available in this unit fixture.", + "probes": [ + _probe("mutable_alias", hypothesis="Caller-owned mutable state may escape validation."), + _probe( + "time_of_check_time_of_use", + hypothesis="A changing getter may differ between validation and use.", + ), + ], + }, + } + + +@pytest.mark.parametrize("container", [True, False]) +def test_boolean_reviewed_line_cannot_alias_integer_coordinate(container: bool) -> None: + verdict = _verdict() + verdict["reviewed_lines"][0]["line"] = container + + with pytest.raises(noema.NoemaModelOutputError, match="canonical positive integer line"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +@pytest.mark.parametrize("container", [True, False]) +def test_boolean_probe_line_cannot_alias_integer_coordinate(container: bool) -> None: + verdict = _verdict() + verdict["adversarial_validation"]["probes"][0]["line"] = container + + with pytest.raises(noema.NoemaModelOutputError, match="canonical positive integer line"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +def test_material_review_requires_distinct_observed_defect_classes() -> None: + verdict = _verdict() + verdict["adversarial_validation"]["probes"] = [ + _probe("mutable_alias", hypothesis="First mutable-alias wording."), + _probe("mutable_alias", hypothesis="Different prose, same defect shape."), + ] + + with pytest.raises(noema.NoemaModelOutputError, match="distinct probe_kind"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +@pytest.mark.parametrize("probe_kind", [[], {}, "unknown_shape"]) +def test_probe_kind_fails_closed_on_malformed_or_unknown_values(probe_kind: object) -> None: + verdict = _verdict() + verdict["adversarial_validation"]["probes"][0]["probe_kind"] = probe_kind + + with pytest.raises(noema.NoemaModelOutputError, match="observed defect taxonomy"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +def test_class_evidence_must_be_source_bound_to_the_probe_location() -> None: + verdict = _verdict() + probe = verdict["adversarial_validation"]["probes"][0] + probe["class_evidence"]["mutation_attempt"] = {"path": "src/tool.py", "line": 1, "side": "LEFT"} + + with pytest.raises(noema.NoemaModelOutputError, match="must bind to the probe location"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +def test_valid_observed_defect_taxonomy_verdict_is_accepted() -> None: + noema.validate_substantive_verdict(_verdict(), DIFF, ["src/tool.py"]) + + +def test_noema_prompt_names_every_observed_defect_class(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + monkeypatch.setattr(noema, "reject_private_llm_url", lambda _url: None) + monkeypatch.setattr(noema, "validate_substantive_verdict", lambda *_args: None) + monkeypatch.setattr( + noema, + "fetch_pr", + lambda _repo, _number: {"state": "OPEN", "headRefOid": "a" * 40}, + ) + seen: dict[str, object] = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + payload = {"choices": [{"message": {"content": json.dumps({"decision": "comment", "summary": "ok", "findings": []})}}]} + return json.dumps(payload).encode("utf-8") + + class Opener: + def open(self, request, timeout=None): + seen["request"] = json.loads(request.data.decode("utf-8")) + return Response() + + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) + pr = {"title": "fixture", "headRefOid": "a" * 40} + + noema.call_llm( + "owner/repo", + 7, + pr, + DIFF, + False, + "a" * 40, + changed_paths=["src/tool.py"], + ) + + prompt = seen["request"]["messages"][1]["content"] + for probe_kind in noema.OBSERVED_REVIEW_PROBE_KINDS: + assert probe_kind in prompt + assert "class_evidence" in prompt + assert "exact changed-side" in prompt From 95b66f935e81a4a5b7acd17c9cce430fae1a9a03 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 05:59:21 +0900 Subject: [PATCH 02/59] ci(temp): apply and verify PR1641 review-corpus repair --- ...mp_pr1641_noema_observed_corpus_repair.yml | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml new file mode 100644 index 0000000000..0e94c3fade --- /dev/null +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -0,0 +1,259 @@ +name: Temporary PR1641 Noema observed-corpus repair + +on: + push: + branches: + - fix/noema-observed-defect-corpus-current-main-20260902 + +permissions: + contents: write + +concurrency: + group: temp-pr1641-${{ github.ref }} + cancel-in-progress: true + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Checkout exact writer head without persisted credential + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Refuse a stale writer head + shell: bash + run: | + set -euo pipefail + live="$(git ls-remote https://github.com/ContextualWisdomLab/.github.git "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + test -n "$live" + test "$live" = "$GITHUB_SHA" + + - name: Install repository-declared pinned review test toolchain + run: >- + python3 -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt + + - name: Verify the committed regression is RED before production mutation + shell: bash + run: | + set -euo pipefail + set +e + PYTHONPATH=. python3 -m pytest -q tests/test_noema_observed_defect_corpus_current_main.py + red_rc=$? + set -e + if [ "$red_rc" -eq 0 ]; then + echo '::error::Expected the current-main regression corpus to fail before the causal repair.' + exit 1 + fi + echo "Verified RED regression corpus against pre-repair source (pytest rc=$red_rc)." + + - name: Apply smallest trusted validator and prompt repair + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + + path = Path('scripts/ci/noema_review_gate.py') + text = path.read_text(encoding='utf-8') + + def replace_once(old: str, new: str) -> None: + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'expected exactly one source anchor, found {count}: {old[:80]!r}') + text = text.replace(old, new, 1) + + constants_anchor = 'DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@")\n' + constants = '''DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@") + OBSERVED_REVIEW_PROBE_KINDS = frozenset( + { + "mutable_alias", + "time_of_check_time_of_use", + "execution_identity", + "coercion_boundary", + "test_oracle", + "cross_contract", + "authority_boundary", + "dependency_context", + "state_machine_race", + } + ) + OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { + "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), + "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), + "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), + "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), + "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), + "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), + "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), + "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), + "state_machine_race": ("initial_state", "event_order", "invariant_observation"), + } + ''' + replace_once(constants_anchor, constants) + + helper_anchor = ''' return value.removeprefix(prefix)\n\n\ndef validate_substantive_verdict(\n''' + helper = ''' return value.removeprefix(prefix) + + +def _canonical_changed_location(record: dict[str, Any], label: str) -> tuple[str, int, str]: + """Return a canonical changed-side location without bool/int coercion.""" + path_value = record.get("path") + line_value = record.get("line") + side_value = record.get("side") + if not isinstance(path_value, str) or not path_value.strip(): + raise NoemaModelOutputError(f"{label} requires a canonical changed-side path") + if type(line_value) is not int or line_value <= 0: + raise NoemaModelOutputError(f"{label} requires a canonical positive integer line") + if side_value not in {"LEFT", "RIGHT"}: + raise NoemaModelOutputError(f"{label} requires canonical LEFT/RIGHT side") + return (path_value, line_value, side_value) + + +def _validate_observed_probe_class_evidence( + probe: dict[str, Any], probe_kind: str, index: int, location: tuple[str, int, str] + ) -> None: + """Require defect-class witnesses to bind to the probe's exact changed line.""" + class_evidence = probe.get("class_evidence") + required_fields = OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[probe_kind] + if not isinstance(class_evidence, dict) or set(class_evidence) != set(required_fields): + expected = ", ".join(required_fields) + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence for {probe_kind} " + f"must contain exactly: {expected}" + ) + for field in required_fields: + source_ref = class_evidence.get(field) + if not isinstance(source_ref, dict) or set(source_ref) != {"path", "line", "side"}: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "source-bound changed-line reference" + ) + source_location = _canonical_changed_location( + source_ref, f"Noema adversarial probe {index} class_evidence.{field}" + ) + if source_location != location: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} must bind to " + "the probe location" + ) + + +def validate_substantive_verdict( + ''' + replace_once(helper_anchor, helper) + + replace_once( + ' location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side"))\n', + ' location = _canonical_changed_location(reviewed, f"Noema reviewed line {index}")\n', + ) + replace_once( + ' identities: set[tuple[Any, ...]] = set()\n for index, probe in enumerate(probes, start=1):\n', + ' identities: set[tuple[Any, ...]] = set()\n probe_kinds: set[str] = set()\n enforce_observed_taxonomy = bool(changed_paths)\n for index, probe in enumerate(probes, start=1):\n', + ) + replace_once( + ' location = (probe.get("path"), probe.get("line"), probe.get("side"))\n if location not in locations:\n', + ' location = _canonical_changed_location(probe, f"Noema adversarial probe {index}")\n if location not in locations:\n', + ) + replace_once( + ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', + ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n if enforce_observed_taxonomy:\n probe_kind = probe.get("probe_kind")\n if not isinstance(probe_kind, str) or probe_kind not in OBSERVED_REVIEW_PROBE_KINDS:\n raise NoemaModelOutputError(\n f"Noema adversarial probe {index} requires probe_kind from the observed defect taxonomy"\n )\n _validate_observed_probe_class_evidence(probe, probe_kind, index, location)\n probe_kinds.add(probe_kind)\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', + ) + replace_once( + ' if decision == "approve" and confirmed:\n', + ' if enforce_observed_taxonomy and len(probe_kinds) < required_probes:\n raise NoemaModelOutputError(\n f"Noema {decision} requires at least {required_probes} distinct probe_kind values"\n )\n\n if decision == "approve" and confirmed:\n', + ) + + replace_once( + ' **location_example,\n "hypothesis": "...",\n', + ' **location_example,\n "probe_kind": "mutable_alias",\n "class_evidence": {\n field: location_example\n for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"]\n },\n "hypothesis": "...",\n', + ) + replace_once( + ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; source or test changes require at least two distinct probes and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n', + ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; material source or test changes require at least two distinct probe_kind values and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n "Observed defect taxonomy and required source-bound class_evidence keys: "\n + json.dumps(\n {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()},\n sort_keys=True,\n separators=(",", ":"),\n ),\n "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.",\n', + ) + replace_once( + ' f"- `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', + ' f"- [{probe.get(\'probe_kind\') or \'legacy\'}] `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', + ) + + path.write_text(text, encoding='utf-8') + PY + + - name: Update operator and product traceability + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + + doctor = Path('docs/doctoring/noema-observed-defect-corpus-current-main.md') + doctor.write_text('''# Noema observed-defect review corpus\n\nThe trusted Noema review gate treats externally demonstrated review misses as executable regression evidence, not as benchmark claims. Material source/test reviews must exercise at least two distinct observed defect classes and every admitted class witness remains bound to an exact changed-side source coordinate.\n\nThe current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator.\n\nThe model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity.\n\nJSON booleans are rejected as line coordinates even though Python considers `True == 1`: changed-line evidence requires `type(line) is int` and a positive value. Production review calls always provide the complete changed-path manifest, which activates the observed taxonomy; direct validator unit tests may omit that manifest to exercise lower-level generic schema boundaries independently.\n\nThis repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim.\n''', encoding='utf-8') + + baseline = Path('docs/product-technical-gap-baseline.md') + baseline_text = baseline.read_text(encoding='utf-8') + marker = '### 2026-09-02 — Noema observed-defect false-negative corpus (#1641)' + if marker not in baseline_text: + baseline_text += f'''\n\n{marker}\n\n- **Verified gap:** protected current main admitted Noema adversarial evidence by count/prose identity and compared model line coordinates with Python integers without excluding booleans. Thus `true` could alias line `1`, and two differently worded probes could satisfy material-change diversity without proving distinct observed defect shapes.\n- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families.\n- **Regression evidence:** `tests/test_noema_observed_defect_corpus_current_main.py` is committed before the causal production change and covers boolean aliasing, malformed/unknown class labels, duplicate-class diversity, witness/source binding, a valid multi-class verdict, and rendered prompt coverage.\n- **Authority boundary:** no reviewer, provider, routing, merge, or repository-write authority is widened. The taxonomy is evaluation/admission evidence only.\n''' + baseline.write_text(baseline_text, encoding='utf-8') + + changelog = Path('CHANGELOG.md') + changelog_text = changelog.read_text(encoding='utf-8') + entry = '- **Require source-bound observed defect classes in Noema formal reviews (#1641).** Canonical changed-line coordinates now reject JSON booleans, material reviews must cover distinct classes from the executable external-finding corpus, class witnesses bind to exact changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure shapes without fabricating benchmark claims.\n' + if entry not in changelog_text: + anchor = '## [Unreleased]\n' + if changelog_text.count(anchor) != 1: + raise SystemExit('could not locate unique Unreleased changelog anchor') + changelog_text = changelog_text.replace(anchor, anchor + entry, 1) + changelog.write_text(changelog_text, encoding='utf-8') + PY + + - name: Verify GREEN focused and broader contracts + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python3 -m pytest -q \ + tests/test_noema_observed_defect_corpus_current_main.py \ + tests/test_noema_model_output_failure_classification.py \ + tests/test_noema_review_gate.py + PYTHONPATH=. python3 -m pytest -q + interrogate --fail-under=100 scripts/ci/noema_review_gate.py + python3 -m compileall -q scripts/ci tests + git diff --check + + - name: Remove temporary repair mechanism and commit publishable tree + shell: bash + run: | + set -euo pipefail + rm -f .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml + test ! -e .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml + git config user.name 'ContextualWisdomLab repair automation' + git config user.email 'actions@users.noreply.github.com' + git add scripts/ci/noema_review_gate.py \ + tests/test_noema_observed_defect_corpus_current_main.py \ + docs/doctoring/noema-observed-defect-corpus-current-main.md \ + docs/product-technical-gap-baseline.md \ + CHANGELOG.md \ + .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml + git diff --cached --check + git commit -m 'fix(noema): enforce observed defect-class evidence' + + - name: Push only if writer head is still exact + shell: bash + env: + GH_PUSH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + live="$(git ls-remote https://github.com/ContextualWisdomLab/.github.git "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + if [ "$live" != "$GITHUB_SHA" ]; then + echo "::error::Writer branch moved from $GITHUB_SHA to ${live:-missing}; refusing overwrite." + exit 75 + fi + git config core.hooksPath /dev/null + git remote set-url origin "https://x-access-token:${GH_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push origin "HEAD:${GITHUB_REF_NAME}" From e27d5c84dc05975ef2e4c4a4b9bc90c0002f550a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:04:32 +0900 Subject: [PATCH 03/59] fix(ci): repair PR1641 temporary writer workflow --- ...mp_pr1641_noema_observed_corpus_repair.yml | 177 +------------- .../ci/_temp_pr1641_apply_review_corpus.py | 216 ++++++++++++++++++ 2 files changed, 224 insertions(+), 169 deletions(-) create mode 100644 scripts/ci/_temp_pr1641_apply_review_corpus.py diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 0e94c3fade..9975460aa4 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -51,167 +51,8 @@ jobs: fi echo "Verified RED regression corpus against pre-repair source (pytest rc=$red_rc)." - - name: Apply smallest trusted validator and prompt repair - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - from pathlib import Path - - path = Path('scripts/ci/noema_review_gate.py') - text = path.read_text(encoding='utf-8') - - def replace_once(old: str, new: str) -> None: - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'expected exactly one source anchor, found {count}: {old[:80]!r}') - text = text.replace(old, new, 1) - - constants_anchor = 'DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@")\n' - constants = '''DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@") - OBSERVED_REVIEW_PROBE_KINDS = frozenset( - { - "mutable_alias", - "time_of_check_time_of_use", - "execution_identity", - "coercion_boundary", - "test_oracle", - "cross_contract", - "authority_boundary", - "dependency_context", - "state_machine_race", - } - ) - OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { - "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), - "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), - "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), - "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), - "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), - "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), - "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), - "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), - "state_machine_race": ("initial_state", "event_order", "invariant_observation"), - } - ''' - replace_once(constants_anchor, constants) - - helper_anchor = ''' return value.removeprefix(prefix)\n\n\ndef validate_substantive_verdict(\n''' - helper = ''' return value.removeprefix(prefix) - - -def _canonical_changed_location(record: dict[str, Any], label: str) -> tuple[str, int, str]: - """Return a canonical changed-side location without bool/int coercion.""" - path_value = record.get("path") - line_value = record.get("line") - side_value = record.get("side") - if not isinstance(path_value, str) or not path_value.strip(): - raise NoemaModelOutputError(f"{label} requires a canonical changed-side path") - if type(line_value) is not int or line_value <= 0: - raise NoemaModelOutputError(f"{label} requires a canonical positive integer line") - if side_value not in {"LEFT", "RIGHT"}: - raise NoemaModelOutputError(f"{label} requires canonical LEFT/RIGHT side") - return (path_value, line_value, side_value) - - -def _validate_observed_probe_class_evidence( - probe: dict[str, Any], probe_kind: str, index: int, location: tuple[str, int, str] - ) -> None: - """Require defect-class witnesses to bind to the probe's exact changed line.""" - class_evidence = probe.get("class_evidence") - required_fields = OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[probe_kind] - if not isinstance(class_evidence, dict) or set(class_evidence) != set(required_fields): - expected = ", ".join(required_fields) - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence for {probe_kind} " - f"must contain exactly: {expected}" - ) - for field in required_fields: - source_ref = class_evidence.get(field) - if not isinstance(source_ref, dict) or set(source_ref) != {"path", "line", "side"}: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "source-bound changed-line reference" - ) - source_location = _canonical_changed_location( - source_ref, f"Noema adversarial probe {index} class_evidence.{field}" - ) - if source_location != location: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} must bind to " - "the probe location" - ) - - -def validate_substantive_verdict( - ''' - replace_once(helper_anchor, helper) - - replace_once( - ' location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side"))\n', - ' location = _canonical_changed_location(reviewed, f"Noema reviewed line {index}")\n', - ) - replace_once( - ' identities: set[tuple[Any, ...]] = set()\n for index, probe in enumerate(probes, start=1):\n', - ' identities: set[tuple[Any, ...]] = set()\n probe_kinds: set[str] = set()\n enforce_observed_taxonomy = bool(changed_paths)\n for index, probe in enumerate(probes, start=1):\n', - ) - replace_once( - ' location = (probe.get("path"), probe.get("line"), probe.get("side"))\n if location not in locations:\n', - ' location = _canonical_changed_location(probe, f"Noema adversarial probe {index}")\n if location not in locations:\n', - ) - replace_once( - ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', - ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n if enforce_observed_taxonomy:\n probe_kind = probe.get("probe_kind")\n if not isinstance(probe_kind, str) or probe_kind not in OBSERVED_REVIEW_PROBE_KINDS:\n raise NoemaModelOutputError(\n f"Noema adversarial probe {index} requires probe_kind from the observed defect taxonomy"\n )\n _validate_observed_probe_class_evidence(probe, probe_kind, index, location)\n probe_kinds.add(probe_kind)\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', - ) - replace_once( - ' if decision == "approve" and confirmed:\n', - ' if enforce_observed_taxonomy and len(probe_kinds) < required_probes:\n raise NoemaModelOutputError(\n f"Noema {decision} requires at least {required_probes} distinct probe_kind values"\n )\n\n if decision == "approve" and confirmed:\n', - ) - - replace_once( - ' **location_example,\n "hypothesis": "...",\n', - ' **location_example,\n "probe_kind": "mutable_alias",\n "class_evidence": {\n field: location_example\n for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"]\n },\n "hypothesis": "...",\n', - ) - replace_once( - ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; source or test changes require at least two distinct probes and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n', - ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; material source or test changes require at least two distinct probe_kind values and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n "Observed defect taxonomy and required source-bound class_evidence keys: "\n + json.dumps(\n {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()},\n sort_keys=True,\n separators=(",", ":"),\n ),\n "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.",\n', - ) - replace_once( - ' f"- `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', - ' f"- [{probe.get(\'probe_kind\') or \'legacy\'}] `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', - ) - - path.write_text(text, encoding='utf-8') - PY - - - name: Update operator and product traceability - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - from pathlib import Path - - doctor = Path('docs/doctoring/noema-observed-defect-corpus-current-main.md') - doctor.write_text('''# Noema observed-defect review corpus\n\nThe trusted Noema review gate treats externally demonstrated review misses as executable regression evidence, not as benchmark claims. Material source/test reviews must exercise at least two distinct observed defect classes and every admitted class witness remains bound to an exact changed-side source coordinate.\n\nThe current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator.\n\nThe model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity.\n\nJSON booleans are rejected as line coordinates even though Python considers `True == 1`: changed-line evidence requires `type(line) is int` and a positive value. Production review calls always provide the complete changed-path manifest, which activates the observed taxonomy; direct validator unit tests may omit that manifest to exercise lower-level generic schema boundaries independently.\n\nThis repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim.\n''', encoding='utf-8') - - baseline = Path('docs/product-technical-gap-baseline.md') - baseline_text = baseline.read_text(encoding='utf-8') - marker = '### 2026-09-02 — Noema observed-defect false-negative corpus (#1641)' - if marker not in baseline_text: - baseline_text += f'''\n\n{marker}\n\n- **Verified gap:** protected current main admitted Noema adversarial evidence by count/prose identity and compared model line coordinates with Python integers without excluding booleans. Thus `true` could alias line `1`, and two differently worded probes could satisfy material-change diversity without proving distinct observed defect shapes.\n- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families.\n- **Regression evidence:** `tests/test_noema_observed_defect_corpus_current_main.py` is committed before the causal production change and covers boolean aliasing, malformed/unknown class labels, duplicate-class diversity, witness/source binding, a valid multi-class verdict, and rendered prompt coverage.\n- **Authority boundary:** no reviewer, provider, routing, merge, or repository-write authority is widened. The taxonomy is evaluation/admission evidence only.\n''' - baseline.write_text(baseline_text, encoding='utf-8') - - changelog = Path('CHANGELOG.md') - changelog_text = changelog.read_text(encoding='utf-8') - entry = '- **Require source-bound observed defect classes in Noema formal reviews (#1641).** Canonical changed-line coordinates now reject JSON booleans, material reviews must cover distinct classes from the executable external-finding corpus, class witnesses bind to exact changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure shapes without fabricating benchmark claims.\n' - if entry not in changelog_text: - anchor = '## [Unreleased]\n' - if changelog_text.count(anchor) != 1: - raise SystemExit('could not locate unique Unreleased changelog anchor') - changelog_text = changelog_text.replace(anchor, anchor + entry, 1) - changelog.write_text(changelog_text, encoding='utf-8') - PY + - name: Apply smallest trusted validator, prompt, and traceability repair + run: python3 scripts/ci/_temp_pr1641_apply_review_corpus.py - name: Verify GREEN focused and broader contracts shell: bash @@ -222,7 +63,7 @@ def validate_substantive_verdict( tests/test_noema_model_output_failure_classification.py \ tests/test_noema_review_gate.py PYTHONPATH=. python3 -m pytest -q - interrogate --fail-under=100 scripts/ci/noema_review_gate.py + interrogate --fail-under=100 scripts/ci/noema_review_gate.py scripts/ci/_temp_pr1641_apply_review_corpus.py python3 -m compileall -q scripts/ci tests git diff --check @@ -230,16 +71,14 @@ def validate_substantive_verdict( shell: bash run: | set -euo pipefail - rm -f .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml + rm -f \ + .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml \ + scripts/ci/_temp_pr1641_apply_review_corpus.py test ! -e .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml + test ! -e scripts/ci/_temp_pr1641_apply_review_corpus.py git config user.name 'ContextualWisdomLab repair automation' git config user.email 'actions@users.noreply.github.com' - git add scripts/ci/noema_review_gate.py \ - tests/test_noema_observed_defect_corpus_current_main.py \ - docs/doctoring/noema-observed-defect-corpus-current-main.md \ - docs/product-technical-gap-baseline.md \ - CHANGELOG.md \ - .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml + git add -A git diff --cached --check git commit -m 'fix(noema): enforce observed defect-class evidence' diff --git a/scripts/ci/_temp_pr1641_apply_review_corpus.py b/scripts/ci/_temp_pr1641_apply_review_corpus.py new file mode 100644 index 0000000000..bbdfd70de6 --- /dev/null +++ b/scripts/ci/_temp_pr1641_apply_review_corpus.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Temporary one-shot PR #1641 source repair; removed by its workflow.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SOURCE = Path("scripts/ci/noema_review_gate.py") + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace exactly one trusted source anchor or fail closed.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") + return text.replace(old, new, 1) + + +def apply_source_repair() -> None: + """Harden canonical locations and observed defect-class review evidence.""" + text = SOURCE.read_text(encoding="utf-8") + + constants_anchor = 'DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@")\n' + constants = '''DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@") +OBSERVED_REVIEW_PROBE_KINDS = frozenset( + { + "mutable_alias", + "time_of_check_time_of_use", + "execution_identity", + "coercion_boundary", + "test_oracle", + "cross_contract", + "authority_boundary", + "dependency_context", + "state_machine_race", + } +) +OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { + "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), + "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), + "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), + "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), + "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), + "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), + "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), + "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), + "state_machine_race": ("initial_state", "event_order", "invariant_observation"), +} +''' + text = replace_once(text, constants_anchor, constants, "taxonomy constants") + + helper_anchor = ''' return value.removeprefix(prefix)\n\n\ndef validate_substantive_verdict(\n''' + helper = ''' return value.removeprefix(prefix) + + +def _canonical_changed_location(record: dict[str, Any], label: str) -> tuple[str, int, str]: + """Return a canonical changed-side location without bool/int coercion.""" + path_value = record.get("path") + line_value = record.get("line") + side_value = record.get("side") + if not isinstance(path_value, str) or not path_value.strip(): + raise NoemaModelOutputError(f"{label} requires a canonical changed-side path") + if type(line_value) is not int or line_value <= 0: + raise NoemaModelOutputError(f"{label} requires a canonical positive integer line") + if side_value not in {"LEFT", "RIGHT"}: + raise NoemaModelOutputError(f"{label} requires canonical LEFT/RIGHT side") + return (path_value, line_value, side_value) + + +def _validate_observed_probe_class_evidence( + probe: dict[str, Any], probe_kind: str, index: int, location: tuple[str, int, str] +) -> None: + """Require defect-class witnesses to bind to the probe's exact changed line.""" + class_evidence = probe.get("class_evidence") + required_fields = OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[probe_kind] + if not isinstance(class_evidence, dict) or set(class_evidence) != set(required_fields): + expected = ", ".join(required_fields) + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence for {probe_kind} " + f"must contain exactly: {expected}" + ) + for field in required_fields: + source_ref = class_evidence.get(field) + if not isinstance(source_ref, dict) or set(source_ref) != {"path", "line", "side"}: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "source-bound changed-line reference" + ) + source_location = _canonical_changed_location( + source_ref, f"Noema adversarial probe {index} class_evidence.{field}" + ) + if source_location != location: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} must bind to " + "the probe location" + ) + + +def validate_substantive_verdict( +''' + text = replace_once(text, helper_anchor, helper, "location helpers") + + text = replace_once( + text, + ' location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side"))\n', + ' location = _canonical_changed_location(reviewed, f"Noema reviewed line {index}")\n', + "reviewed-line location", + ) + text = replace_once( + text, + ' identities: set[tuple[Any, ...]] = set()\n for index, probe in enumerate(probes, start=1):\n', + ' identities: set[tuple[Any, ...]] = set()\n probe_kinds: set[str] = set()\n enforce_observed_taxonomy = bool(changed_paths)\n for index, probe in enumerate(probes, start=1):\n', + "probe taxonomy state", + ) + text = replace_once( + text, + ' location = (probe.get("path"), probe.get("line"), probe.get("side"))\n if location not in locations:\n', + ' location = _canonical_changed_location(probe, f"Noema adversarial probe {index}")\n if location not in locations:\n', + "probe location", + ) + text = replace_once( + text, + ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', + ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n if enforce_observed_taxonomy:\n probe_kind = probe.get("probe_kind")\n if not isinstance(probe_kind, str) or probe_kind not in OBSERVED_REVIEW_PROBE_KINDS:\n raise NoemaModelOutputError(\n f"Noema adversarial probe {index} requires probe_kind from the observed defect taxonomy"\n )\n _validate_observed_probe_class_evidence(probe, probe_kind, index, location)\n probe_kinds.add(probe_kind)\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', + "probe class validation", + ) + text = replace_once( + text, + ' if decision == "approve" and confirmed:\n', + ' if enforce_observed_taxonomy and len(probe_kinds) < required_probes:\n raise NoemaModelOutputError(\n f"Noema {decision} requires at least {required_probes} distinct probe_kind values"\n )\n\n if decision == "approve" and confirmed:\n', + "probe diversity validation", + ) + + text = replace_once( + text, + ' **location_example,\n "hypothesis": "...",\n', + ' **location_example,\n "probe_kind": "mutable_alias",\n "class_evidence": {\n field: location_example\n for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"]\n },\n "hypothesis": "...",\n', + "prompt schema example", + ) + text = replace_once( + text, + ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; source or test changes require at least two distinct probes and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n', + ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; material source or test changes require at least two distinct probe_kind values and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n "Observed defect taxonomy and required source-bound class_evidence keys: "\n + json.dumps(\n {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()},\n sort_keys=True,\n separators=(",", ":"),\n ),\n "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.",\n', + "prompt taxonomy instruction", + ) + text = replace_once( + text, + ' f"- `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', + ' f"- [{probe.get(\'probe_kind\') or \'legacy\'}] `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', + "review evidence class", + ) + + ast.parse(text, filename=str(SOURCE)) + SOURCE.write_text(text, encoding="utf-8") + + +def update_docs() -> None: + """Record the operating contract and the externally observed regression corpus.""" + doctor = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") + doctor.write_text( + """# Noema observed-defect review corpus + +The trusted Noema review gate treats externally demonstrated review misses as executable regression evidence, not as benchmark claims. Material source/test reviews must exercise at least two distinct observed defect classes and every admitted class witness remains bound to an exact changed-side source coordinate. + +The current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator. + +The model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. + +JSON booleans are rejected as line coordinates even though Python considers `True == 1`: changed-line evidence requires `type(line) is int` and a positive value. Production review calls always provide the complete changed-path manifest, which activates the observed taxonomy; direct validator unit tests may omit that manifest to exercise lower-level generic schema boundaries independently. + +This repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim. +""", + encoding="utf-8", + ) + + baseline = Path("docs/product-technical-gap-baseline.md") + baseline_text = baseline.read_text(encoding="utf-8") + marker = "### 2026-09-02 — Noema observed-defect false-negative corpus (#1641)" + if marker not in baseline_text: + baseline_text += f""" + +{marker} + +- **Verified gap:** protected current main admitted Noema adversarial evidence by count/prose identity and compared model line coordinates with Python integers without excluding booleans. Thus `true` could alias line `1`, and two differently worded probes could satisfy material-change diversity without proving distinct observed defect shapes. +- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families. +- **Regression evidence:** `tests/test_noema_observed_defect_corpus_current_main.py` is committed before the causal production change and covers boolean aliasing, malformed/unknown class labels, duplicate-class diversity, witness/source binding, a valid multi-class verdict, and rendered prompt coverage. +- **Authority boundary:** no reviewer, provider, routing, merge, or repository-write authority is widened. The taxonomy is evaluation/admission evidence only. +""" + baseline.write_text(baseline_text, encoding="utf-8") + + changelog = Path("CHANGELOG.md") + changelog_text = changelog.read_text(encoding="utf-8") + entry = ( + "- **Require source-bound observed defect classes in Noema formal reviews (#1641).** " + "Canonical changed-line coordinates now reject JSON booleans, material reviews must cover " + "distinct classes from the executable external-finding corpus, class witnesses bind to exact " + "changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, " + "identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure " + "shapes without fabricating benchmark claims.\n" + ) + if entry not in changelog_text: + anchor = "## [Unreleased]\n" + if changelog_text.count(anchor) != 1: + raise SystemExit("could not locate unique Unreleased changelog anchor") + changelog.write_text(changelog_text.replace(anchor, anchor + entry, 1), encoding="utf-8") + + +def main() -> None: + """Apply the one-shot code and traceability repair.""" + apply_source_repair() + update_docs() + + +if __name__ == "__main__": + main() From eae62d120a1a6ebf691cca07622f04c490cc62ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:05:41 +0900 Subject: [PATCH 04/59] ci(temp): trigger repaired PR1641 writer --- scripts/ci/_temp_pr1641_apply_review_corpus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/_temp_pr1641_apply_review_corpus.py b/scripts/ci/_temp_pr1641_apply_review_corpus.py index bbdfd70de6..58d661f553 100644 --- a/scripts/ci/_temp_pr1641_apply_review_corpus.py +++ b/scripts/ci/_temp_pr1641_apply_review_corpus.py @@ -6,6 +6,7 @@ import ast from pathlib import Path +# Contents-API touch intentionally triggers the now-valid one-shot workflow. SOURCE = Path("scripts/ci/noema_review_gate.py") From f702789bedb0dff11b6c236709c6d313bb260987 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:18:10 +0900 Subject: [PATCH 05/59] ci(temp): gate PR1641 writer while fixing review findings --- ...mp_pr1641_noema_observed_corpus_repair.yml | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 9975460aa4..23fc79f592 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -14,7 +14,9 @@ concurrency: jobs: repair: - if: github.repository == 'ContextualWisdomLab/.github' + if: >- + github.repository == 'ContextualWisdomLab/.github' && + github.event.head_commit.message == 'ci(temp): execute repaired PR1641 writer' runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -37,33 +39,72 @@ jobs: run: >- python3 -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt - - name: Verify the committed regression is RED before production mutation + - name: Verify the committed taxonomy regression is specifically RED shell: bash run: | set -euo pipefail + red_log="${RUNNER_TEMP}/pr1641-taxonomy-red.log" set +e - PYTHONPATH=. python3 -m pytest -q tests/test_noema_observed_defect_corpus_current_main.py + PYTHONPATH=. python3 -m pytest -q tests/test_noema_observed_defect_corpus_current_main.py >"$red_log" 2>&1 red_rc=$? set -e + cat "$red_log" if [ "$red_rc" -eq 0 ]; then echo '::error::Expected the current-main regression corpus to fail before the causal repair.' exit 1 fi - echo "Verified RED regression corpus against pre-repair source (pytest rc=$red_rc)." + if ! grep -Fq "has no attribute 'OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS'" "$red_log"; then + echo '::error::RED failed for an unexpected reason; refusing production mutation.' + exit 1 + fi + if grep -Fq 'ERROR collecting' "$red_log"; then + echo '::error::RED was a collection/environment error rather than the expected missing-contract assertion path.' + exit 1 + fi + echo "Verified expected missing-taxonomy RED against pre-repair source (pytest rc=$red_rc)." - - name: Apply smallest trusted validator, prompt, and traceability repair + - name: Apply first-stage trusted validator and prompt repair run: python3 scripts/ci/_temp_pr1641_apply_review_corpus.py + - name: Prove generic relabeling remains RED before the review-followup repair + shell: bash + run: | + set -euo pipefail + red_log="${RUNNER_TEMP}/pr1641-generic-evidence-red.log" + set +e + PYTHONPATH=. python3 -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py::test_location_only_class_evidence_cannot_relabel_generic_probes \ + >"$red_log" 2>&1 + red_rc=$? + set -e + cat "$red_log" + if [ "$red_rc" -eq 0 ]; then + echo '::error::Expected location-only generic evidence to be accepted by the first-stage implementation.' + exit 1 + fi + if ! grep -Fq 'DID NOT RAISE' "$red_log"; then + echo '::error::Generic-evidence RED failed for an unexpected reason; refusing the follow-up production mutation.' + exit 1 + fi + echo "Verified generic relabeling RED against first-stage implementation (pytest rc=$red_rc)." + + - name: Apply review-followup observation-evidence repair + run: python3 scripts/ci/_temp_pr1641_finish_review_findings.py + - name: Verify GREEN focused and broader contracts shell: bash run: | set -euo pipefail PYTHONPATH=. python3 -m pytest -q \ tests/test_noema_observed_defect_corpus_current_main.py \ + tests/test_noema_class_evidence_observation_contract.py \ tests/test_noema_model_output_failure_classification.py \ tests/test_noema_review_gate.py PYTHONPATH=. python3 -m pytest -q - interrogate --fail-under=100 scripts/ci/noema_review_gate.py scripts/ci/_temp_pr1641_apply_review_corpus.py + interrogate --fail-under=100 \ + scripts/ci/noema_review_gate.py \ + scripts/ci/_temp_pr1641_apply_review_corpus.py \ + scripts/ci/_temp_pr1641_finish_review_findings.py python3 -m compileall -q scripts/ci tests git diff --check @@ -73,9 +114,11 @@ jobs: set -euo pipefail rm -f \ .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml \ - scripts/ci/_temp_pr1641_apply_review_corpus.py + scripts/ci/_temp_pr1641_apply_review_corpus.py \ + scripts/ci/_temp_pr1641_finish_review_findings.py test ! -e .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml test ! -e scripts/ci/_temp_pr1641_apply_review_corpus.py + test ! -e scripts/ci/_temp_pr1641_finish_review_findings.py git config user.name 'ContextualWisdomLab repair automation' git config user.email 'actions@users.noreply.github.com' git add -A From fe9b6da0a737ef3b5d75b4748ce179a84062ef73 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:18:27 +0900 Subject: [PATCH 06/59] test(noema): add generic class-evidence relabel regression --- ...ema_class_evidence_observation_contract.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/test_noema_class_evidence_observation_contract.py diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py new file mode 100644 index 0000000000..84e405d0dd --- /dev/null +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -0,0 +1,84 @@ +"""Regression tests for class-specific Noema probe observation evidence.""" + +from __future__ import annotations + +import pytest + +from scripts.ci import noema_review_gate as noema + + +DIFF = """diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++new = 1 +""" + + +def _location() -> dict[str, object]: + """Return the single exact changed-side location used by this fixture.""" + return {"path": "src/tool.py", "line": 1, "side": "RIGHT"} + + +def _class_evidence(kind: str, *, observations: bool, repeated: bool = False) -> dict[str, object]: + """Build class evidence with or without concrete observation text.""" + evidence: dict[str, object] = {} + for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind]: + witness = _location() + if observations: + witness["observation"] = ( + "same generic observation" if repeated else f"{kind}:{field} observed at the exact changed line" + ) + evidence[field] = witness + return evidence + + +def _probe(kind: str, *, observations: bool, repeated: bool = False) -> dict[str, object]: + """Build one adversarial probe for the requested observed defect class.""" + return { + **_location(), + "probe_kind": kind, + "class_evidence": _class_evidence(kind, observations=observations, repeated=repeated), + "hypothesis": f"Generic hypothesis relabeled as {kind}.", + "attack_or_counterexample": f"Generic attack relabeled as {kind}.", + "evidence": f"Probe evidence for {kind}.", + "outcome": "falsified", + } + + +def _verdict(*, observations: bool, repeated: bool = False) -> dict[str, object]: + """Build an otherwise-valid approval verdict with two distinct class labels.""" + return { + "decision": "approve", + "summary": "Two observed defect classes were attacked.", + "findings": [], + "reviewed_lines": [{**_location(), "analysis": "Reviewed exact changed line."}], + "adversarial_validation": { + "status": "passed", + "residual_risk": "Unit fixture does not exercise an external runtime.", + "probes": [ + _probe("mutable_alias", observations=observations, repeated=repeated), + _probe("time_of_check_time_of_use", observations=observations, repeated=repeated), + ], + }, + } + + +def test_location_only_class_evidence_cannot_relabel_generic_probes() -> None: + """Different taxonomy labels cannot make coordinate-only generic probes substantive.""" + with pytest.raises(noema.NoemaModelOutputError, match="non-empty observation"): + noema.validate_substantive_verdict(_verdict(observations=False), DIFF, ["src/tool.py"]) + + +def test_repeated_generic_observations_do_not_satisfy_class_specific_witnesses() -> None: + """A probe must provide distinct observations for its class-specific witness fields.""" + with pytest.raises(noema.NoemaModelOutputError, match="distinct class-specific observations"): + noema.validate_substantive_verdict( + _verdict(observations=True, repeated=True), DIFF, ["src/tool.py"] + ) + + +def test_distinct_source_bound_class_observations_are_accepted() -> None: + """Concrete distinct observations preserve an otherwise-valid multi-class verdict.""" + noema.validate_substantive_verdict(_verdict(observations=True), DIFF, ["src/tool.py"]) From 468c450dabf045732c31580e63f958d077b1d1d3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:19:01 +0900 Subject: [PATCH 07/59] fix(noema): stage concrete class-observation repair --- .../ci/_temp_pr1641_finish_review_findings.py | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 scripts/ci/_temp_pr1641_finish_review_findings.py diff --git a/scripts/ci/_temp_pr1641_finish_review_findings.py b/scripts/ci/_temp_pr1641_finish_review_findings.py new file mode 100644 index 0000000000..aeeb51d0c5 --- /dev/null +++ b/scripts/ci/_temp_pr1641_finish_review_findings.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Finish PR #1641 after proving the first-stage generic-evidence false negative.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SOURCE = Path("scripts/ci/noema_review_gate.py") +CORPUS_TEST = Path("tests/test_noema_observed_defect_corpus_current_main.py") +DOCTOR = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") +BASELINE = Path("docs/product-technical-gap-baseline.md") +CHANGELOG = Path("CHANGELOG.md") + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace one exact trusted anchor or stop instead of guessing.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") + return text.replace(old, new, 1) + + +def patch_validator() -> None: + """Require concrete distinct class observations in addition to coordinates.""" + text = SOURCE.read_text(encoding="utf-8") + old_block = ''' for field in required_fields: + source_ref = class_evidence.get(field) + if not isinstance(source_ref, dict) or set(source_ref) != {"path", "line", "side"}: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "source-bound changed-line reference" + ) + source_location = _canonical_changed_location( + source_ref, f"Noema adversarial probe {index} class_evidence.{field}" + ) + if source_location != location: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} must bind to " + "the probe location" + ) +''' + new_block = ''' normalized_observations: list[str] = [] + for field in required_fields: + source_ref = class_evidence.get(field) + if not isinstance(source_ref, dict) or set(source_ref) != { + "path", + "line", + "side", + "observation", + }: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "source-bound changed-line reference with a non-empty observation" + ) + observation = source_ref.get("observation") + if not isinstance(observation, str) or not observation.strip(): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "non-empty observation" + ) + if len(observation) > MAX_THREAD_BODY_CHARS: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + f"exceeds {MAX_THREAD_BODY_CHARS} characters" + ) + normalized_observations.append(observation.strip().casefold()) + source_location = _canonical_changed_location( + source_ref, f"Noema adversarial probe {index} class_evidence.{field}" + ) + if source_location != location: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} must bind to " + "the probe location" + ) + if len(set(normalized_observations)) != len(normalized_observations): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} requires distinct class-specific observations" + ) +''' + text = replace_once(text, old_block, new_block, "class observation validation") + + old_schema = ''' "class_evidence": { + field: location_example + for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] + }, +''' + new_schema = ''' "class_evidence": { + field: { + **location_example, + "observation": f"Concrete {field} observation at this changed line.", + } + for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] + }, +''' + text = replace_once(text, old_schema, new_schema, "prompt class-evidence schema") + + old_prompt = ''' "Observed defect taxonomy and required source-bound class_evidence keys: " + + json.dumps( + {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()}, + sort_keys=True, + separators=(",", ":"), + ), + "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", +''' + new_prompt = ''' "Observed defect taxonomy and required source-bound class_evidence keys: " + + json.dumps( + {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()}, + sort_keys=True, + separators=(",", ":"), + ), + "Every class_evidence witness must include path, line, side, and a non-empty concrete observation of that class-specific field at the cited changed line; observation strings within one probe must be distinct. Coordinates or taxonomy labels alone are not evidence.", + "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", +''' + text = replace_once(text, old_prompt, new_prompt, "prompt observation contract") + ast.parse(text, filename=str(SOURCE)) + SOURCE.write_text(text, encoding="utf-8") + + +def patch_regression_corpus() -> None: + """Update the original corpus fixtures to the stronger observation schema.""" + text = CORPUS_TEST.read_text(encoding="utf-8") + text = replace_once( + text, + '''def _class_evidence(kind: str) -> dict[str, dict[str, object]]: + return {field: _source_ref() for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind]} +''', + '''def _class_evidence(kind: str) -> dict[str, dict[str, object]]: + return { + field: { + **_source_ref(), + "observation": f"{kind}:{field} observed at the exact changed line.", + } + for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind] + } +''', + "corpus evidence fixture", + ) + text = replace_once( + text, + ''' probe["class_evidence"]["mutation_attempt"] = {"path": "src/tool.py", "line": 1, "side": "LEFT"} +''', + ''' probe["class_evidence"]["mutation_attempt"] = { + "path": "src/tool.py", + "line": 1, + "side": "LEFT", + "observation": "Mutation attempt observed on the wrong diff side.", + } +''', + "wrong-side corpus fixture", + ) + ast.parse(text, filename=str(CORPUS_TEST)) + CORPUS_TEST.write_text(text, encoding="utf-8") + + +def patch_traceability() -> None: + """Keep doctoring, baseline, and changelog aligned with executable evidence.""" + doctor = DOCTOR.read_text(encoding="utf-8") + doctor = replace_once( + doctor, + "Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator.", + "Witness values are exact `{path,line,side,observation}` records bound to the probe location; every observation must be non-empty, bounded, and distinct across that probe's class-specific witness fields, so coordinates or prose taxonomy labels alone do not satisfy the deterministic validator.", + "doctoring observation contract", + ) + DOCTOR.write_text(doctor, encoding="utf-8") + + baseline = BASELINE.read_text(encoding="utf-8") + baseline = replace_once( + baseline, + "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families.", + "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific source-bound witnesses that each carry non-empty bounded observations, reject repeated generic observations, and require distinct classes for material changes; the prompt actively attacks the same external-review failure families and states that coordinates or labels alone are insufficient evidence.", + "baseline observation contract", + ) + BASELINE.write_text(baseline, encoding="utf-8") + + changelog = CHANGELOG.read_text(encoding="utf-8") + changelog = replace_once( + changelog, + "class witnesses bind to exact changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, ", + "class witnesses bind to exact changed-side source coordinates with non-empty distinct observations, and the prompt explicitly attacks mutable-alias, TOCTOU, ", + "changelog observation contract", + ) + CHANGELOG.write_text(changelog, encoding="utf-8") + + +def main() -> None: + """Apply the independently demonstrated review-followup repair.""" + patch_validator() + patch_regression_corpus() + patch_traceability() + + +if __name__ == "__main__": + main() From 6e766c0a6dc81cb578432654a5b149785058b427 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:19:30 +0900 Subject: [PATCH 08/59] ci(temp): execute repaired PR1641 writer --- .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 23fc79f592..2384967801 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,4 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair +# execution nonce: review-followup-v1 on: push: From 0dcfa8f9dc0ac9053e52573a54f871df5aabb003 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:53:37 +0900 Subject: [PATCH 09/59] test(noema): reject vacuous class evidence --- ...ema_class_evidence_observation_contract.py | 104 +++++++++++++++--- 1 file changed, 90 insertions(+), 14 deletions(-) diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index 84e405d0dd..4fc3bd6a46 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -21,25 +21,52 @@ def _location() -> dict[str, object]: return {"path": "src/tool.py", "line": 1, "side": "RIGHT"} -def _class_evidence(kind: str, *, observations: bool, repeated: bool = False) -> dict[str, object]: - """Build class evidence with or without concrete observation text.""" +def _class_evidence( + kind: str, + *, + observations: bool, + repeated: bool = False, + source_excerpt: bool = False, + generic_but_different: bool = False, +) -> dict[str, object]: + """Build class evidence spanning the intentionally weak and hardened schemas.""" evidence: dict[str, object] = {} - for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind]: + for index, field in enumerate(noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind], start=1): witness = _location() if observations: - witness["observation"] = ( - "same generic observation" if repeated else f"{kind}:{field} observed at the exact changed line" - ) + if repeated: + witness["observation"] = "same generic observation" + elif generic_but_different: + witness["observation"] = f"Different generic concern number {index} appears in this area." + else: + witness["observation"] = ( + f"The `new` assignment preserves runtime relationship {index} relevant to {field}." + ) + if source_excerpt: + witness["source_excerpt"] = "new = 1" evidence[field] = witness return evidence -def _probe(kind: str, *, observations: bool, repeated: bool = False) -> dict[str, object]: +def _probe( + kind: str, + *, + observations: bool, + repeated: bool = False, + source_excerpt: bool = False, + generic_but_different: bool = False, +) -> dict[str, object]: """Build one adversarial probe for the requested observed defect class.""" return { **_location(), "probe_kind": kind, - "class_evidence": _class_evidence(kind, observations=observations, repeated=repeated), + "class_evidence": _class_evidence( + kind, + observations=observations, + repeated=repeated, + source_excerpt=source_excerpt, + generic_but_different=generic_but_different, + ), "hypothesis": f"Generic hypothesis relabeled as {kind}.", "attack_or_counterexample": f"Generic attack relabeled as {kind}.", "evidence": f"Probe evidence for {kind}.", @@ -47,7 +74,13 @@ def _probe(kind: str, *, observations: bool, repeated: bool = False) -> dict[str } -def _verdict(*, observations: bool, repeated: bool = False) -> dict[str, object]: +def _verdict( + *, + observations: bool, + repeated: bool = False, + source_excerpt: bool = False, + generic_but_different: bool = False, +) -> dict[str, object]: """Build an otherwise-valid approval verdict with two distinct class labels.""" return { "decision": "approve", @@ -58,8 +91,20 @@ def _verdict(*, observations: bool, repeated: bool = False) -> dict[str, object] "status": "passed", "residual_risk": "Unit fixture does not exercise an external runtime.", "probes": [ - _probe("mutable_alias", observations=observations, repeated=repeated), - _probe("time_of_check_time_of_use", observations=observations, repeated=repeated), + _probe( + "mutable_alias", + observations=observations, + repeated=repeated, + source_excerpt=source_excerpt, + generic_but_different=generic_but_different, + ), + _probe( + "time_of_check_time_of_use", + observations=observations, + repeated=repeated, + source_excerpt=source_excerpt, + generic_but_different=generic_but_different, + ), ], }, } @@ -75,10 +120,41 @@ def test_repeated_generic_observations_do_not_satisfy_class_specific_witnesses() """A probe must provide distinct observations for its class-specific witness fields.""" with pytest.raises(noema.NoemaModelOutputError, match="distinct class-specific observations"): noema.validate_substantive_verdict( - _verdict(observations=True, repeated=True), DIFF, ["src/tool.py"] + _verdict(observations=True, repeated=True, source_excerpt=True), + DIFF, + ["src/tool.py"], ) +def test_differently_worded_generic_observations_without_source_signal_are_rejected() -> None: + """Unique prose labels are not evidence unless they name concrete changed-source content.""" + with pytest.raises(noema.NoemaModelOutputError, match="concrete token from source_excerpt"): + noema.validate_substantive_verdict( + _verdict( + observations=True, + source_excerpt=True, + generic_but_different=True, + ), + DIFF, + ["src/tool.py"], + ) + + +def test_fabricated_source_excerpt_is_rejected() -> None: + """A model cannot attach a plausible observation to source text absent from the cited line.""" + verdict = _verdict(observations=True, source_excerpt=True) + verdict["adversarial_validation"]["probes"][0]["class_evidence"]["mutation_attempt"][ + "source_excerpt" + ] = "fabricated = 2" + + with pytest.raises(noema.NoemaModelOutputError, match="exact changed-line source_excerpt"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + def test_distinct_source_bound_class_observations_are_accepted() -> None: - """Concrete distinct observations preserve an otherwise-valid multi-class verdict.""" - noema.validate_substantive_verdict(_verdict(observations=True), DIFF, ["src/tool.py"]) + """Concrete source-backed observations preserve an otherwise-valid multi-class verdict.""" + noema.validate_substantive_verdict( + _verdict(observations=True, source_excerpt=True), + DIFF, + ["src/tool.py"], + ) From d8eb254eac4cf74bc33988ec32a8361ee49be3ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:53:55 +0900 Subject: [PATCH 10/59] test(noema): preserve workflow-trigger review regression --- tests/test_noema_observed_defect_corpus_current_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_noema_observed_defect_corpus_current_main.py b/tests/test_noema_observed_defect_corpus_current_main.py index da20fcb405..df5ca5622b 100644 --- a/tests/test_noema_observed_defect_corpus_current_main.py +++ b/tests/test_noema_observed_defect_corpus_current_main.py @@ -161,3 +161,6 @@ def open(self, request, timeout=None): assert probe_kind in prompt assert "class_evidence" in prompt assert "exact changed-side" in prompt + assert "source_excerpt" in prompt + assert "workflow-starting credential" in prompt + assert "downstream required checks" in prompt From 10f6d11873fc79a3629417ccec5963b881430b8e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:56:06 +0900 Subject: [PATCH 11/59] fix(noema): bind class evidence to exact source --- .../ci/_temp_pr1641_finish_review_findings.py | 190 ++++++++++++++++-- 1 file changed, 170 insertions(+), 20 deletions(-) diff --git a/scripts/ci/_temp_pr1641_finish_review_findings.py b/scripts/ci/_temp_pr1641_finish_review_findings.py index aeeb51d0c5..0275269c58 100644 --- a/scripts/ci/_temp_pr1641_finish_review_findings.py +++ b/scripts/ci/_temp_pr1641_finish_review_findings.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Finish PR #1641 after proving the first-stage generic-evidence false negative.""" +"""Finish PR #1641 after proving first-stage review-evidence false negatives.""" from __future__ import annotations @@ -22,7 +22,7 @@ def replace_once(text: str, old: str, new: str, label: str) -> str: def patch_validator() -> None: - """Require concrete distinct class observations in addition to coordinates.""" + """Require exact source excerpts and non-vacuous class observations.""" text = SOURCE.read_text(encoding="utf-8") old_block = ''' for field in required_fields: source_ref = class_evidence.get(field) @@ -41,17 +41,39 @@ def patch_validator() -> None: ) ''' new_block = ''' normalized_observations: list[str] = [] + source_texts = changed_diff_line_texts(diff) for field in required_fields: source_ref = class_evidence.get(field) if not isinstance(source_ref, dict) or set(source_ref) != { "path", "line", "side", + "source_excerpt", "observation", }: raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "source-bound changed-line reference with a non-empty observation" + f"Noema adversarial probe {index} class_evidence.{field} requires " + "path, line, side, exact source_excerpt, and observation" + ) + source_location = _canonical_changed_location( + source_ref, f"Noema adversarial probe {index} class_evidence.{field}" + ) + if source_location != location: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} must bind to " + "the probe location" + ) + expected_excerpt = source_texts.get(source_location) + source_excerpt = source_ref.get("source_excerpt") + if ( + not isinstance(source_excerpt, str) + or not source_excerpt.strip() + or expected_excerpt is None + or source_excerpt != expected_excerpt + ): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires the " + "exact changed-line source_excerpt" ) observation = source_ref.get("observation") if not isinstance(observation, str) or not observation.strip(): @@ -64,15 +86,52 @@ def patch_validator() -> None: f"Noema adversarial probe {index} class_evidence.{field} observation " f"exceeds {MAX_THREAD_BODY_CHARS} characters" ) - normalized_observations.append(observation.strip().casefold()) - source_location = _canonical_changed_location( - source_ref, f"Noema adversarial probe {index} class_evidence.{field}" - ) - if source_location != location: + source_tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", source_excerpt) + } + observation_tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", observation) + } + if not source_tokens or not source_tokens.intersection(observation_tokens): raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} must bind to " - "the probe location" + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must name a concrete token from source_excerpt" + ) + label_tokens = { + token.casefold() + for token in re.findall( + r"[A-Za-z_][A-Za-z0-9_]{2,}", + f"{probe_kind} {field}".replace("_", " "), + ) + } + filler_tokens = { + "area", + "changed", + "concern", + "concrete", + "evidence", + "exact", + "generic", + "here", + "line", + "nearby", + "observed", + "observation", + "probe", + "review", + "source", + "this", + "value", + } + causal_tokens = observation_tokens - source_tokens - label_tokens - filler_tokens + if not causal_tokens: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "concrete causal observation beyond source and taxonomy labels" ) + normalized_observations.append(observation.strip().casefold()) if len(set(normalized_observations)) != len(normalized_observations): raise NoemaModelOutputError( f"Noema adversarial probe {index} requires distinct class-specific observations" @@ -80,6 +139,82 @@ def patch_validator() -> None: ''' text = replace_once(text, old_block, new_block, "class observation validation") + text = replace_once( + text, + '''def _validate_observed_probe_class_evidence( + probe: dict[str, Any], probe_kind: str, index: int, location: tuple[str, int, str] +) -> None: +''', + '''def _validate_observed_probe_class_evidence( + probe: dict[str, Any], + probe_kind: str, + index: int, + location: tuple[str, int, str], + diff: str, +) -> None: +''', + "class evidence validator signature", + ) + + helper_anchor = ''' return locations + + +def parse_diff_path(raw: str, prefix: str) -> str: +''' + helper = ''' return locations + + +def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: + """Return exact changed-side source text keyed by canonical diff location.""" + texts: dict[tuple[str, int, str], str] = {} + old_path = new_path = "" + old_line = new_line = 0 + in_hunk = False + for raw_line in diff.splitlines(): + if raw_line.startswith("diff --git "): + old_path = new_path = "" + in_hunk = False + continue + if not in_hunk and raw_line.startswith("--- "): + old_path = parse_diff_path(raw_line[4:], "a/") + continue + if not in_hunk and raw_line.startswith("+++ "): + new_path = parse_diff_path(raw_line[4:], "b/") + continue + match = DIFF_HUNK_RE.match(raw_line) + if match: + old_line, new_line = map(int, match.groups()) + in_hunk = True + continue + if not in_hunk or raw_line.startswith("\\ No newline"): + continue + if raw_line.startswith("+"): + if not new_path: + return {} + texts[(new_path, new_line, "RIGHT")] = raw_line[1:] + new_line += 1 + elif raw_line.startswith("-"): + if not old_path: + return {} + texts[(old_path, old_line, "LEFT")] = raw_line[1:] + old_line += 1 + else: + old_line += 1 + new_line += 1 + return texts + + +def parse_diff_path(raw: str, prefix: str) -> str: +''' + text = replace_once(text, helper_anchor, helper, "changed-line source helper") + + text = replace_once( + text, + " _validate_observed_probe_class_evidence(probe, probe_kind, index, location)\n", + " _validate_observed_probe_class_evidence(probe, probe_kind, index, location, diff)\n", + "class evidence validator call", + ) + old_schema = ''' "class_evidence": { field: location_example for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] @@ -88,7 +223,11 @@ def patch_validator() -> None: new_schema = ''' "class_evidence": { field: { **location_example, - "observation": f"Concrete {field} observation at this changed line.", + "source_excerpt": "exact changed-line text", + "observation": ( + f"Concrete {field} causal observation naming a token " + "from source_excerpt." + ), } for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] }, @@ -109,8 +248,8 @@ def patch_validator() -> None: sort_keys=True, separators=(",", ":"), ), - "Every class_evidence witness must include path, line, side, and a non-empty concrete observation of that class-specific field at the cited changed line; observation strings within one probe must be distinct. Coordinates or taxonomy labels alone are not evidence.", - "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", + "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line. The observation must name a concrete token from that source_excerpt and explain a causal or behavioral relation beyond taxonomy labels; differently worded generic labels are not evidence.", + "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. For automation or CI that mutates a branch or source and then relies on later events, verify that the mutation uses a workflow-starting credential/actor and that downstream required checks can actually be created on the successor head. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", ''' text = replace_once(text, old_prompt, new_prompt, "prompt observation contract") ast.parse(text, filename=str(SOURCE)) @@ -129,9 +268,15 @@ def patch_regression_corpus() -> None: return { field: { **_source_ref(), - "observation": f"{kind}:{field} observed at the exact changed line.", + "source_excerpt": "new = 1", + "observation": ( + f"The `new` assignment preserves runtime relationship {index} relevant to {field}." + ), } - for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind] + for index, field in enumerate( + noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind], + start=1, + ) } ''', "corpus evidence fixture", @@ -144,7 +289,8 @@ def patch_regression_corpus() -> None: "path": "src/tool.py", "line": 1, "side": "LEFT", - "observation": "Mutation attempt observed on the wrong diff side.", + "source_excerpt": "old = 1", + "observation": "The `old` assignment is removed before the attempted mutation relationship.", } ''', "wrong-side corpus fixture", @@ -159,16 +305,20 @@ def patch_traceability() -> None: doctor = replace_once( doctor, "Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator.", - "Witness values are exact `{path,line,side,observation}` records bound to the probe location; every observation must be non-empty, bounded, and distinct across that probe's class-specific witness fields, so coordinates or prose taxonomy labels alone do not satisfy the deterministic validator.", + "Witness values are `{path,line,side,source_excerpt,observation}` records bound to the probe location. `source_excerpt` must equal the exact changed-side line, and `observation` must name a concrete source token plus a causal/behavioral relation beyond taxonomy labels; repeated or differently worded generic labels do not satisfy the deterministic validator.", "doctoring observation contract", ) + doctor = doctor.replace( + "A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity.", + "A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. For CI/automation changes, the review prompt also requires checking whether the mutation credential can create the downstream events/checks the state machine depends on.", + ) DOCTOR.write_text(doctor, encoding="utf-8") baseline = BASELINE.read_text(encoding="utf-8") baseline = replace_once( baseline, "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families.", - "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific source-bound witnesses that each carry non-empty bounded observations, reject repeated generic observations, and require distinct classes for material changes; the prompt actively attacks the same external-review failure families and states that coordinates or labels alone are insufficient evidence.", + "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific source-bound witnesses whose exact `source_excerpt` must match the cited changed line and whose observation must name concrete source content plus causal behavior. Material changes require distinct classes, and the prompt explicitly checks workflow-starting mutation credentials before relying on downstream required checks.", "baseline observation contract", ) BASELINE.write_text(baseline, encoding="utf-8") @@ -177,7 +327,7 @@ def patch_traceability() -> None: changelog = replace_once( changelog, "class witnesses bind to exact changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, ", - "class witnesses bind to exact changed-side source coordinates with non-empty distinct observations, and the prompt explicitly attacks mutable-alias, TOCTOU, ", + "class witnesses bind to exact changed-side source text with non-vacuous causal observations, and the prompt explicitly attacks workflow-event authority plus mutable-alias, TOCTOU, ", "changelog observation contract", ) CHANGELOG.write_text(changelog, encoding="utf-8") From d0c8b1f8edb08f3ad7a63d69b5dbc5c2cd802ee6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:57:10 +0900 Subject: [PATCH 12/59] fix(noema): harden nonvacuous review evidence --- scripts/ci/_temp_pr1641_finish_review_findings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/_temp_pr1641_finish_review_findings.py b/scripts/ci/_temp_pr1641_finish_review_findings.py index 0275269c58..ad8b46a5d3 100644 --- a/scripts/ci/_temp_pr1641_finish_review_findings.py +++ b/scripts/ci/_temp_pr1641_finish_review_findings.py @@ -53,7 +53,7 @@ def patch_validator() -> None: }: raise NoemaModelOutputError( f"Noema adversarial probe {index} class_evidence.{field} requires " - "path, line, side, exact source_excerpt, and observation" + "path, line, side, exact source_excerpt, and non-empty observation" ) source_location = _canonical_changed_location( source_ref, f"Noema adversarial probe {index} class_evidence.{field}" @@ -99,7 +99,7 @@ def patch_validator() -> None: f"Noema adversarial probe {index} class_evidence.{field} observation " "must name a concrete token from source_excerpt" ) - label_tokens = { + label_tokens = {probe_kind.casefold(), field.casefold()} | { token.casefold() for token in re.findall( r"[A-Za-z_][A-Za-z0-9_]{2,}", From 8c1c82a10abece7dc46b48bc49a5bfe742c06e50 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:57:53 +0900 Subject: [PATCH 13/59] ci(temp): execute repaired PR1641 writer --- ...emp_pr1641_noema_observed_corpus_repair.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 2384967801..480d2296d4 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v1 +# execution nonce: review-followup-v2 on: push: @@ -129,7 +129,8 @@ jobs: - name: Push only if writer head is still exact shell: bash env: - GH_PUSH_TOKEN: ${{ github.token }} + PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }} + OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }} run: | set -euo pipefail live="$(git ls-remote https://github.com/ContextualWisdomLab/.github.git "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" @@ -137,6 +138,17 @@ jobs: echo "::error::Writer branch moved from $GITHUB_SHA to ${live:-missing}; refusing overwrite." exit 75 fi + if [ -n "${PR_REVIEW_MERGE_TOKEN:-}" ]; then + push_token="$PR_REVIEW_MERGE_TOKEN" + push_source='PR_REVIEW_MERGE_TOKEN' + elif [ -n "${OPENCODE_APPROVE_TOKEN:-}" ]; then + push_token="$OPENCODE_APPROVE_TOKEN" + push_source='OPENCODE_APPROVE_TOKEN' + else + echo '::error::No workflow-starting branch-mutation credential is configured. Refusing a github.token push because it would suppress successor-head required workflows.' + exit 78 + fi + echo "Using workflow-starting mutation credential source: $push_source" git config core.hooksPath /dev/null - git remote set-url origin "https://x-access-token:${GH_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git remote set-url origin "https://x-access-token:${push_token}@github.com/${GITHUB_REPOSITORY}.git" git push origin "HEAD:${GITHUB_REF_NAME}" From e1a4406f3ea2b1b43366953f50c0d5e2c8814434 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:58:41 +0900 Subject: [PATCH 14/59] test(noema): avoid accidental source-token match --- tests/test_noema_class_evidence_observation_contract.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index 4fc3bd6a46..dda7481494 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -37,7 +37,9 @@ def _class_evidence( if repeated: witness["observation"] = "same generic observation" elif generic_but_different: - witness["observation"] = f"Different generic concern number {index} appears in this area." + witness["observation"] = ( + f"Generic {field.replace('_', ' ')} concern appears in this area." + ) else: witness["observation"] = ( f"The `new` assignment preserves runtime relationship {index} relevant to {field}." From b501d266a7de6d9c34d64893fb08d5aa82ff5336 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 07:00:26 +0900 Subject: [PATCH 15/59] test(noema): keep duplicate observation regression precise --- tests/test_noema_class_evidence_observation_contract.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index dda7481494..6e596f05ee 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -35,7 +35,9 @@ def _class_evidence( witness = _location() if observations: if repeated: - witness["observation"] = "same generic observation" + witness["observation"] = ( + "The `new` assignment preserves one repeated runtime relationship." + ) elif generic_but_different: witness["observation"] = ( f"Generic {field.replace('_', ' ')} concern appears in this area." From b046568e21893166231d7a5ccb6b8b12f8ca4230 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 07:01:04 +0900 Subject: [PATCH 16/59] ci(temp): execute repaired PR1641 writer --- .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 480d2296d4..23091ef153 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v2 +# execution nonce: review-followup-v3 on: push: From 579d5a316f1b67294298f597ac4d2e60d1268fed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:18:41 +0900 Subject: [PATCH 17/59] fix(noema): add exact-source follow-up repair --- ...mp_pr1641_finish_review_findings_round2.py | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 scripts/ci/_temp_pr1641_finish_review_findings_round2.py diff --git a/scripts/ci/_temp_pr1641_finish_review_findings_round2.py b/scripts/ci/_temp_pr1641_finish_review_findings_round2.py new file mode 100644 index 0000000000..b2c210a925 --- /dev/null +++ b/scripts/ci/_temp_pr1641_finish_review_findings_round2.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Finish PR #1641 after exact-head review exposed source-binding edge cases.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SOURCE = Path("scripts/ci/noema_review_gate.py") +CORPUS_TEST = Path("tests/test_noema_observed_defect_corpus_current_main.py") +DOCTOR = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") +BASELINE = Path("docs/product-technical-gap-baseline.md") +CHANGELOG = Path("CHANGELOG.md") + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace exactly one trusted generated-source anchor or fail closed.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") + return text.replace(old, new, 1) + + +def patch_validator() -> None: + """Bind evidence to exact source text without ASCII/token-shape heuristics.""" + text = SOURCE.read_text(encoding="utf-8") + + text = replace_once( + text, + ''' if ( + not isinstance(source_excerpt, str) + or not source_excerpt.strip() + or expected_excerpt is None + or source_excerpt != expected_excerpt + ): +''', + ''' if ( + not isinstance(source_excerpt, str) + or expected_excerpt is None + or source_excerpt != expected_excerpt + ): +''', + "blank exact-source admission", + ) + + old_tokens = ''' source_tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", source_excerpt) + } + observation_tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", observation) + } + if not source_tokens or not source_tokens.intersection(observation_tokens): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must name a concrete token from source_excerpt" + ) + label_tokens = {probe_kind.casefold(), field.casefold()} | { + token.casefold() + for token in re.findall( + r"[A-Za-z_][A-Za-z0-9_]{2,}", + f"{probe_kind} {field}".replace("_", " "), + ) + } + filler_tokens = { + "area", + "changed", + "concern", + "concrete", + "evidence", + "exact", + "generic", + "here", + "line", + "nearby", + "observed", + "observation", + "probe", + "review", + "source", + "this", + "value", + } + causal_tokens = observation_tokens - source_tokens - label_tokens - filler_tokens + if not causal_tokens: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "concrete causal observation beyond source and taxonomy labels" + ) +''' + new_tokens = ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + relation_tokens = { + "accepts", + "after", + "aliases", + "allows", + "before", + "because", + "blocks", + "bypasses", + "cancels", + "causes", + "changes", + "conflicts", + "depends", + "differs", + "escapes", + "fails", + "mismatches", + "mutates", + "prevents", + "preserves", + "races", + "reads", + "rejects", + "relationship", + "reuses", + "shares", + "truncates", + "when", + "while", + "without", + "writes", + } + observation_tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}", observation) + } + if not relation_tokens.intersection(observation_tokens): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "causal relationship, not an arbitrary source-adjacent word" + ) +''' + text = replace_once(text, old_tokens, new_tokens, "causal source binding") + + text = replace_once( + text, + ''' if raw_line.startswith("+"): + if not new_path: + return {} + texts[(new_path, new_line, "RIGHT")] = raw_line[1:] + new_line += 1 + elif raw_line.startswith("-"): + if not old_path: + return {} + texts[(old_path, old_line, "LEFT")] = raw_line[1:] + old_line += 1 +''', + ''' if raw_line.startswith("+"): + if not new_path: + return {} + source_text = raw_line[1:] + if source_text != "[overlong changed line content omitted]": + texts[(new_path, new_line, "RIGHT")] = source_text + new_line += 1 + elif raw_line.startswith("-"): + if not old_path: + return {} + source_text = raw_line[1:] + if source_text != "[overlong changed line content omitted]": + texts[(old_path, old_line, "LEFT")] = source_text + old_line += 1 +''', + "truncated source exclusion", + ) + + text = replace_once( + text, + ''' "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line. The observation must name a concrete token from that source_excerpt and explain a causal or behavioral relation beyond taxonomy labels; differently worded generic labels are not evidence.", +''', + ''' "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. The observation must quote that exact source_excerpt (or ) and state a causal/behavioral relationship; an arbitrary adjacent word or differently worded generic label is not evidence.", +''', + "prompt exact-source contract", + ) + + ast.parse(text, filename=str(SOURCE)) + SOURCE.write_text(text, encoding="utf-8") + + +def patch_corpus_fixture() -> None: + """Make the durable corpus satisfy the strengthened exact-source relation contract.""" + text = CORPUS_TEST.read_text(encoding="utf-8") + text = replace_once( + text, + 'f"The `new` assignment preserves runtime relationship {index} relevant to {field}."', + 'f"The exact `new = 1` source preserves runtime relationship {index} relevant to {field}."', + "corpus source quotation", + ) + text = replace_once( + text, + '"The `old` assignment is removed before the attempted mutation relationship."', + '"The exact `old = 1` source is removed before the attempted mutation relationship."', + "wrong-side source quotation", + ) + ast.parse(text, filename=str(CORPUS_TEST)) + CORPUS_TEST.write_text(text, encoding="utf-8") + + +def patch_traceability() -> None: + """Record why lexical heuristics and bounded-diff omission markers are non-authoritative.""" + doctor = DOCTOR.read_text(encoding="utf-8") + doctor = doctor.replace( + "`observation` must name a concrete source token plus a causal/behavioral relation beyond taxonomy labels", + "`observation` must quote the exact source line (or ``) plus a causal/behavioral relation beyond taxonomy labels; ASCII token shape is not admission authority", + ) + doctor += ( + "\n\nExact-head follow-up also makes bounded-diff omission markers ineligible as source evidence. " + "Short identifiers, symbol-only lines, blank changed lines, and non-ASCII source remain admissible " + "through exact string equality rather than lexical guessing.\n" + ) + DOCTOR.write_text(doctor, encoding="utf-8") + + baseline = BASELINE.read_text(encoding="utf-8") + baseline = baseline.replace( + "whose exact `source_excerpt` must match the cited changed line and whose observation must name concrete source content plus causal behavior", + "whose exact `source_excerpt` must match the cited changed line and whose observation must quote that exact source (or ``) plus causal behavior without ASCII/token-shape heuristics", + ) + baseline += ( + "\n- **Noema exact-source follow-up (PR #1641):** bounded-diff overlong-line omission markers are not admissible source evidence; " + "short, symbol-only, blank, and non-ASCII changed lines use exact source equality, while arbitrary source-adjacent words do not satisfy causal evidence.\n" + ) + BASELINE.write_text(baseline, encoding="utf-8") + + changelog = CHANGELOG.read_text(encoding="utf-8") + changelog = changelog.replace( + "class witnesses bind to exact changed-side source text with non-vacuous causal observations", + "class witnesses bind to exact changed-side source text (including lexical-shape-independent blank/non-ASCII lines) with non-vacuous causal observations, while bounded-diff omission markers are rejected", + ) + CHANGELOG.write_text(changelog, encoding="utf-8") + + +def main() -> None: + """Apply the second exact-head review follow-up.""" + patch_validator() + patch_corpus_fixture() + patch_traceability() + + +if __name__ == "__main__": + main() From 397d9d296f89f4e5ed7910d14e1bea157a707355 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:27:00 +0900 Subject: [PATCH 18/59] ci(temp): execute repaired PR1641 writer --- .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 23091ef153..164bd576d2 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v3 +# execution nonce: review-followup-v4 on: push: From cc3c9802c0da7ec86612cd6c22ee35ba7398d632 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:29:24 +0900 Subject: [PATCH 19/59] fix(noema): replace lexical causal admission with structural roles --- ...mp_pr1641_finish_review_findings_round3.py | 278 ++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 scripts/ci/_temp_pr1641_finish_review_findings_round3.py diff --git a/scripts/ci/_temp_pr1641_finish_review_findings_round3.py b/scripts/ci/_temp_pr1641_finish_review_findings_round3.py new file mode 100644 index 0000000000..3178432b0f --- /dev/null +++ b/scripts/ci/_temp_pr1641_finish_review_findings_round3.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Finish PR #1641 by replacing lexical causality guesses with structural evidence roles.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SOURCE = Path("scripts/ci/noema_review_gate.py") +CORPUS_TEST = Path("tests/test_noema_observed_defect_corpus_current_main.py") +OBSERVATION_TEST = Path("tests/test_noema_class_evidence_observation_contract.py") +DOCTOR = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") +BASELINE = Path("docs/product-technical-gap-baseline.md") +CHANGELOG = Path("CHANGELOG.md") + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace exactly one trusted post-round-two anchor or fail closed.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") + return text.replace(old, new, 1) + + +def patch_validator() -> None: + """Make source grounding language-neutral and class semantics structurally explicit.""" + text = SOURCE.read_text(encoding="utf-8") + + evidence_anchor = '''OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { + "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), + "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), + "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), + "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), + "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), + "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), + "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), + "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), + "state_machine_race": ("initial_state", "event_order", "invariant_observation"), +} +''' + evidence_with_roles = evidence_anchor + '''OBSERVED_REVIEW_PROBE_CLAIM_ROLES: dict[str, dict[str, str]] = { + kind: {field: f"{kind}:{field}" for field in fields} + for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items() +} +''' + text = replace_once(text, evidence_anchor, evidence_with_roles, "claim-role contract") + + text = replace_once( + text, + ''' if not isinstance(source_ref, dict) or set(source_ref) != { + "path", + "line", + "side", + "source_excerpt", + "observation", + }: +''', + ''' if not isinstance(source_ref, dict) or set(source_ref) != { + "path", + "line", + "side", + "source_excerpt", + "claim_role", + "observation", + }: +''', + "witness schema", + ) + text = replace_once( + text, + ''' "path, line, side, exact source_excerpt, and non-empty observation" +''', + ''' "path, line, side, exact source_excerpt, class-specific claim_role, and non-empty observation" +''', + "witness schema diagnostic", + ) + + relation_block = ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + relation_tokens = { + "accepts", + "after", + "aliases", + "allows", + "before", + "because", + "blocks", + "bypasses", + "cancels", + "causes", + "changes", + "conflicts", + "depends", + "differs", + "escapes", + "fails", + "mismatches", + "mutates", + "prevents", + "preserves", + "races", + "reads", + "rejects", + "relationship", + "reuses", + "shares", + "truncates", + "when", + "while", + "without", + "writes", + } + observation_tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}", observation) + } + if not relation_tokens.intersection(observation_tokens): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "causal relationship, not an arbitrary source-adjacent word" + ) +''' + structural_block = ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + expected_claim_role = OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field] + claim_role = source_ref.get("claim_role") + if claim_role != expected_claim_role: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} claim_role " + f"must be {expected_claim_role!r}" + ) +''' + text = replace_once(text, relation_block, structural_block, "lexical relation heuristic") + + text = replace_once( + text, + ''' "source_excerpt": "exact changed-line text", + "observation": ( + f"Concrete {field} causal observation naming a token " + "from source_excerpt." + ), +''', + ''' "source_excerpt": "exact changed-line text", + "claim_role": OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"][field], + "observation": ( + "Quote the exact source_excerpt (or ) and explain " + f"the behavior for the structured {field} claim role." + ), +''', + "prompt witness example", + ) + + text = replace_once( + text, + ''' "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. The observation must quote that exact source_excerpt (or ) and state a causal/behavioral relationship; an arbitrary adjacent word or differently worded generic label is not evidence.", +''', + ''' "Every class_evidence witness must include path, line, side, source_excerpt, claim_role, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. claim_role is the exact class-and-field role emitted by the schema. The observation must quote that exact source_excerpt (or ) and explain the claimed behavior. The deterministic gate validates source identity and the structural role; it deliberately does not guess causality from an English relation-word list.", +''', + "prompt language-neutral contract", + ) + + ast.parse(text, filename=str(SOURCE)) + SOURCE.write_text(text, encoding="utf-8") + + +def patch_tests() -> None: + """Make final regressions exercise structural roles and language-neutral source binding.""" + corpus = CORPUS_TEST.read_text(encoding="utf-8") + corpus = replace_once( + corpus, + ''' "source_excerpt": "new = 1", + "observation": ( + f"The exact `new = 1` source preserves runtime relationship {index} relevant to {field}." + ), +''', + ''' "source_excerpt": "new = 1", + "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field], + "observation": ( + f"The exact `new = 1` source is evidence for structured role {index}: {field}." + ), +''', + "corpus claim roles", + ) + corpus = replace_once( + corpus, + ''' "source_excerpt": "old = 1", + "observation": "The exact `old = 1` source is removed before the attempted mutation relationship.", +''', + ''' "source_excerpt": "old = 1", + "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"]["mutation_attempt"], + "observation": "The exact `old = 1` source is evidence for the mutation-attempt role.", +''', + "wrong-side claim role fixture", + ) + ast.parse(corpus, filename=str(CORPUS_TEST)) + CORPUS_TEST.write_text(corpus, encoding="utf-8") + + observations = OBSERVATION_TEST.read_text(encoding="utf-8") + observations = replace_once( + observations, + ''' witness = _location() + if observations: +''', + ''' witness = _location() + witness["claim_role"] = noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field] + if observations: +''', + "observation claim-role fixture", + ) + observations = observations.replace( + 'match="concrete token from source_excerpt"', + 'match="quote the exact source_excerpt"', + 1, + ) + acceptance_anchor = '''def test_distinct_source_bound_class_observations_are_accepted() -> None: +''' + role_test = '''def test_invented_claim_role_cannot_replace_class_specific_evidence() -> None: + """Free-form labels cannot substitute for the schema's exact class-and-field role.""" + verdict = _verdict(observations=True, source_excerpt=True) + verdict["adversarial_validation"]["probes"][0]["class_evidence"]["mutation_attempt"][ + "claim_role" + ] = "banana" + + with pytest.raises(noema.NoemaModelOutputError, match="claim_role must be"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +def test_distinct_source_bound_class_observations_are_accepted() -> None: +''' + observations = replace_once(observations, acceptance_anchor, role_test, "claim-role regression") + ast.parse(observations, filename=str(OBSERVATION_TEST)) + OBSERVATION_TEST.write_text(observations, encoding="utf-8") + + +def patch_traceability() -> None: + """Document the deterministic/semantic boundary instead of claiming lexical proof.""" + doctor = DOCTOR.read_text(encoding="utf-8") + doctor += ( + "\nThe exact-head structural follow-up removes the fixed English relation-word list. " + "Formal evidence now carries a schema-derived `claim_role` for each defect-class witness, " + "while the deterministic gate verifies exact source identity, canonical coordinates, role identity, " + "and distinct observations. Semantic causal adequacy remains a reviewer/evaluation responsibility; " + "the validator does not pretend English keyword presence proves causality.\n" + ) + DOCTOR.write_text(doctor, encoding="utf-8") + + baseline = BASELINE.read_text(encoding="utf-8") + baseline += ( + "\n- **Noema structural-causality follow-up (PR #1641):** removed fixed English relation-word admission. " + "Each class witness now carries an exact schema-derived `claim_role` plus exact changed-line source text; " + "deterministic validation stays language-neutral and semantic causality is tested through reviewer/evaluation regressions rather than guessed from keywords.\n" + ) + BASELINE.write_text(baseline, encoding="utf-8") + + changelog = CHANGELOG.read_text(encoding="utf-8") + changelog += ( + "\n- Noema review evidence now uses exact class-and-field claim roles and source excerpts instead of a fixed English causal-word heuristic, preserving non-ASCII and symbol-only review evidence without treating keywords as proof.\n" + ) + CHANGELOG.write_text(changelog, encoding="utf-8") + + +def main() -> None: + """Apply the structural evidence-role repair after the exact-source follow-up.""" + patch_validator() + patch_tests() + patch_traceability() + + +if __name__ == "__main__": + main() From 33b696693c0c655da145dfa1272cf26b28788846 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:29:46 +0900 Subject: [PATCH 20/59] ci(temp): execute repaired PR1641 writer --- ...mp_pr1641_noema_observed_corpus_repair.yml | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 164bd576d2..1f64fcd0ab 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v4 +# execution nonce: review-followup-v5-structural-roles on: push: @@ -89,10 +89,16 @@ jobs: fi echo "Verified generic relabeling RED against first-stage implementation (pytest rc=$red_rc)." - - name: Apply review-followup observation-evidence repair + - name: Apply exact-source observation repair run: python3 scripts/ci/_temp_pr1641_finish_review_findings.py - - name: Verify GREEN focused and broader contracts + - name: Apply token-shape and truncation repair + run: python3 scripts/ci/_temp_pr1641_finish_review_findings_round2.py + + - name: Replace lexical causal admission with structural claim roles + run: python3 scripts/ci/_temp_pr1641_finish_review_findings_round3.py + + - name: Verify GREEN focused, full, coverage, and documentation contracts shell: bash run: | set -euo pipefail @@ -101,30 +107,43 @@ jobs: tests/test_noema_class_evidence_observation_contract.py \ tests/test_noema_model_output_failure_classification.py \ tests/test_noema_review_gate.py - PYTHONPATH=. python3 -m pytest -q + PYTHONPATH=. python3 -m pytest -q \ + --cov=scripts.ci.noema_review_gate \ + --cov-branch \ + --cov-fail-under=100 interrogate --fail-under=100 \ scripts/ci/noema_review_gate.py \ scripts/ci/_temp_pr1641_apply_review_corpus.py \ - scripts/ci/_temp_pr1641_finish_review_findings.py + scripts/ci/_temp_pr1641_finish_review_findings.py \ + scripts/ci/_temp_pr1641_finish_review_findings_round2.py \ + scripts/ci/_temp_pr1641_finish_review_findings_round3.py python3 -m compileall -q scripts/ci tests git diff --check - - name: Remove temporary repair mechanism and commit publishable tree + - name: Remove temporary repair mechanism and commit only publishable tree shell: bash run: | set -euo pipefail rm -f \ .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml \ scripts/ci/_temp_pr1641_apply_review_corpus.py \ - scripts/ci/_temp_pr1641_finish_review_findings.py + scripts/ci/_temp_pr1641_finish_review_findings.py \ + scripts/ci/_temp_pr1641_finish_review_findings_round2.py \ + scripts/ci/_temp_pr1641_finish_review_findings_round3.py test ! -e .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml test ! -e scripts/ci/_temp_pr1641_apply_review_corpus.py test ! -e scripts/ci/_temp_pr1641_finish_review_findings.py + test ! -e scripts/ci/_temp_pr1641_finish_review_findings_round2.py + test ! -e scripts/ci/_temp_pr1641_finish_review_findings_round3.py git config user.name 'ContextualWisdomLab repair automation' git config user.email 'actions@users.noreply.github.com' git add -A git diff --cached --check - git commit -m 'fix(noema): enforce observed defect-class evidence' + if git diff --cached --quiet; then + echo '::error::Repair produced no publishable change.' + exit 1 + fi + git commit -m 'fix(noema): enforce structural observed-defect evidence' - name: Push only if writer head is still exact shell: bash From f3ae6d20074cf9d6eda1bbcda46d541168d201bf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:01:51 +0900 Subject: [PATCH 21/59] ci(temp): execute repaired PR1641 writer --- .../workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 1f64fcd0ab..d8e3800199 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v5-structural-roles +# execution nonce: review-followup-v6-unsaturated-runner on: push: @@ -18,7 +18,7 @@ jobs: if: >- github.repository == 'ContextualWisdomLab/.github' && github.event.head_commit.message == 'ci(temp): execute repaired PR1641 writer' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim timeout-minutes: 45 steps: - name: Checkout exact writer head without persisted credential From 8da5c3fb30b2ba8b4bd930a490cb9a9c511b9581 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:06:12 +0900 Subject: [PATCH 22/59] fix(noema): repair exact-source evidence fixtures --- ...mp_pr1641_finish_review_findings_round3.py | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/ci/_temp_pr1641_finish_review_findings_round3.py b/scripts/ci/_temp_pr1641_finish_review_findings_round3.py index 3178432b0f..f1ddfdef57 100644 --- a/scripts/ci/_temp_pr1641_finish_review_findings_round3.py +++ b/scripts/ci/_temp_pr1641_finish_review_findings_round3.py @@ -167,12 +167,17 @@ def patch_validator() -> None: "prompt language-neutral contract", ) + # Round two intentionally materializes the literal Git diff marker. Use a raw + # string in the generated validator so Python does not interpret `\ ` as an + # invalid escape sequence and exact-head verification stays warning-free. + text = text.replace('raw_line.startswith("\\ No newline")', 'raw_line.startswith(r"\\ No newline")') + ast.parse(text, filename=str(SOURCE)) SOURCE.write_text(text, encoding="utf-8") def patch_tests() -> None: - """Make final regressions exercise structural roles and language-neutral source binding.""" + """Make final regressions exercise structural roles and exact-source observation binding.""" corpus = CORPUS_TEST.read_text(encoding="utf-8") corpus = replace_once( corpus, @@ -184,7 +189,7 @@ def patch_tests() -> None: ''' "source_excerpt": "new = 1", "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field], "observation": ( - f"The exact `new = 1` source is evidence for structured role {index}: {field}." + f"new = 1 is exact source evidence for structured role {index}: {field}." ), ''', "corpus claim roles", @@ -196,7 +201,7 @@ def patch_tests() -> None: ''', ''' "source_excerpt": "old = 1", "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"]["mutation_attempt"], - "observation": "The exact `old = 1` source is evidence for the mutation-attempt role.", + "observation": "old = 1 is exact source evidence for the mutation-attempt role.", ''', "wrong-side claim role fixture", ) @@ -208,12 +213,34 @@ def patch_tests() -> None: observations, ''' witness = _location() if observations: + if repeated: + witness["observation"] = ( + "The `new` assignment preserves one repeated runtime relationship." + ) + elif generic_but_different: + witness["observation"] = ( + f"Generic {field.replace('_', ' ')} concern appears in this area." + ) + else: + witness["observation"] = ( + f"The `new` assignment preserves runtime relationship {index} relevant to {field}." + ) ''', ''' witness = _location() witness["claim_role"] = noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field] if observations: + if repeated: + witness["observation"] = "new = 1 is the same repeated source observation." + elif generic_but_different: + witness["observation"] = ( + f"Generic {field.replace('_', ' ')} concern appears in this area." + ) + else: + witness["observation"] = ( + f"new = 1 is exact source evidence for structured witness {index}: {field}." + ) ''', - "observation claim-role fixture", + "observation claim-role/source fixture", ) observations = observations.replace( 'match="concrete token from source_excerpt"', From 1c3346e5864395fcf323723f4303b55b35219613 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:07:25 +0900 Subject: [PATCH 23/59] ci(temp): execute repaired PR1641 writer --- .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index d8e3800199..898010f706 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v6-unsaturated-runner +# execution nonce: review-followup-v7-fixture-repair on: push: From 33ec0a71d6fe11a077a8206afc483ded11b5984f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:15:57 +0900 Subject: [PATCH 24/59] test(noema): isolate repair deadline from external DNS --- tests/conftest.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 6f0c91d00f..c3b50c68e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,26 @@ def clear_trusted_uv_process_caches() -> Iterator[None]: opener_cache_clear() +@pytest.fixture(autouse=True) +def isolate_noema_repair_deadline_from_external_dns(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + """Keep the repair-deadline unit test about wall time, not external DNS latency. + + ``call_llm`` deliberately resolves configured public hosts as part of its + SSRF guard. The deadline regression replaces the HTTP opener but used to + leave that DNS lookup live, so a cold/slow resolver could consume several + seconds before the synthetic slow-read path even began and make the + otherwise-correct 50 ms process-timer assertion fail nondeterministically. + Other Noema SSRF tests retain the real resolver/mocked resolver behavior; + only this single unit test gets a no-op URL guard because URL admission is + outside the behavior it is asserting. + """ + if request.node.name != "test_total_repair_wall_clock_deadline_interrupts_slow_read": + return + from scripts.ci import noema_review_gate as gate + + monkeypatch.setattr(gate, "reject_private_llm_url", lambda _url: None) + + class FakeHttpResponse: """Expose bounded context-managed reads from one deterministic final URL.""" From f33a22eee36ceb45125d8d6b4c8c5bfbbf091191 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:17:12 +0900 Subject: [PATCH 25/59] ci(temp): stage repaired PR1641 retrigger --- .github/pr1641-repair-retrigger.tmp | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/pr1641-repair-retrigger.tmp diff --git a/.github/pr1641-repair-retrigger.tmp b/.github/pr1641-repair-retrigger.tmp new file mode 100644 index 0000000000..90e7be86a4 --- /dev/null +++ b/.github/pr1641-repair-retrigger.tmp @@ -0,0 +1 @@ +ephemeral retrigger marker From 39ae2106497ddcaa1fab38eea2b3d37ad30d78f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:17:23 +0900 Subject: [PATCH 26/59] ci(temp): execute repaired PR1641 writer --- .github/pr1641-repair-retrigger.tmp | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .github/pr1641-repair-retrigger.tmp diff --git a/.github/pr1641-repair-retrigger.tmp b/.github/pr1641-repair-retrigger.tmp deleted file mode 100644 index 90e7be86a4..0000000000 --- a/.github/pr1641-repair-retrigger.tmp +++ /dev/null @@ -1 +0,0 @@ -ephemeral retrigger marker From e8e08cc48a169c220b24931203b56136f321d6d4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:32:59 +0900 Subject: [PATCH 27/59] test(noema): cover observed-evidence validator branches --- ...ema_class_evidence_observation_contract.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index 6e596f05ee..ba84b64cfb 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -162,3 +162,101 @@ def test_distinct_source_bound_class_observations_are_accepted() -> None: DIFF, ["src/tool.py"], ) + + +@pytest.mark.parametrize( + ("record", "message"), + [ + ({"path": "", "line": 1, "side": "RIGHT"}, "canonical changed-side path"), + ({"path": "src/tool.py", "line": True, "side": "RIGHT"}, "canonical positive integer line"), + ({"path": "src/tool.py", "line": 0, "side": "RIGHT"}, "canonical positive integer line"), + ({"path": "src/tool.py", "line": 1, "side": "right"}, "canonical LEFT/RIGHT side"), + ], +) +def test_canonical_changed_location_rejects_noncanonical_coordinates( + record: dict[str, object], message: str +) -> None: + """Canonical source coordinates reject empty paths, bool/int aliases, and invalid sides.""" + with pytest.raises(noema.NoemaModelOutputError, match=message): + noema._canonical_changed_location(record, "fixture") + + +def test_changed_diff_line_texts_covers_context_markers_and_no_newline_marker() -> None: + """Exact-source extraction skips omission markers while preserving neighboring changed text.""" + diff = """diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1,3 +1,3 @@ + context +-[overlong changed line content omitted] ++[overlong changed line content omitted] +-old ++new +\\ No newline at end of file +""" + assert noema.changed_diff_line_texts(diff) == { + ("src/tool.py", 3, "LEFT"): "old", + ("src/tool.py", 3, "RIGHT"): "new", + } + + +def test_changed_diff_line_texts_fails_closed_when_hunk_paths_are_missing() -> None: + """A hunk without its canonical file headers cannot manufacture source evidence.""" + assert noema.changed_diff_line_texts("@@ -1 +1 @@\n+new\n") == {} + assert noema.changed_diff_line_texts("@@ -1 +1 @@\n-old\n") == {} + + +def test_changed_diff_line_texts_handles_dev_null_addition() -> None: + """New files may have an empty old path while their RIGHT-side source remains exact.""" + diff = """diff --git a/new.py b/new.py +--- /dev/null ++++ b/new.py +@@ -0,0 +1 @@ ++value = 1 +""" + assert noema.changed_diff_line_texts(diff) == {("new.py", 1, "RIGHT"): "value = 1"} + + +def test_blank_changed_source_uses_explicit_blank_marker() -> None: + """A blank changed line remains admissible through exact equality and the explicit marker.""" + diff = """diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++ +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = "" + witness["observation"] = f" is exact source evidence for {probe['probe_kind']}:{field}." + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + +def test_overlong_omission_marker_cannot_be_source_evidence() -> None: + """A bounded-diff omission marker cannot be reintroduced as an exact source excerpt.""" + marker = "[overlong changed line content omitted]" + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{marker} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = marker + witness["observation"] = f"{marker} is exact source evidence for {probe['probe_kind']}:{field}." + with pytest.raises(noema.NoemaModelOutputError, match="exact changed-line source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + +def test_overlong_class_observation_is_rejected_before_semantic_admission() -> None: + """Bounded review evidence refuses oversized witness prose without weakening source checks.""" + verdict = _verdict(observations=True, source_excerpt=True) + witness = verdict["adversarial_validation"]["probes"][0]["class_evidence"]["mutation_attempt"] + witness["observation"] = "x" * (noema.MAX_THREAD_BODY_CHARS + 1) + with pytest.raises(noema.NoemaModelOutputError, match="exceeds"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) From 02724d3db5252c91675c0ca234130186fdbb2af5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:35:12 +0900 Subject: [PATCH 28/59] ci(temp): execute repaired PR1641 writer --- .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 898010f706..162a61573c 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v7-fixture-repair +# execution nonce: review-followup-v8-coverage-repair on: push: From 6a4ff1d6e1cb93a511ba86336f1d50d13f35f307 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:41:19 +0900 Subject: [PATCH 29/59] test(noema): cover final class-evidence failure branches --- ...ema_class_evidence_observation_contract.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index ba84b64cfb..acdc8a560b 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -120,6 +120,26 @@ def test_location_only_class_evidence_cannot_relabel_generic_probes() -> None: noema.validate_substantive_verdict(_verdict(observations=False), DIFF, ["src/tool.py"]) +def test_class_evidence_requires_exact_observed_field_set() -> None: + """A defect-class label cannot omit one of its schema-defined witness roles.""" + verdict = _verdict(observations=True, source_excerpt=True) + verdict["adversarial_validation"]["probes"][0]["class_evidence"].pop("mutation_attempt") + + with pytest.raises(noema.NoemaModelOutputError, match="must contain exactly"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +def test_empty_class_observation_is_rejected() -> None: + """Exact source coordinates and excerpts do not substitute for an observation.""" + verdict = _verdict(observations=True, source_excerpt=True) + verdict["adversarial_validation"]["probes"][0]["class_evidence"]["mutation_attempt"][ + "observation" + ] = "" + + with pytest.raises(noema.NoemaModelOutputError, match="non-empty observation"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + def test_repeated_generic_observations_do_not_satisfy_class_specific_witnesses() -> None: """A probe must provide distinct observations for its class-specific witness fields.""" with pytest.raises(noema.NoemaModelOutputError, match="distinct class-specific observations"): From f5cf383de6eb72657cc393f5d4a7d991f6499b1d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:41:46 +0900 Subject: [PATCH 30/59] ci(temp): execute repaired PR1641 writer --- .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 162a61573c..35df959533 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v8-coverage-repair +# execution nonce: review-followup-v9-final-coverage-repair on: push: From 0918cb3bc68f510218a842cf572e12c4f2d52156 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:47:34 +0900 Subject: [PATCH 31/59] ci(temp): execute repaired PR1641 writer --- ...mp_pr1641_noema_observed_corpus_repair.yml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index 35df959533..ae6111600e 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v9-final-coverage-repair +# execution nonce: review-followup-v10-artifact-export on: push: @@ -145,6 +145,24 @@ jobs: fi git commit -m 'fix(noema): enforce structural observed-defect evidence' + - name: Export exact verified publishable patch + shell: bash + run: | + set -euo pipefail + mkdir -p repair-evidence + git format-patch -1 --stdout > repair-evidence/pr1641-final.patch + git rev-parse HEAD > repair-evidence/generated-commit.txt + git diff-tree --no-commit-id --name-status -r HEAD > repair-evidence/name-status.txt + test -s repair-evidence/pr1641-final.patch + + - name: Preserve exact verified publishable patch + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pr1641-verified-publishable-patch-${{ github.run_id }}-${{ github.run_attempt }} + path: repair-evidence/ + if-no-files-found: error + retention-days: 1 + - name: Push only if writer head is still exact shell: bash env: From 249f28ce3f87b78e3c992e45aff520bcca9b6b9e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 09:56:40 +0900 Subject: [PATCH 32/59] ci(temp): execute repaired PR1641 writer --- .../_temp_pr1641_noema_observed_corpus_repair.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml index ae6111600e..a29a35d7ce 100644 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml @@ -1,5 +1,5 @@ name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v10-artifact-export +# execution nonce: review-followup-v11-connected-successor-trigger on: push: @@ -168,6 +168,7 @@ jobs: env: PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }} OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} run: | set -euo pipefail live="$(git ls-remote https://github.com/ContextualWisdomLab/.github.git "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" @@ -182,10 +183,11 @@ jobs: push_token="$OPENCODE_APPROVE_TOKEN" push_source='OPENCODE_APPROVE_TOKEN' else - echo '::error::No workflow-starting branch-mutation credential is configured. Refusing a github.token push because it would suppress successor-head required workflows.' - exit 78 + push_token="$GITHUB_TOKEN" + push_source='github.token-temporary-bootstrap' + echo '::warning::Using github.token only to publish the already-verified self-deleting repair commit. This push will not be accepted as successor-check evidence; a connected owner-side follow-up mutation must create the final workflow-starting head before merge.' fi - echo "Using workflow-starting mutation credential source: $push_source" + echo "Using branch-mutation credential source: $push_source" git config core.hooksPath /dev/null git remote set-url origin "https://x-access-token:${push_token}@github.com/${GITHUB_REPOSITORY}.git" git push origin "HEAD:${GITHUB_REF_NAME}" From 7c7e2fcb23f028c0c1d537402dbc786f55e852a3 Mon Sep 17 00:00:00 2001 From: ContextualWisdomLab repair automation Date: Wed, 2 Sep 2026 00:59:30 +0000 Subject: [PATCH 33/59] fix(noema): enforce structural observed-defect evidence --- ...mp_pr1641_noema_observed_corpus_repair.yml | 193 ---------- CHANGELOG.md | 3 + ...ema-observed-defect-corpus-current-main.md | 16 + docs/product-technical-gap-baseline.md | 12 + .../ci/_temp_pr1641_apply_review_corpus.py | 217 ----------- .../ci/_temp_pr1641_finish_review_findings.py | 344 ------------------ ...mp_pr1641_finish_review_findings_round2.py | 247 ------------- ...mp_pr1641_finish_review_findings_round3.py | 305 ---------------- scripts/ci/noema_review_gate.py | 212 ++++++++++- ...ema_class_evidence_observation_contract.py | 20 +- ...ema_observed_defect_corpus_current_main.py | 24 +- 11 files changed, 276 insertions(+), 1317 deletions(-) delete mode 100644 .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml create mode 100644 docs/doctoring/noema-observed-defect-corpus-current-main.md delete mode 100644 scripts/ci/_temp_pr1641_apply_review_corpus.py delete mode 100644 scripts/ci/_temp_pr1641_finish_review_findings.py delete mode 100644 scripts/ci/_temp_pr1641_finish_review_findings_round2.py delete mode 100644 scripts/ci/_temp_pr1641_finish_review_findings_round3.py diff --git a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml b/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml deleted file mode 100644 index a29a35d7ce..0000000000 --- a/.github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml +++ /dev/null @@ -1,193 +0,0 @@ -name: Temporary PR1641 Noema observed-corpus repair -# execution nonce: review-followup-v11-connected-successor-trigger - -on: - push: - branches: - - fix/noema-observed-defect-corpus-current-main-20260902 - -permissions: - contents: write - -concurrency: - group: temp-pr1641-${{ github.ref }} - cancel-in-progress: true - -jobs: - repair: - if: >- - github.repository == 'ContextualWisdomLab/.github' && - github.event.head_commit.message == 'ci(temp): execute repaired PR1641 writer' - runs-on: ubuntu-slim - timeout-minutes: 45 - steps: - - name: Checkout exact writer head without persisted credential - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Refuse a stale writer head - shell: bash - run: | - set -euo pipefail - live="$(git ls-remote https://github.com/ContextualWisdomLab/.github.git "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - test -n "$live" - test "$live" = "$GITHUB_SHA" - - - name: Install repository-declared pinned review test toolchain - run: >- - python3 -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt - - - name: Verify the committed taxonomy regression is specifically RED - shell: bash - run: | - set -euo pipefail - red_log="${RUNNER_TEMP}/pr1641-taxonomy-red.log" - set +e - PYTHONPATH=. python3 -m pytest -q tests/test_noema_observed_defect_corpus_current_main.py >"$red_log" 2>&1 - red_rc=$? - set -e - cat "$red_log" - if [ "$red_rc" -eq 0 ]; then - echo '::error::Expected the current-main regression corpus to fail before the causal repair.' - exit 1 - fi - if ! grep -Fq "has no attribute 'OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS'" "$red_log"; then - echo '::error::RED failed for an unexpected reason; refusing production mutation.' - exit 1 - fi - if grep -Fq 'ERROR collecting' "$red_log"; then - echo '::error::RED was a collection/environment error rather than the expected missing-contract assertion path.' - exit 1 - fi - echo "Verified expected missing-taxonomy RED against pre-repair source (pytest rc=$red_rc)." - - - name: Apply first-stage trusted validator and prompt repair - run: python3 scripts/ci/_temp_pr1641_apply_review_corpus.py - - - name: Prove generic relabeling remains RED before the review-followup repair - shell: bash - run: | - set -euo pipefail - red_log="${RUNNER_TEMP}/pr1641-generic-evidence-red.log" - set +e - PYTHONPATH=. python3 -m pytest -q \ - tests/test_noema_class_evidence_observation_contract.py::test_location_only_class_evidence_cannot_relabel_generic_probes \ - >"$red_log" 2>&1 - red_rc=$? - set -e - cat "$red_log" - if [ "$red_rc" -eq 0 ]; then - echo '::error::Expected location-only generic evidence to be accepted by the first-stage implementation.' - exit 1 - fi - if ! grep -Fq 'DID NOT RAISE' "$red_log"; then - echo '::error::Generic-evidence RED failed for an unexpected reason; refusing the follow-up production mutation.' - exit 1 - fi - echo "Verified generic relabeling RED against first-stage implementation (pytest rc=$red_rc)." - - - name: Apply exact-source observation repair - run: python3 scripts/ci/_temp_pr1641_finish_review_findings.py - - - name: Apply token-shape and truncation repair - run: python3 scripts/ci/_temp_pr1641_finish_review_findings_round2.py - - - name: Replace lexical causal admission with structural claim roles - run: python3 scripts/ci/_temp_pr1641_finish_review_findings_round3.py - - - name: Verify GREEN focused, full, coverage, and documentation contracts - shell: bash - run: | - set -euo pipefail - PYTHONPATH=. python3 -m pytest -q \ - tests/test_noema_observed_defect_corpus_current_main.py \ - tests/test_noema_class_evidence_observation_contract.py \ - tests/test_noema_model_output_failure_classification.py \ - tests/test_noema_review_gate.py - PYTHONPATH=. python3 -m pytest -q \ - --cov=scripts.ci.noema_review_gate \ - --cov-branch \ - --cov-fail-under=100 - interrogate --fail-under=100 \ - scripts/ci/noema_review_gate.py \ - scripts/ci/_temp_pr1641_apply_review_corpus.py \ - scripts/ci/_temp_pr1641_finish_review_findings.py \ - scripts/ci/_temp_pr1641_finish_review_findings_round2.py \ - scripts/ci/_temp_pr1641_finish_review_findings_round3.py - python3 -m compileall -q scripts/ci tests - git diff --check - - - name: Remove temporary repair mechanism and commit only publishable tree - shell: bash - run: | - set -euo pipefail - rm -f \ - .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml \ - scripts/ci/_temp_pr1641_apply_review_corpus.py \ - scripts/ci/_temp_pr1641_finish_review_findings.py \ - scripts/ci/_temp_pr1641_finish_review_findings_round2.py \ - scripts/ci/_temp_pr1641_finish_review_findings_round3.py - test ! -e .github/workflows/_temp_pr1641_noema_observed_corpus_repair.yml - test ! -e scripts/ci/_temp_pr1641_apply_review_corpus.py - test ! -e scripts/ci/_temp_pr1641_finish_review_findings.py - test ! -e scripts/ci/_temp_pr1641_finish_review_findings_round2.py - test ! -e scripts/ci/_temp_pr1641_finish_review_findings_round3.py - git config user.name 'ContextualWisdomLab repair automation' - git config user.email 'actions@users.noreply.github.com' - git add -A - git diff --cached --check - if git diff --cached --quiet; then - echo '::error::Repair produced no publishable change.' - exit 1 - fi - git commit -m 'fix(noema): enforce structural observed-defect evidence' - - - name: Export exact verified publishable patch - shell: bash - run: | - set -euo pipefail - mkdir -p repair-evidence - git format-patch -1 --stdout > repair-evidence/pr1641-final.patch - git rev-parse HEAD > repair-evidence/generated-commit.txt - git diff-tree --no-commit-id --name-status -r HEAD > repair-evidence/name-status.txt - test -s repair-evidence/pr1641-final.patch - - - name: Preserve exact verified publishable patch - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: pr1641-verified-publishable-patch-${{ github.run_id }}-${{ github.run_attempt }} - path: repair-evidence/ - if-no-files-found: error - retention-days: 1 - - - name: Push only if writer head is still exact - shell: bash - env: - PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }} - OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }} - GITHUB_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - live="$(git ls-remote https://github.com/ContextualWisdomLab/.github.git "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - if [ "$live" != "$GITHUB_SHA" ]; then - echo "::error::Writer branch moved from $GITHUB_SHA to ${live:-missing}; refusing overwrite." - exit 75 - fi - if [ -n "${PR_REVIEW_MERGE_TOKEN:-}" ]; then - push_token="$PR_REVIEW_MERGE_TOKEN" - push_source='PR_REVIEW_MERGE_TOKEN' - elif [ -n "${OPENCODE_APPROVE_TOKEN:-}" ]; then - push_token="$OPENCODE_APPROVE_TOKEN" - push_source='OPENCODE_APPROVE_TOKEN' - else - push_token="$GITHUB_TOKEN" - push_source='github.token-temporary-bootstrap' - echo '::warning::Using github.token only to publish the already-verified self-deleting repair commit. This push will not be accepted as successor-check evidence; a connected owner-side follow-up mutation must create the final workflow-starting head before merge.' - fi - echo "Using branch-mutation credential source: $push_source" - git config core.hooksPath /dev/null - git remote set-url origin "https://x-access-token:${push_token}@github.com/${GITHUB_REPOSITORY}.git" - git push origin "HEAD:${GITHUB_REF_NAME}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b661772cb..732660052b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- **Require source-bound observed defect classes in Noema formal reviews (#1641).** Canonical changed-line coordinates now reject JSON booleans, material reviews must cover distinct classes from the executable external-finding corpus, class witnesses bind to exact changed-side source text (including lexical-shape-independent blank/non-ASCII lines) with non-vacuous causal observations, while bounded-diff omission markers are rejected, and the prompt explicitly attacks workflow-event authority plus mutable-alias, TOCTOU, identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure shapes without fabricating benchmark claims. - **Pin the three central required review workflows (Strix, OpenCode Review, Noema Review) off the observed starved floating `ubuntu-latest` runner image.** Following the same repair already rolled out to security gates (`#1618`) and the merge scheduler (`#1609`), `strix.yml`, `opencode-review.yml`, and `noema-review.yml` now request the explicit `ubuntu-24.04` image on every job. These three workflows are the org's own required-workflow gate for every sibling repository, so a starved floating image here directly contributes to organization-wide required-check queuing. New `tests/test_required_review_runner_image_contract.py` asserts no job in any of the three files still requests the floating image. Also fixed 4 pre-existing, unrelated test failures on `main` left by `#1630`'s organization-sweep rotation cadence change (every 15 minutes to hourly, to reduce control-plane pressure under the same Actions saturation): `tests/test_required_workflow_queue_contract.py`'s rotation-index tests still asserted the old `/ 900` (15-minute) divisor against the new `/ 3600` (hourly) production value. - **Refresh Noema reviewer App authority after long model work (`#1616`).** A real `naruon#1497` review outlived its repository-scoped GitHub App installation token and failed the next exact-head GitHub operation with HTTP 401. The trusted workflow now prepares the validated verdict into a private runner-local envelope, remints the same least-privilege repository-scoped App authority after model work, independently re-fetches exact live head/reviewer identity, and only then publishes. Skipped preparation creates no envelope, predecessor App tokens cannot authorize publication, PAT/OIDC remain explicit fail-closed sources, malformed handoffs are cleaned up, and executable plus step-scoped regressions cover stale-head, identity, alias, workflow wiring, and migration of legacy broader-suite contracts away from the retired single-process reviewer path. - Fix `existing_noema_review()` treating a "legacy" Noema review (one posted before @@ -1157,3 +1158,5 @@ Semantic Versioning where the repository publishes a release. - Added an organization-owned reusable exact-artifact SBOM attestation boundary that validates inert six-file wheel/sdist evidence, binds CycloneDX 1.7 predicates to exact SHA-256 subjects, signs through least-privilege GitHub artifact attestations, and exports online and offline verification bundles. - Hardened exact-artifact SBOM verification with strict finite RFC 8259 JSON, integer CycloneDX document versions, deterministic UUIDv5 subject identities, exact filename properties and single SHA-256 root bindings, environment-only shell input transfer, pinned Ubuntu 24.04 quality runners, and checksum-sealed beginner-readable offline evidence. The decision record now cites Bray (2017) so NaN and Infinity cannot be treated as sealed SBOM numbers. - Recorded the org control-plane architecture, including exact-artifact SBOM attestation, so agents reconstruct the signing trust boundary from the repo instead of private memory. + +- Noema review evidence now uses exact class-and-field claim roles and source excerpts instead of a fixed English causal-word heuristic, preserving non-ASCII and symbol-only review evidence without treating keywords as proof. diff --git a/docs/doctoring/noema-observed-defect-corpus-current-main.md b/docs/doctoring/noema-observed-defect-corpus-current-main.md new file mode 100644 index 0000000000..d984186a43 --- /dev/null +++ b/docs/doctoring/noema-observed-defect-corpus-current-main.md @@ -0,0 +1,16 @@ +# Noema observed-defect review corpus + +The trusted Noema review gate treats externally demonstrated review misses as executable regression evidence, not as benchmark claims. Material source/test reviews must exercise at least two distinct observed defect classes and every admitted class witness remains bound to an exact changed-side source coordinate. + +The current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are `{path,line,side,source_excerpt,observation}` records bound to the probe location. `source_excerpt` must equal the exact changed-side line, and `observation` must quote the exact source line (or ``) plus a causal/behavioral relation beyond taxonomy labels; ASCII token shape is not admission authority; repeated or differently worded generic labels do not satisfy the deterministic validator. + +The model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. For CI/automation changes, the review prompt also requires checking whether the mutation credential can create the downstream events/checks the state machine depends on. + +JSON booleans are rejected as line coordinates even though Python considers `True == 1`: changed-line evidence requires `type(line) is int` and a positive value. Production review calls always provide the complete changed-path manifest, which activates the observed taxonomy; direct validator unit tests may omit that manifest to exercise lower-level generic schema boundaries independently. + +This repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim. + + +Exact-head follow-up also makes bounded-diff omission markers ineligible as source evidence. Short identifiers, symbol-only lines, blank changed lines, and non-ASCII source remain admissible through exact string equality rather than lexical guessing. + +The exact-head structural follow-up removes the fixed English relation-word list. Formal evidence now carries a schema-derived `claim_role` for each defect-class witness, while the deterministic gate verifies exact source identity, canonical coordinates, role identity, and distinct observations. Semantic causal adequacy remains a reviewer/evaluation responsibility; the validator does not pretend English keyword presence proves causality. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 2a8f4c7b54..b7f824a172 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2590,3 +2590,15 @@ Higgins, S. S., Crepalde, N., & Fernandes, L. (2021). Segmented multiplexity: A **Validation.** Full suite `2407 passed, 1 skipped, 21 subtests`; `coverage` 100% on `scripts/ci`; `interrogate` 100%; all four touched/added workflow files re-parse as valid YAML; `test_opencode_workflow_shell_syntax.py` and related shell-syntax tests pass unchanged. **Residual.** This closes the specific floating-image contribution from these three central workflows; it does not by itself guarantee the organization-wide Actions queue is fully drained, since other repositories' own workflows and any remaining unpinned central workflows may still request the floating image. Worth a follow-up sweep across the rest of `.github/workflows/` and sibling-repo workflows if queuing persists after this lands. + + +### 2026-09-02 — Noema observed-defect false-negative corpus (#1641) + +- **Verified gap:** protected current main admitted Noema adversarial evidence by count/prose identity and compared model line coordinates with Python integers without excluding booleans. Thus `true` could alias line `1`, and two differently worded probes could satisfy material-change diversity without proving distinct observed defect shapes. +- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific source-bound witnesses whose exact `source_excerpt` must match the cited changed line and whose observation must quote that exact source (or ``) plus causal behavior without ASCII/token-shape heuristics. Material changes require distinct classes, and the prompt explicitly checks workflow-starting mutation credentials before relying on downstream required checks. +- **Regression evidence:** `tests/test_noema_observed_defect_corpus_current_main.py` is committed before the causal production change and covers boolean aliasing, malformed/unknown class labels, duplicate-class diversity, witness/source binding, a valid multi-class verdict, and rendered prompt coverage. +- **Authority boundary:** no reviewer, provider, routing, merge, or repository-write authority is widened. The taxonomy is evaluation/admission evidence only. + +- **Noema exact-source follow-up (PR #1641):** bounded-diff overlong-line omission markers are not admissible source evidence; short, symbol-only, blank, and non-ASCII changed lines use exact source equality, while arbitrary source-adjacent words do not satisfy causal evidence. + +- **Noema structural-causality follow-up (PR #1641):** removed fixed English relation-word admission. Each class witness now carries an exact schema-derived `claim_role` plus exact changed-line source text; deterministic validation stays language-neutral and semantic causality is tested through reviewer/evaluation regressions rather than guessed from keywords. diff --git a/scripts/ci/_temp_pr1641_apply_review_corpus.py b/scripts/ci/_temp_pr1641_apply_review_corpus.py deleted file mode 100644 index 58d661f553..0000000000 --- a/scripts/ci/_temp_pr1641_apply_review_corpus.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""Temporary one-shot PR #1641 source repair; removed by its workflow.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -# Contents-API touch intentionally triggers the now-valid one-shot workflow. -SOURCE = Path("scripts/ci/noema_review_gate.py") - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace exactly one trusted source anchor or fail closed.""" - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") - return text.replace(old, new, 1) - - -def apply_source_repair() -> None: - """Harden canonical locations and observed defect-class review evidence.""" - text = SOURCE.read_text(encoding="utf-8") - - constants_anchor = 'DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@")\n' - constants = '''DIFF_HUNK_RE = re.compile(r"^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@") -OBSERVED_REVIEW_PROBE_KINDS = frozenset( - { - "mutable_alias", - "time_of_check_time_of_use", - "execution_identity", - "coercion_boundary", - "test_oracle", - "cross_contract", - "authority_boundary", - "dependency_context", - "state_machine_race", - } -) -OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { - "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), - "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), - "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), - "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), - "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), - "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), - "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), - "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), - "state_machine_race": ("initial_state", "event_order", "invariant_observation"), -} -''' - text = replace_once(text, constants_anchor, constants, "taxonomy constants") - - helper_anchor = ''' return value.removeprefix(prefix)\n\n\ndef validate_substantive_verdict(\n''' - helper = ''' return value.removeprefix(prefix) - - -def _canonical_changed_location(record: dict[str, Any], label: str) -> tuple[str, int, str]: - """Return a canonical changed-side location without bool/int coercion.""" - path_value = record.get("path") - line_value = record.get("line") - side_value = record.get("side") - if not isinstance(path_value, str) or not path_value.strip(): - raise NoemaModelOutputError(f"{label} requires a canonical changed-side path") - if type(line_value) is not int or line_value <= 0: - raise NoemaModelOutputError(f"{label} requires a canonical positive integer line") - if side_value not in {"LEFT", "RIGHT"}: - raise NoemaModelOutputError(f"{label} requires canonical LEFT/RIGHT side") - return (path_value, line_value, side_value) - - -def _validate_observed_probe_class_evidence( - probe: dict[str, Any], probe_kind: str, index: int, location: tuple[str, int, str] -) -> None: - """Require defect-class witnesses to bind to the probe's exact changed line.""" - class_evidence = probe.get("class_evidence") - required_fields = OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[probe_kind] - if not isinstance(class_evidence, dict) or set(class_evidence) != set(required_fields): - expected = ", ".join(required_fields) - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence for {probe_kind} " - f"must contain exactly: {expected}" - ) - for field in required_fields: - source_ref = class_evidence.get(field) - if not isinstance(source_ref, dict) or set(source_ref) != {"path", "line", "side"}: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "source-bound changed-line reference" - ) - source_location = _canonical_changed_location( - source_ref, f"Noema adversarial probe {index} class_evidence.{field}" - ) - if source_location != location: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} must bind to " - "the probe location" - ) - - -def validate_substantive_verdict( -''' - text = replace_once(text, helper_anchor, helper, "location helpers") - - text = replace_once( - text, - ' location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side"))\n', - ' location = _canonical_changed_location(reviewed, f"Noema reviewed line {index}")\n', - "reviewed-line location", - ) - text = replace_once( - text, - ' identities: set[tuple[Any, ...]] = set()\n for index, probe in enumerate(probes, start=1):\n', - ' identities: set[tuple[Any, ...]] = set()\n probe_kinds: set[str] = set()\n enforce_observed_taxonomy = bool(changed_paths)\n for index, probe in enumerate(probes, start=1):\n', - "probe taxonomy state", - ) - text = replace_once( - text, - ' location = (probe.get("path"), probe.get("line"), probe.get("side"))\n if location not in locations:\n', - ' location = _canonical_changed_location(probe, f"Noema adversarial probe {index}")\n if location not in locations:\n', - "probe location", - ) - text = replace_once( - text, - ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', - ' outcome = probe.get("outcome")\n if outcome not in {"falsified", "confirmed"}:\n raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed")\n if enforce_observed_taxonomy:\n probe_kind = probe.get("probe_kind")\n if not isinstance(probe_kind, str) or probe_kind not in OBSERVED_REVIEW_PROBE_KINDS:\n raise NoemaModelOutputError(\n f"Noema adversarial probe {index} requires probe_kind from the observed defect taxonomy"\n )\n _validate_observed_probe_class_evidence(probe, probe_kind, index, location)\n probe_kinds.add(probe_kind)\n identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold())\n', - "probe class validation", - ) - text = replace_once( - text, - ' if decision == "approve" and confirmed:\n', - ' if enforce_observed_taxonomy and len(probe_kinds) < required_probes:\n raise NoemaModelOutputError(\n f"Noema {decision} requires at least {required_probes} distinct probe_kind values"\n )\n\n if decision == "approve" and confirmed:\n', - "probe diversity validation", - ) - - text = replace_once( - text, - ' **location_example,\n "hypothesis": "...",\n', - ' **location_example,\n "probe_kind": "mutable_alias",\n "class_evidence": {\n field: location_example\n for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"]\n },\n "hypothesis": "...",\n', - "prompt schema example", - ) - text = replace_once( - text, - ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; source or test changes require at least two distinct probes and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n', - ' "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; material source or test changes require at least two distinct probe_kind values and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.",\n "Observed defect taxonomy and required source-bound class_evidence keys: "\n + json.dumps(\n {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()},\n sort_keys=True,\n separators=(",", ":"),\n ),\n "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.",\n', - "prompt taxonomy instruction", - ) - text = replace_once( - text, - ' f"- `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', - ' f"- [{probe.get(\'probe_kind\') or \'legacy\'}] `{probe.get(\'path\')}:{probe.get(\'line\')} ({probe.get(\'side\')})` "\n f"{probe.get(\'outcome\')}: {str(probe.get(\'hypothesis\') or \'\').strip()} — "\n', - "review evidence class", - ) - - ast.parse(text, filename=str(SOURCE)) - SOURCE.write_text(text, encoding="utf-8") - - -def update_docs() -> None: - """Record the operating contract and the externally observed regression corpus.""" - doctor = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") - doctor.write_text( - """# Noema observed-defect review corpus - -The trusted Noema review gate treats externally demonstrated review misses as executable regression evidence, not as benchmark claims. Material source/test reviews must exercise at least two distinct observed defect classes and every admitted class witness remains bound to an exact changed-side source coordinate. - -The current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator. - -The model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. - -JSON booleans are rejected as line coordinates even though Python considers `True == 1`: changed-line evidence requires `type(line) is int` and a positive value. Production review calls always provide the complete changed-path manifest, which activates the observed taxonomy; direct validator unit tests may omit that manifest to exercise lower-level generic schema boundaries independently. - -This repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim. -""", - encoding="utf-8", - ) - - baseline = Path("docs/product-technical-gap-baseline.md") - baseline_text = baseline.read_text(encoding="utf-8") - marker = "### 2026-09-02 — Noema observed-defect false-negative corpus (#1641)" - if marker not in baseline_text: - baseline_text += f""" - -{marker} - -- **Verified gap:** protected current main admitted Noema adversarial evidence by count/prose identity and compared model line coordinates with Python integers without excluding booleans. Thus `true` could alias line `1`, and two differently worded probes could satisfy material-change diversity without proving distinct observed defect shapes. -- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families. -- **Regression evidence:** `tests/test_noema_observed_defect_corpus_current_main.py` is committed before the causal production change and covers boolean aliasing, malformed/unknown class labels, duplicate-class diversity, witness/source binding, a valid multi-class verdict, and rendered prompt coverage. -- **Authority boundary:** no reviewer, provider, routing, merge, or repository-write authority is widened. The taxonomy is evaluation/admission evidence only. -""" - baseline.write_text(baseline_text, encoding="utf-8") - - changelog = Path("CHANGELOG.md") - changelog_text = changelog.read_text(encoding="utf-8") - entry = ( - "- **Require source-bound observed defect classes in Noema formal reviews (#1641).** " - "Canonical changed-line coordinates now reject JSON booleans, material reviews must cover " - "distinct classes from the executable external-finding corpus, class witnesses bind to exact " - "changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, " - "identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure " - "shapes without fabricating benchmark claims.\n" - ) - if entry not in changelog_text: - anchor = "## [Unreleased]\n" - if changelog_text.count(anchor) != 1: - raise SystemExit("could not locate unique Unreleased changelog anchor") - changelog.write_text(changelog_text.replace(anchor, anchor + entry, 1), encoding="utf-8") - - -def main() -> None: - """Apply the one-shot code and traceability repair.""" - apply_source_repair() - update_docs() - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/_temp_pr1641_finish_review_findings.py b/scripts/ci/_temp_pr1641_finish_review_findings.py deleted file mode 100644 index ad8b46a5d3..0000000000 --- a/scripts/ci/_temp_pr1641_finish_review_findings.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -"""Finish PR #1641 after proving first-stage review-evidence false negatives.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -SOURCE = Path("scripts/ci/noema_review_gate.py") -CORPUS_TEST = Path("tests/test_noema_observed_defect_corpus_current_main.py") -DOCTOR = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") -BASELINE = Path("docs/product-technical-gap-baseline.md") -CHANGELOG = Path("CHANGELOG.md") - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace one exact trusted anchor or stop instead of guessing.""" - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") - return text.replace(old, new, 1) - - -def patch_validator() -> None: - """Require exact source excerpts and non-vacuous class observations.""" - text = SOURCE.read_text(encoding="utf-8") - old_block = ''' for field in required_fields: - source_ref = class_evidence.get(field) - if not isinstance(source_ref, dict) or set(source_ref) != {"path", "line", "side"}: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "source-bound changed-line reference" - ) - source_location = _canonical_changed_location( - source_ref, f"Noema adversarial probe {index} class_evidence.{field}" - ) - if source_location != location: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} must bind to " - "the probe location" - ) -''' - new_block = ''' normalized_observations: list[str] = [] - source_texts = changed_diff_line_texts(diff) - for field in required_fields: - source_ref = class_evidence.get(field) - if not isinstance(source_ref, dict) or set(source_ref) != { - "path", - "line", - "side", - "source_excerpt", - "observation", - }: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires " - "path, line, side, exact source_excerpt, and non-empty observation" - ) - source_location = _canonical_changed_location( - source_ref, f"Noema adversarial probe {index} class_evidence.{field}" - ) - if source_location != location: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} must bind to " - "the probe location" - ) - expected_excerpt = source_texts.get(source_location) - source_excerpt = source_ref.get("source_excerpt") - if ( - not isinstance(source_excerpt, str) - or not source_excerpt.strip() - or expected_excerpt is None - or source_excerpt != expected_excerpt - ): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires the " - "exact changed-line source_excerpt" - ) - observation = source_ref.get("observation") - if not isinstance(observation, str) or not observation.strip(): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "non-empty observation" - ) - if len(observation) > MAX_THREAD_BODY_CHARS: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - f"exceeds {MAX_THREAD_BODY_CHARS} characters" - ) - source_tokens = { - token.casefold() - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", source_excerpt) - } - observation_tokens = { - token.casefold() - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", observation) - } - if not source_tokens or not source_tokens.intersection(observation_tokens): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must name a concrete token from source_excerpt" - ) - label_tokens = {probe_kind.casefold(), field.casefold()} | { - token.casefold() - for token in re.findall( - r"[A-Za-z_][A-Za-z0-9_]{2,}", - f"{probe_kind} {field}".replace("_", " "), - ) - } - filler_tokens = { - "area", - "changed", - "concern", - "concrete", - "evidence", - "exact", - "generic", - "here", - "line", - "nearby", - "observed", - "observation", - "probe", - "review", - "source", - "this", - "value", - } - causal_tokens = observation_tokens - source_tokens - label_tokens - filler_tokens - if not causal_tokens: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "concrete causal observation beyond source and taxonomy labels" - ) - normalized_observations.append(observation.strip().casefold()) - if len(set(normalized_observations)) != len(normalized_observations): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} requires distinct class-specific observations" - ) -''' - text = replace_once(text, old_block, new_block, "class observation validation") - - text = replace_once( - text, - '''def _validate_observed_probe_class_evidence( - probe: dict[str, Any], probe_kind: str, index: int, location: tuple[str, int, str] -) -> None: -''', - '''def _validate_observed_probe_class_evidence( - probe: dict[str, Any], - probe_kind: str, - index: int, - location: tuple[str, int, str], - diff: str, -) -> None: -''', - "class evidence validator signature", - ) - - helper_anchor = ''' return locations - - -def parse_diff_path(raw: str, prefix: str) -> str: -''' - helper = ''' return locations - - -def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: - """Return exact changed-side source text keyed by canonical diff location.""" - texts: dict[tuple[str, int, str], str] = {} - old_path = new_path = "" - old_line = new_line = 0 - in_hunk = False - for raw_line in diff.splitlines(): - if raw_line.startswith("diff --git "): - old_path = new_path = "" - in_hunk = False - continue - if not in_hunk and raw_line.startswith("--- "): - old_path = parse_diff_path(raw_line[4:], "a/") - continue - if not in_hunk and raw_line.startswith("+++ "): - new_path = parse_diff_path(raw_line[4:], "b/") - continue - match = DIFF_HUNK_RE.match(raw_line) - if match: - old_line, new_line = map(int, match.groups()) - in_hunk = True - continue - if not in_hunk or raw_line.startswith("\\ No newline"): - continue - if raw_line.startswith("+"): - if not new_path: - return {} - texts[(new_path, new_line, "RIGHT")] = raw_line[1:] - new_line += 1 - elif raw_line.startswith("-"): - if not old_path: - return {} - texts[(old_path, old_line, "LEFT")] = raw_line[1:] - old_line += 1 - else: - old_line += 1 - new_line += 1 - return texts - - -def parse_diff_path(raw: str, prefix: str) -> str: -''' - text = replace_once(text, helper_anchor, helper, "changed-line source helper") - - text = replace_once( - text, - " _validate_observed_probe_class_evidence(probe, probe_kind, index, location)\n", - " _validate_observed_probe_class_evidence(probe, probe_kind, index, location, diff)\n", - "class evidence validator call", - ) - - old_schema = ''' "class_evidence": { - field: location_example - for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] - }, -''' - new_schema = ''' "class_evidence": { - field: { - **location_example, - "source_excerpt": "exact changed-line text", - "observation": ( - f"Concrete {field} causal observation naming a token " - "from source_excerpt." - ), - } - for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] - }, -''' - text = replace_once(text, old_schema, new_schema, "prompt class-evidence schema") - - old_prompt = ''' "Observed defect taxonomy and required source-bound class_evidence keys: " - + json.dumps( - {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()}, - sort_keys=True, - separators=(",", ":"), - ), - "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", -''' - new_prompt = ''' "Observed defect taxonomy and required source-bound class_evidence keys: " - + json.dumps( - {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()}, - sort_keys=True, - separators=(",", ":"), - ), - "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line. The observation must name a concrete token from that source_excerpt and explain a causal or behavioral relation beyond taxonomy labels; differently worded generic labels are not evidence.", - "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. For automation or CI that mutates a branch or source and then relies on later events, verify that the mutation uses a workflow-starting credential/actor and that downstream required checks can actually be created on the successor head. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", -''' - text = replace_once(text, old_prompt, new_prompt, "prompt observation contract") - ast.parse(text, filename=str(SOURCE)) - SOURCE.write_text(text, encoding="utf-8") - - -def patch_regression_corpus() -> None: - """Update the original corpus fixtures to the stronger observation schema.""" - text = CORPUS_TEST.read_text(encoding="utf-8") - text = replace_once( - text, - '''def _class_evidence(kind: str) -> dict[str, dict[str, object]]: - return {field: _source_ref() for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind]} -''', - '''def _class_evidence(kind: str) -> dict[str, dict[str, object]]: - return { - field: { - **_source_ref(), - "source_excerpt": "new = 1", - "observation": ( - f"The `new` assignment preserves runtime relationship {index} relevant to {field}." - ), - } - for index, field in enumerate( - noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind], - start=1, - ) - } -''', - "corpus evidence fixture", - ) - text = replace_once( - text, - ''' probe["class_evidence"]["mutation_attempt"] = {"path": "src/tool.py", "line": 1, "side": "LEFT"} -''', - ''' probe["class_evidence"]["mutation_attempt"] = { - "path": "src/tool.py", - "line": 1, - "side": "LEFT", - "source_excerpt": "old = 1", - "observation": "The `old` assignment is removed before the attempted mutation relationship.", - } -''', - "wrong-side corpus fixture", - ) - ast.parse(text, filename=str(CORPUS_TEST)) - CORPUS_TEST.write_text(text, encoding="utf-8") - - -def patch_traceability() -> None: - """Keep doctoring, baseline, and changelog aligned with executable evidence.""" - doctor = DOCTOR.read_text(encoding="utf-8") - doctor = replace_once( - doctor, - "Witness values are exact `{path,line,side}` references to the probe location; prose labels alone do not satisfy the deterministic validator.", - "Witness values are `{path,line,side,source_excerpt,observation}` records bound to the probe location. `source_excerpt` must equal the exact changed-side line, and `observation` must name a concrete source token plus a causal/behavioral relation beyond taxonomy labels; repeated or differently worded generic labels do not satisfy the deterministic validator.", - "doctoring observation contract", - ) - doctor = doctor.replace( - "A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity.", - "A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. For CI/automation changes, the review prompt also requires checking whether the mutation credential can create the downstream events/checks the state machine depends on.", - ) - DOCTOR.write_text(doctor, encoding="utf-8") - - baseline = BASELINE.read_text(encoding="utf-8") - baseline = replace_once( - baseline, - "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific, source-bound witness fields and distinct classes for material changes; the prompt actively attacks the same external-review failure families.", - "- **Repair:** exact changed-side coordinates now require canonical positive integers; production review verdicts use a closed observed-defect taxonomy with class-specific source-bound witnesses whose exact `source_excerpt` must match the cited changed line and whose observation must name concrete source content plus causal behavior. Material changes require distinct classes, and the prompt explicitly checks workflow-starting mutation credentials before relying on downstream required checks.", - "baseline observation contract", - ) - BASELINE.write_text(baseline, encoding="utf-8") - - changelog = CHANGELOG.read_text(encoding="utf-8") - changelog = replace_once( - changelog, - "class witnesses bind to exact changed-side source coordinates, and the prompt explicitly attacks mutable-alias, TOCTOU, ", - "class witnesses bind to exact changed-side source text with non-vacuous causal observations, and the prompt explicitly attacks workflow-event authority plus mutable-alias, TOCTOU, ", - "changelog observation contract", - ) - CHANGELOG.write_text(changelog, encoding="utf-8") - - -def main() -> None: - """Apply the independently demonstrated review-followup repair.""" - patch_validator() - patch_regression_corpus() - patch_traceability() - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/_temp_pr1641_finish_review_findings_round2.py b/scripts/ci/_temp_pr1641_finish_review_findings_round2.py deleted file mode 100644 index b2c210a925..0000000000 --- a/scripts/ci/_temp_pr1641_finish_review_findings_round2.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python3 -"""Finish PR #1641 after exact-head review exposed source-binding edge cases.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -SOURCE = Path("scripts/ci/noema_review_gate.py") -CORPUS_TEST = Path("tests/test_noema_observed_defect_corpus_current_main.py") -DOCTOR = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") -BASELINE = Path("docs/product-technical-gap-baseline.md") -CHANGELOG = Path("CHANGELOG.md") - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace exactly one trusted generated-source anchor or fail closed.""" - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") - return text.replace(old, new, 1) - - -def patch_validator() -> None: - """Bind evidence to exact source text without ASCII/token-shape heuristics.""" - text = SOURCE.read_text(encoding="utf-8") - - text = replace_once( - text, - ''' if ( - not isinstance(source_excerpt, str) - or not source_excerpt.strip() - or expected_excerpt is None - or source_excerpt != expected_excerpt - ): -''', - ''' if ( - not isinstance(source_excerpt, str) - or expected_excerpt is None - or source_excerpt != expected_excerpt - ): -''', - "blank exact-source admission", - ) - - old_tokens = ''' source_tokens = { - token.casefold() - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", source_excerpt) - } - observation_tokens = { - token.casefold() - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}|\\d+", observation) - } - if not source_tokens or not source_tokens.intersection(observation_tokens): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must name a concrete token from source_excerpt" - ) - label_tokens = {probe_kind.casefold(), field.casefold()} | { - token.casefold() - for token in re.findall( - r"[A-Za-z_][A-Za-z0-9_]{2,}", - f"{probe_kind} {field}".replace("_", " "), - ) - } - filler_tokens = { - "area", - "changed", - "concern", - "concrete", - "evidence", - "exact", - "generic", - "here", - "line", - "nearby", - "observed", - "observation", - "probe", - "review", - "source", - "this", - "value", - } - causal_tokens = observation_tokens - source_tokens - label_tokens - filler_tokens - if not causal_tokens: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "concrete causal observation beyond source and taxonomy labels" - ) -''' - new_tokens = ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - relation_tokens = { - "accepts", - "after", - "aliases", - "allows", - "before", - "because", - "blocks", - "bypasses", - "cancels", - "causes", - "changes", - "conflicts", - "depends", - "differs", - "escapes", - "fails", - "mismatches", - "mutates", - "prevents", - "preserves", - "races", - "reads", - "rejects", - "relationship", - "reuses", - "shares", - "truncates", - "when", - "while", - "without", - "writes", - } - observation_tokens = { - token.casefold() - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}", observation) - } - if not relation_tokens.intersection(observation_tokens): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "causal relationship, not an arbitrary source-adjacent word" - ) -''' - text = replace_once(text, old_tokens, new_tokens, "causal source binding") - - text = replace_once( - text, - ''' if raw_line.startswith("+"): - if not new_path: - return {} - texts[(new_path, new_line, "RIGHT")] = raw_line[1:] - new_line += 1 - elif raw_line.startswith("-"): - if not old_path: - return {} - texts[(old_path, old_line, "LEFT")] = raw_line[1:] - old_line += 1 -''', - ''' if raw_line.startswith("+"): - if not new_path: - return {} - source_text = raw_line[1:] - if source_text != "[overlong changed line content omitted]": - texts[(new_path, new_line, "RIGHT")] = source_text - new_line += 1 - elif raw_line.startswith("-"): - if not old_path: - return {} - source_text = raw_line[1:] - if source_text != "[overlong changed line content omitted]": - texts[(old_path, old_line, "LEFT")] = source_text - old_line += 1 -''', - "truncated source exclusion", - ) - - text = replace_once( - text, - ''' "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line. The observation must name a concrete token from that source_excerpt and explain a causal or behavioral relation beyond taxonomy labels; differently worded generic labels are not evidence.", -''', - ''' "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. The observation must quote that exact source_excerpt (or ) and state a causal/behavioral relationship; an arbitrary adjacent word or differently worded generic label is not evidence.", -''', - "prompt exact-source contract", - ) - - ast.parse(text, filename=str(SOURCE)) - SOURCE.write_text(text, encoding="utf-8") - - -def patch_corpus_fixture() -> None: - """Make the durable corpus satisfy the strengthened exact-source relation contract.""" - text = CORPUS_TEST.read_text(encoding="utf-8") - text = replace_once( - text, - 'f"The `new` assignment preserves runtime relationship {index} relevant to {field}."', - 'f"The exact `new = 1` source preserves runtime relationship {index} relevant to {field}."', - "corpus source quotation", - ) - text = replace_once( - text, - '"The `old` assignment is removed before the attempted mutation relationship."', - '"The exact `old = 1` source is removed before the attempted mutation relationship."', - "wrong-side source quotation", - ) - ast.parse(text, filename=str(CORPUS_TEST)) - CORPUS_TEST.write_text(text, encoding="utf-8") - - -def patch_traceability() -> None: - """Record why lexical heuristics and bounded-diff omission markers are non-authoritative.""" - doctor = DOCTOR.read_text(encoding="utf-8") - doctor = doctor.replace( - "`observation` must name a concrete source token plus a causal/behavioral relation beyond taxonomy labels", - "`observation` must quote the exact source line (or ``) plus a causal/behavioral relation beyond taxonomy labels; ASCII token shape is not admission authority", - ) - doctor += ( - "\n\nExact-head follow-up also makes bounded-diff omission markers ineligible as source evidence. " - "Short identifiers, symbol-only lines, blank changed lines, and non-ASCII source remain admissible " - "through exact string equality rather than lexical guessing.\n" - ) - DOCTOR.write_text(doctor, encoding="utf-8") - - baseline = BASELINE.read_text(encoding="utf-8") - baseline = baseline.replace( - "whose exact `source_excerpt` must match the cited changed line and whose observation must name concrete source content plus causal behavior", - "whose exact `source_excerpt` must match the cited changed line and whose observation must quote that exact source (or ``) plus causal behavior without ASCII/token-shape heuristics", - ) - baseline += ( - "\n- **Noema exact-source follow-up (PR #1641):** bounded-diff overlong-line omission markers are not admissible source evidence; " - "short, symbol-only, blank, and non-ASCII changed lines use exact source equality, while arbitrary source-adjacent words do not satisfy causal evidence.\n" - ) - BASELINE.write_text(baseline, encoding="utf-8") - - changelog = CHANGELOG.read_text(encoding="utf-8") - changelog = changelog.replace( - "class witnesses bind to exact changed-side source text with non-vacuous causal observations", - "class witnesses bind to exact changed-side source text (including lexical-shape-independent blank/non-ASCII lines) with non-vacuous causal observations, while bounded-diff omission markers are rejected", - ) - CHANGELOG.write_text(changelog, encoding="utf-8") - - -def main() -> None: - """Apply the second exact-head review follow-up.""" - patch_validator() - patch_corpus_fixture() - patch_traceability() - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/_temp_pr1641_finish_review_findings_round3.py b/scripts/ci/_temp_pr1641_finish_review_findings_round3.py deleted file mode 100644 index f1ddfdef57..0000000000 --- a/scripts/ci/_temp_pr1641_finish_review_findings_round3.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -"""Finish PR #1641 by replacing lexical causality guesses with structural evidence roles.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -SOURCE = Path("scripts/ci/noema_review_gate.py") -CORPUS_TEST = Path("tests/test_noema_observed_defect_corpus_current_main.py") -OBSERVATION_TEST = Path("tests/test_noema_class_evidence_observation_contract.py") -DOCTOR = Path("docs/doctoring/noema-observed-defect-corpus-current-main.md") -BASELINE = Path("docs/product-technical-gap-baseline.md") -CHANGELOG = Path("CHANGELOG.md") - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace exactly one trusted post-round-two anchor or fail closed.""" - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one source anchor, found {count}") - return text.replace(old, new, 1) - - -def patch_validator() -> None: - """Make source grounding language-neutral and class semantics structurally explicit.""" - text = SOURCE.read_text(encoding="utf-8") - - evidence_anchor = '''OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { - "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), - "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), - "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), - "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), - "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), - "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), - "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), - "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), - "state_machine_race": ("initial_state", "event_order", "invariant_observation"), -} -''' - evidence_with_roles = evidence_anchor + '''OBSERVED_REVIEW_PROBE_CLAIM_ROLES: dict[str, dict[str, str]] = { - kind: {field: f"{kind}:{field}" for field in fields} - for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items() -} -''' - text = replace_once(text, evidence_anchor, evidence_with_roles, "claim-role contract") - - text = replace_once( - text, - ''' if not isinstance(source_ref, dict) or set(source_ref) != { - "path", - "line", - "side", - "source_excerpt", - "observation", - }: -''', - ''' if not isinstance(source_ref, dict) or set(source_ref) != { - "path", - "line", - "side", - "source_excerpt", - "claim_role", - "observation", - }: -''', - "witness schema", - ) - text = replace_once( - text, - ''' "path, line, side, exact source_excerpt, and non-empty observation" -''', - ''' "path, line, side, exact source_excerpt, class-specific claim_role, and non-empty observation" -''', - "witness schema diagnostic", - ) - - relation_block = ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - relation_tokens = { - "accepts", - "after", - "aliases", - "allows", - "before", - "because", - "blocks", - "bypasses", - "cancels", - "causes", - "changes", - "conflicts", - "depends", - "differs", - "escapes", - "fails", - "mismatches", - "mutates", - "prevents", - "preserves", - "races", - "reads", - "rejects", - "relationship", - "reuses", - "shares", - "truncates", - "when", - "while", - "without", - "writes", - } - observation_tokens = { - token.casefold() - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}", observation) - } - if not relation_tokens.intersection(observation_tokens): - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} requires a " - "causal relationship, not an arbitrary source-adjacent word" - ) -''' - structural_block = ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - expected_claim_role = OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field] - claim_role = source_ref.get("claim_role") - if claim_role != expected_claim_role: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} claim_role " - f"must be {expected_claim_role!r}" - ) -''' - text = replace_once(text, relation_block, structural_block, "lexical relation heuristic") - - text = replace_once( - text, - ''' "source_excerpt": "exact changed-line text", - "observation": ( - f"Concrete {field} causal observation naming a token " - "from source_excerpt." - ), -''', - ''' "source_excerpt": "exact changed-line text", - "claim_role": OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"][field], - "observation": ( - "Quote the exact source_excerpt (or ) and explain " - f"the behavior for the structured {field} claim role." - ), -''', - "prompt witness example", - ) - - text = replace_once( - text, - ''' "Every class_evidence witness must include path, line, side, source_excerpt, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. The observation must quote that exact source_excerpt (or ) and state a causal/behavioral relationship; an arbitrary adjacent word or differently worded generic label is not evidence.", -''', - ''' "Every class_evidence witness must include path, line, side, source_excerpt, claim_role, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. claim_role is the exact class-and-field role emitted by the schema. The observation must quote that exact source_excerpt (or ) and explain the claimed behavior. The deterministic gate validates source identity and the structural role; it deliberately does not guess causality from an English relation-word list.", -''', - "prompt language-neutral contract", - ) - - # Round two intentionally materializes the literal Git diff marker. Use a raw - # string in the generated validator so Python does not interpret `\ ` as an - # invalid escape sequence and exact-head verification stays warning-free. - text = text.replace('raw_line.startswith("\\ No newline")', 'raw_line.startswith(r"\\ No newline")') - - ast.parse(text, filename=str(SOURCE)) - SOURCE.write_text(text, encoding="utf-8") - - -def patch_tests() -> None: - """Make final regressions exercise structural roles and exact-source observation binding.""" - corpus = CORPUS_TEST.read_text(encoding="utf-8") - corpus = replace_once( - corpus, - ''' "source_excerpt": "new = 1", - "observation": ( - f"The exact `new = 1` source preserves runtime relationship {index} relevant to {field}." - ), -''', - ''' "source_excerpt": "new = 1", - "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field], - "observation": ( - f"new = 1 is exact source evidence for structured role {index}: {field}." - ), -''', - "corpus claim roles", - ) - corpus = replace_once( - corpus, - ''' "source_excerpt": "old = 1", - "observation": "The exact `old = 1` source is removed before the attempted mutation relationship.", -''', - ''' "source_excerpt": "old = 1", - "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"]["mutation_attempt"], - "observation": "old = 1 is exact source evidence for the mutation-attempt role.", -''', - "wrong-side claim role fixture", - ) - ast.parse(corpus, filename=str(CORPUS_TEST)) - CORPUS_TEST.write_text(corpus, encoding="utf-8") - - observations = OBSERVATION_TEST.read_text(encoding="utf-8") - observations = replace_once( - observations, - ''' witness = _location() - if observations: - if repeated: - witness["observation"] = ( - "The `new` assignment preserves one repeated runtime relationship." - ) - elif generic_but_different: - witness["observation"] = ( - f"Generic {field.replace('_', ' ')} concern appears in this area." - ) - else: - witness["observation"] = ( - f"The `new` assignment preserves runtime relationship {index} relevant to {field}." - ) -''', - ''' witness = _location() - witness["claim_role"] = noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field] - if observations: - if repeated: - witness["observation"] = "new = 1 is the same repeated source observation." - elif generic_but_different: - witness["observation"] = ( - f"Generic {field.replace('_', ' ')} concern appears in this area." - ) - else: - witness["observation"] = ( - f"new = 1 is exact source evidence for structured witness {index}: {field}." - ) -''', - "observation claim-role/source fixture", - ) - observations = observations.replace( - 'match="concrete token from source_excerpt"', - 'match="quote the exact source_excerpt"', - 1, - ) - acceptance_anchor = '''def test_distinct_source_bound_class_observations_are_accepted() -> None: -''' - role_test = '''def test_invented_claim_role_cannot_replace_class_specific_evidence() -> None: - """Free-form labels cannot substitute for the schema's exact class-and-field role.""" - verdict = _verdict(observations=True, source_excerpt=True) - verdict["adversarial_validation"]["probes"][0]["class_evidence"]["mutation_attempt"][ - "claim_role" - ] = "banana" - - with pytest.raises(noema.NoemaModelOutputError, match="claim_role must be"): - noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) - - -def test_distinct_source_bound_class_observations_are_accepted() -> None: -''' - observations = replace_once(observations, acceptance_anchor, role_test, "claim-role regression") - ast.parse(observations, filename=str(OBSERVATION_TEST)) - OBSERVATION_TEST.write_text(observations, encoding="utf-8") - - -def patch_traceability() -> None: - """Document the deterministic/semantic boundary instead of claiming lexical proof.""" - doctor = DOCTOR.read_text(encoding="utf-8") - doctor += ( - "\nThe exact-head structural follow-up removes the fixed English relation-word list. " - "Formal evidence now carries a schema-derived `claim_role` for each defect-class witness, " - "while the deterministic gate verifies exact source identity, canonical coordinates, role identity, " - "and distinct observations. Semantic causal adequacy remains a reviewer/evaluation responsibility; " - "the validator does not pretend English keyword presence proves causality.\n" - ) - DOCTOR.write_text(doctor, encoding="utf-8") - - baseline = BASELINE.read_text(encoding="utf-8") - baseline += ( - "\n- **Noema structural-causality follow-up (PR #1641):** removed fixed English relation-word admission. " - "Each class witness now carries an exact schema-derived `claim_role` plus exact changed-line source text; " - "deterministic validation stays language-neutral and semantic causality is tested through reviewer/evaluation regressions rather than guessed from keywords.\n" - ) - BASELINE.write_text(baseline, encoding="utf-8") - - changelog = CHANGELOG.read_text(encoding="utf-8") - changelog += ( - "\n- Noema review evidence now uses exact class-and-field claim roles and source excerpts instead of a fixed English causal-word heuristic, preserving non-ASCII and symbol-only review evidence without treating keywords as proof.\n" - ) - CHANGELOG.write_text(changelog, encoding="utf-8") - - -def main() -> None: - """Apply the structural evidence-role repair after the exact-source follow-up.""" - patch_validator() - patch_tests() - patch_traceability() - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 4f82281fc3..07e12c3543 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -60,6 +60,34 @@ MAX_REVIEW_CONTEXT_CHARS = 24000 MAX_THREAD_BODY_CHARS = 1200 DIFF_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@") +OBSERVED_REVIEW_PROBE_KINDS = frozenset( + { + "mutable_alias", + "time_of_check_time_of_use", + "execution_identity", + "coercion_boundary", + "test_oracle", + "cross_contract", + "authority_boundary", + "dependency_context", + "state_machine_race", + } +) +OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS: dict[str, tuple[str, ...]] = { + "mutable_alias": ("alias_origin", "mutation_attempt", "post_validation_observation"), + "time_of_check_time_of_use": ("check_observation", "intervening_change", "use_observation"), + "execution_identity": ("incoming_identity", "retained_identity", "mismatch_guard"), + "coercion_boundary": ("raw_value", "conversion_path", "canonicality_guard"), + "test_oracle": ("assertion_under_test", "negative_control", "distinguishing_observation"), + "cross_contract": ("first_contract", "second_contract", "contradiction_or_alignment"), + "authority_boundary": ("component_authority", "external_authority", "enforcement_boundary"), + "dependency_context": ("dependency", "omitted_or_included_context", "causal_effect"), + "state_machine_race": ("initial_state", "event_order", "invariant_observation"), +} +OBSERVED_REVIEW_PROBE_CLAIM_ROLES: dict[str, dict[str, str]] = { + kind: {field: f"{kind}:{field}" for field in fields} + for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items() +} ORCHESTRATOR_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1"}) ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL" @@ -409,6 +437,50 @@ def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: return locations +def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: + """Return exact changed-side source text keyed by canonical diff location.""" + texts: dict[tuple[str, int, str], str] = {} + old_path = new_path = "" + old_line = new_line = 0 + in_hunk = False + for raw_line in diff.splitlines(): + if raw_line.startswith("diff --git "): + old_path = new_path = "" + in_hunk = False + continue + if not in_hunk and raw_line.startswith("--- "): + old_path = parse_diff_path(raw_line[4:], "a/") + continue + if not in_hunk and raw_line.startswith("+++ "): + new_path = parse_diff_path(raw_line[4:], "b/") + continue + match = DIFF_HUNK_RE.match(raw_line) + if match: + old_line, new_line = map(int, match.groups()) + in_hunk = True + continue + if not in_hunk or raw_line.startswith(r"\ No newline"): + continue + if raw_line.startswith("+"): + if not new_path: + return {} + source_text = raw_line[1:] + if source_text != "[overlong changed line content omitted]": + texts[(new_path, new_line, "RIGHT")] = source_text + new_line += 1 + elif raw_line.startswith("-"): + if not old_path: + return {} + source_text = raw_line[1:] + if source_text != "[overlong changed line content omitted]": + texts[(old_path, old_line, "LEFT")] = source_text + old_line += 1 + else: + old_line += 1 + new_line += 1 + return texts + + def parse_diff_path(raw: str, prefix: str) -> str: """Decode a Git unified-diff path, including C-quoted UTF-8 paths.""" value = raw.split("\t", 1)[0] @@ -423,6 +495,102 @@ def parse_diff_path(raw: str, prefix: str) -> str: return value.removeprefix(prefix) +def _canonical_changed_location(record: dict[str, Any], label: str) -> tuple[str, int, str]: + """Return a canonical changed-side location without bool/int coercion.""" + path_value = record.get("path") + line_value = record.get("line") + side_value = record.get("side") + if not isinstance(path_value, str) or not path_value.strip(): + raise NoemaModelOutputError(f"{label} requires a canonical changed-side path") + if type(line_value) is not int or line_value <= 0: + raise NoemaModelOutputError(f"{label} requires a canonical positive integer line") + if side_value not in {"LEFT", "RIGHT"}: + raise NoemaModelOutputError(f"{label} requires canonical LEFT/RIGHT side") + return (path_value, line_value, side_value) + + +def _validate_observed_probe_class_evidence( + probe: dict[str, Any], + probe_kind: str, + index: int, + location: tuple[str, int, str], + diff: str, +) -> None: + """Require defect-class witnesses to bind to the probe's exact changed line.""" + class_evidence = probe.get("class_evidence") + required_fields = OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[probe_kind] + if not isinstance(class_evidence, dict) or set(class_evidence) != set(required_fields): + expected = ", ".join(required_fields) + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence for {probe_kind} " + f"must contain exactly: {expected}" + ) + normalized_observations: list[str] = [] + source_texts = changed_diff_line_texts(diff) + for field in required_fields: + source_ref = class_evidence.get(field) + if not isinstance(source_ref, dict) or set(source_ref) != { + "path", + "line", + "side", + "source_excerpt", + "claim_role", + "observation", + }: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires " + "path, line, side, exact source_excerpt, class-specific claim_role, and non-empty observation" + ) + source_location = _canonical_changed_location( + source_ref, f"Noema adversarial probe {index} class_evidence.{field}" + ) + if source_location != location: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} must bind to " + "the probe location" + ) + expected_excerpt = source_texts.get(source_location) + source_excerpt = source_ref.get("source_excerpt") + if ( + not isinstance(source_excerpt, str) + or expected_excerpt is None + or source_excerpt != expected_excerpt + ): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires the " + "exact changed-line source_excerpt" + ) + observation = source_ref.get("observation") + if not isinstance(observation, str) or not observation.strip(): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} requires a " + "non-empty observation" + ) + if len(observation) > MAX_THREAD_BODY_CHARS: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + f"exceeds {MAX_THREAD_BODY_CHARS} characters" + ) + source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + expected_claim_role = OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field] + claim_role = source_ref.get("claim_role") + if claim_role != expected_claim_role: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} claim_role " + f"must be {expected_claim_role!r}" + ) + normalized_observations.append(observation.strip().casefold()) + if len(set(normalized_observations)) != len(normalized_observations): + raise NoemaModelOutputError( + f"Noema adversarial probe {index} requires distinct class-specific observations" + ) + + def validate_substantive_verdict( verdict: dict[str, Any], diff: str, changed_paths: Sequence[str] = () ) -> None: @@ -440,7 +608,7 @@ def validate_substantive_verdict( for index, reviewed in enumerate(reviewed_lines, start=1): if not isinstance(reviewed, dict): raise NoemaModelOutputError(f"Noema reviewed line {index} must be an object") - location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side")) + location = _canonical_changed_location(reviewed, f"Noema reviewed line {index}") if location not in locations: raise NoemaModelOutputError(f"Noema reviewed line {index} is not an exact changed-side line") analysis = reviewed.get("analysis") @@ -465,10 +633,12 @@ def validate_substantive_verdict( confirmed: set[tuple[str, int, str]] = set() identities: set[tuple[Any, ...]] = set() + probe_kinds: set[str] = set() + enforce_observed_taxonomy = bool(changed_paths) for index, probe in enumerate(probes, start=1): if not isinstance(probe, dict): raise NoemaModelOutputError(f"Noema adversarial probe {index} must be an object") - location = (probe.get("path"), probe.get("line"), probe.get("side")) + location = _canonical_changed_location(probe, f"Noema adversarial probe {index}") if location not in locations: raise NoemaModelOutputError(f"Noema adversarial probe {index} is not an exact changed-side line") for field in ("hypothesis", "attack_or_counterexample", "evidence"): @@ -478,6 +648,14 @@ def validate_substantive_verdict( outcome = probe.get("outcome") if outcome not in {"falsified", "confirmed"}: raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed") + if enforce_observed_taxonomy: + probe_kind = probe.get("probe_kind") + if not isinstance(probe_kind, str) or probe_kind not in OBSERVED_REVIEW_PROBE_KINDS: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} requires probe_kind from the observed defect taxonomy" + ) + _validate_observed_probe_class_evidence(probe, probe_kind, index, location, diff) + probe_kinds.add(probe_kind) identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold()) if identity in identities: raise NoemaModelOutputError(f"Noema adversarial probe {index} duplicates an earlier probe") @@ -485,6 +663,11 @@ def validate_substantive_verdict( if outcome == "confirmed": confirmed.add((str(probe["path"]), int(probe["line"]), str(probe["side"]))) + if enforce_observed_taxonomy and len(probe_kinds) < required_probes: + raise NoemaModelOutputError( + f"Noema {decision} requires at least {required_probes} distinct probe_kind values" + ) + if decision == "approve" and confirmed: raise NoemaModelOutputError("Noema approve cannot contain a confirmed adversarial probe") if decision == "request_changes": @@ -1232,6 +1415,19 @@ def call_llm( "probes": [ { **location_example, + "probe_kind": "mutable_alias", + "class_evidence": { + field: { + **location_example, + "source_excerpt": "exact changed-line text", + "claim_role": OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"][field], + "observation": ( + "Quote the exact source_excerpt (or ) and explain " + f"the behavior for the structured {field} claim role." + ), + } + for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS["mutable_alias"] + }, "hypothesis": "...", "attack_or_counterexample": "...", "evidence": "observed or source-traced result", @@ -1251,7 +1447,15 @@ def call_llm( }, separators=(",", ":"), ), - "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; source or test changes require at least two distinct probes and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.", + "Every formal verdict must cite exact changed-side lines. APPROVE requires falsifying concrete regression hypotheses; material source or test changes require at least two distinct probe_kind values and other changes require at least one. REQUEST_CHANGES requires a confirmed probe at a finding location.", + "Observed defect taxonomy and required source-bound class_evidence keys: " + + json.dumps( + {kind: list(fields) for kind, fields in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS.items()}, + sort_keys=True, + separators=(",", ":"), + ), + "Every class_evidence witness must include path, line, side, source_excerpt, claim_role, and observation. source_excerpt must be the exact cited changed-side line, including an empty string for a blank line; an overlong-line omission marker is never source evidence. claim_role is the exact class-and-field role emitted by the schema. The observation must quote that exact source_excerpt (or ) and explain the claimed behavior. The deterministic gate validates source identity and the structural role; it deliberately does not guess causality from an English relation-word list.", + "Actively attack mutable alias/immutability escapes, time-of-check/time-of-use or changing-getter behavior, execution/tenant/request identity confusion, coercion boundaries, weak or vacuous test oracles, cross-file/cross-document contract contradictions, internal-vs-external authority overreach, missing causal dependency context, and security/reliability state-machine races. For automation or CI that mutates a branch or source and then relies on later events, verify that the mutation uses a workflow-starting credential/actor and that downstream required checks can actually be created on the successor head. Distinguish confirmed defects from falsified hypotheses; do not manufacture findings to satisfy the taxonomy.", "Use request_changes only for blocking, concrete issues. A generic no-issues statement is not review evidence.", *( [ @@ -1404,7 +1608,7 @@ def format_review_evidence(verdict: dict[str, Any]) -> list[str]: for probe in (validation.get("probes") or [])[:20]: if isinstance(probe, dict): lines.append( - f"- `{probe.get('path')}:{probe.get('line')} ({probe.get('side')})` " + f"- [{probe.get('probe_kind') or 'legacy'}] `{probe.get('path')}:{probe.get('line')} ({probe.get('side')})` " f"{probe.get('outcome')}: {str(probe.get('hypothesis') or '').strip()} — " f"{str(probe.get('evidence') or '').strip()}" ) diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index acdc8a560b..d1bb158c8f 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -33,18 +33,17 @@ def _class_evidence( evidence: dict[str, object] = {} for index, field in enumerate(noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind], start=1): witness = _location() + witness["claim_role"] = noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field] if observations: if repeated: - witness["observation"] = ( - "The `new` assignment preserves one repeated runtime relationship." - ) + witness["observation"] = "new = 1 is the same repeated source observation." elif generic_but_different: witness["observation"] = ( f"Generic {field.replace('_', ' ')} concern appears in this area." ) else: witness["observation"] = ( - f"The `new` assignment preserves runtime relationship {index} relevant to {field}." + f"new = 1 is exact source evidence for structured witness {index}: {field}." ) if source_excerpt: witness["source_excerpt"] = "new = 1" @@ -152,7 +151,7 @@ def test_repeated_generic_observations_do_not_satisfy_class_specific_witnesses() def test_differently_worded_generic_observations_without_source_signal_are_rejected() -> None: """Unique prose labels are not evidence unless they name concrete changed-source content.""" - with pytest.raises(noema.NoemaModelOutputError, match="concrete token from source_excerpt"): + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): noema.validate_substantive_verdict( _verdict( observations=True, @@ -175,6 +174,17 @@ def test_fabricated_source_excerpt_is_rejected() -> None: noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) +def test_invented_claim_role_cannot_replace_class_specific_evidence() -> None: + """Free-form labels cannot substitute for the schema's exact class-and-field role.""" + verdict = _verdict(observations=True, source_excerpt=True) + verdict["adversarial_validation"]["probes"][0]["class_evidence"]["mutation_attempt"][ + "claim_role" + ] = "banana" + + with pytest.raises(noema.NoemaModelOutputError, match="claim_role must be"): + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + def test_distinct_source_bound_class_observations_are_accepted() -> None: """Concrete source-backed observations preserve an otherwise-valid multi-class verdict.""" noema.validate_substantive_verdict( diff --git a/tests/test_noema_observed_defect_corpus_current_main.py b/tests/test_noema_observed_defect_corpus_current_main.py index df5ca5622b..cc54ce41a3 100644 --- a/tests/test_noema_observed_defect_corpus_current_main.py +++ b/tests/test_noema_observed_defect_corpus_current_main.py @@ -29,7 +29,20 @@ def _source_ref() -> dict[str, object]: def _class_evidence(kind: str) -> dict[str, dict[str, object]]: - return {field: _source_ref() for field in noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind]} + return { + field: { + **_source_ref(), + "source_excerpt": "new = 1", + "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES[kind][field], + "observation": ( + f"new = 1 is exact source evidence for structured role {index}: {field}." + ), + } + for index, field in enumerate( + noema.OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[kind], + start=1, + ) + } def _probe(kind: str, *, hypothesis: str) -> dict[str, object]: @@ -105,7 +118,14 @@ def test_probe_kind_fails_closed_on_malformed_or_unknown_values(probe_kind: obje def test_class_evidence_must_be_source_bound_to_the_probe_location() -> None: verdict = _verdict() probe = verdict["adversarial_validation"]["probes"][0] - probe["class_evidence"]["mutation_attempt"] = {"path": "src/tool.py", "line": 1, "side": "LEFT"} + probe["class_evidence"]["mutation_attempt"] = { + "path": "src/tool.py", + "line": 1, + "side": "LEFT", + "source_excerpt": "old = 1", + "claim_role": noema.OBSERVED_REVIEW_PROBE_CLAIM_ROLES["mutable_alias"]["mutation_attempt"], + "observation": "old = 1 is exact source evidence for the mutation-attempt role.", + } with pytest.raises(noema.NoemaModelOutputError, match="must bind to the probe location"): noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) From b196d3fe65d2d78f98a2f4348afadaf668025346 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:03:25 +0900 Subject: [PATCH 34/59] docs(noema): record successor-check proof --- docs/doctoring/noema-observed-defect-corpus-current-main.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/noema-observed-defect-corpus-current-main.md b/docs/doctoring/noema-observed-defect-corpus-current-main.md index d984186a43..e6cf516ed4 100644 --- a/docs/doctoring/noema-observed-defect-corpus-current-main.md +++ b/docs/doctoring/noema-observed-defect-corpus-current-main.md @@ -10,7 +10,8 @@ JSON booleans are rejected as line coordinates even though Python considers `Tru This repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim. - Exact-head follow-up also makes bounded-diff omission markers ineligible as source evidence. Short identifiers, symbol-only lines, blank changed lines, and non-ASCII source remain admissible through exact string equality rather than lexical guessing. The exact-head structural follow-up removes the fixed English relation-word list. Formal evidence now carries a schema-derived `claim_role` for each defect-class witness, while the deterministic gate verifies exact source identity, canonical coordinates, role identity, and distinct observations. Semantic causal adequacy remains a reviewer/evaluation responsibility; the validator does not pretend English keyword presence proves causality. + +Workflow-local bootstrap or generated commits are not accepted as final review/check proof merely because their source transaction verified locally. The merge candidate must be a workflow-starting successor writer head produced through ordinary owner-side mutation, with the required review and quality checks observed on that exact unchanged head before merge. From 7b145bdf704b8afc671d826229078112b527f655 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:02:31 +0900 Subject: [PATCH 35/59] ci(temp): repair Noema source-evidence edge cases --- ...emp_pr1641_source_evidence_edge_repair.yml | 287 ++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml diff --git a/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml b/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml new file mode 100644 index 0000000000..889b3abd18 --- /dev/null +++ b/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml @@ -0,0 +1,287 @@ +name: Temporary PR1641 source evidence edge repair + +on: + push: + branches: + - fix/noema-observed-defect-corpus-current-main-20260902 + paths: + - .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml + +permissions: + contents: write + +concurrency: + group: temp-pr1641-source-evidence-edge-repair + cancel-in-progress: true + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} + steps: + - name: Checkout exact writer head without persisted mutation credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install hash-locked review dependencies + shell: bash + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ + -r requirements-opencode-review-ci-hashes.txt + + - name: Revalidate exact writer head + shell: bash + run: | + set -euo pipefail + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + local_head="$(git rev-parse HEAD)" + if [ -z "$remote_head" ] || [ "$remote_head" != "$local_head" ]; then + echo "::error::writer head moved: local=$local_head remote=$remote_head" + exit 1 + fi + + - name: Materialize externally demonstrated RED regressions + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + target = Path("tests/test_noema_class_evidence_observation_contract.py") + text = target.read_text(encoding="utf-8") + marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" + if marker not in text: + text = text.rstrip() + r''' + + + def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{source} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." + ) + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + + def test_long_changed_line_uses_structural_exact_source_binding() -> None: + """A source line longer than the prose cap remains reviewable without a vacuous quote.""" + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{source} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." + ) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + ''' + "\n" + target.write_text(text, encoding="utf-8") + PY + + - name: Prove both edge regressions are specifically RED + shell: bash + run: | + set -euo pipefail + specs=( + "tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker" + "tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding" + ) + for spec in "${specs[@]}"; do + log="$(mktemp)" + set +e + python -m pytest -q "$spec" >"$log" 2>&1 + status=$? + set -e + cat "$log" + if [ "$status" -eq 0 ]; then + echo "::error::expected RED regression was already GREEN: $spec" + exit 1 + fi + if ! grep -q "1 failed" "$log"; then + echo "::error::RED was not a test assertion/failure outcome: $spec" + exit 1 + fi + done + + - name: Apply smallest source, prompt, test, and traceability repair + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(path: str, old: str, new: str) -> None: + target = Path(path) + text = target.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"{path}: expected one exact replacement, found {count}") + target.write_text(text.replace(old, new, 1), encoding="utf-8") + + def append_once(path: str, marker: str, addition: str) -> None: + target = Path(path) + text = target.read_text(encoding="utf-8") + if marker in text: + return + target.write_text(text.rstrip() + "\n\n" + addition.strip() + "\n", encoding="utf-8") + + gate = "scripts/ci/noema_review_gate.py" + replace_once( + gate, + ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''', + ''' source_is_blank = not source_excerpt.strip() + source_marker = "" if source_is_blank else source_excerpt + # Exact source identity is already established by equality against the + # trusted changed-line map above. Repeating that source inside bounded + # prose is an additional anti-vacuity signal only when it can fit. + if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''', + ) + replace_once( + gate, + "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior.", + "For an empty or whitespace-only line the observation must quote ; for a nonblank source_excerpt no longer than MAX_THREAD_BODY_CHARS it must quote the exact source_excerpt. Longer nonblank lines remain exactly bound by the separately validated source_excerpt field, while observation stays bounded and explains the claimed behavior.", + ) + + append_once( + "docs/product-technical-gap-baseline.md", + "## 2026-09-02 — Noema exact-source edge binding", + '''## 2026-09-02 — Noema exact-source edge binding + + **Observed review misses.** External review found two edge cases in the + exact changed-line witness contract after the main observed-defect corpus + landed. A whitespace-only changed line was truthy, so incidental spacing + in arbitrary prose could satisfy the quote predicate. Conversely, a + nonblank changed line longer than the 1,200-character observation cap + could never be repeated inside the bounded observation, making every + otherwise valid formal verdict impossible for that location. + + **Owner repair.** Exact source identity remains the equality check between + `class_evidence.source_excerpt` and the trusted changed-side diff map. + Whitespace-only source is normalized to the explicit `` observation + marker. For ordinary bounded lines the observation still quotes the exact + source text. For longer nonblank lines the already-validated structural + `source_excerpt` carries exact identity while the observation stays bounded, + distinct, and bound to the schema-defined claim role. The omission marker + remains ineligible source evidence. + + **Regression corpus.** The class-evidence suite now proves incidental + whitespace cannot stand in for a source quote and that an over-cap changed + line remains admissible without weakening exact source equality. These are + durable false-positive/false-negative fixtures, not vendor-specific wording. + ''', + ) + + changelog = Path("CHANGELOG.md") + text = changelog.read_text(encoding="utf-8") + entry = ( + "- Harden Noema exact-source evidence edge cases: whitespace-only changed lines now require " + "the explicit `` marker, while over-cap nonblank lines use the already-validated " + "structural source excerpt without forcing impossible repetition inside bounded prose.\n" + ) + if entry not in text: + if "## [Unreleased]" in text: + text = text.replace("## [Unreleased]\n", "## [Unreleased]\n" + entry, 1) + elif "## Unreleased" in text: + text = text.replace("## Unreleased\n", "## Unreleased\n" + entry, 1) + else: + text = entry + "\n" + text + changelog.write_text(text, encoding="utf-8") + PY + + - name: Verify focused GREEN contracts + shell: bash + run: | + set -euo pipefail + python -m compileall -q scripts/ci + python -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py \ + tests/test_noema_observed_defect_corpus_current_main.py + git diff --check + + - name: Verify broader Noema suite + shell: bash + run: | + set -euo pipefail + python -m pytest -q tests/test_noema_*.py + git diff --check + + - name: Remove temporary repair identity + shell: bash + run: | + set -euo pipefail + rm -f .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml + if git ls-files | grep -F '_temp_pr1641_source_evidence_edge_repair'; then + echo "::error::temporary PR1641 repair identity remains tracked" + exit 1 + fi + git diff --check + + - name: Commit verified successor and trigger exact-head checks + shell: bash + run: | + set -euo pipefail + if [ -z "${WORKFLOW_PUSH_TOKEN:-}" ]; then + echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." + exit 1 + fi + git fetch origin "${GITHUB_REF_NAME}" + remote_head="$(git rev-parse "origin/${GITHUB_REF_NAME}")" + local_parent="$(git rev-parse HEAD)" + if [ "$remote_head" != "$local_parent" ]; then + echo "::error::writer branch moved before publish: local=$local_parent remote=$remote_head" + exit 1 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + if git diff --cached --quiet; then + echo "::error::repair produced no publishable delta" + exit 1 + fi + git commit -m "fix(noema): harden exact-source evidence edge cases" + git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push origin "HEAD:${GITHUB_REF_NAME}" From 04435650ce884e8cf3e403cf1934294da3f72a1a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:10:11 +0900 Subject: [PATCH 36/59] ci: move PR 1641 edge repair off saturated runner pool --- .../workflows/_temp_pr1641_source_evidence_edge_repair.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml b/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml index 889b3abd18..43ca0701bf 100644 --- a/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml +++ b/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml @@ -11,13 +11,13 @@ permissions: contents: write concurrency: - group: temp-pr1641-source-evidence-edge-repair + group: temp-pr1641-source-evidence-edge-repair-${{ github.repository }}-${{ github.ref_name }} cancel-in-progress: true jobs: repair: if: github.repository == 'ContextualWisdomLab/.github' - runs-on: ubuntu-latest + runs-on: ubuntu-slim timeout-minutes: 45 env: WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} From 812853eb90f831280e9420267efb50ab4090da0c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:15:27 +0900 Subject: [PATCH 37/59] ci: add robust PR 1641 edge repair --- .../workflows/_temp_pr1641_edge_repair_v2.yml | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 .github/workflows/_temp_pr1641_edge_repair_v2.yml diff --git a/.github/workflows/_temp_pr1641_edge_repair_v2.yml b/.github/workflows/_temp_pr1641_edge_repair_v2.yml new file mode 100644 index 0000000000..4fa9e60138 --- /dev/null +++ b/.github/workflows/_temp_pr1641_edge_repair_v2.yml @@ -0,0 +1,168 @@ +name: Temporary PR1641 robust edge repair + +on: + push: + branches: + - fix/noema-observed-defect-corpus-current-main-20260902 + +concurrency: + group: temp-pr1641-edge-v2-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + repair: + runs-on: ubuntu-slim + timeout-minutes: 30 + steps: + - name: Checkout exact writer head + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14" + + - name: Install hash-locked test tooling + run: >- + python -m pip install --disable-pip-version-check --require-hashes + --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt + + - name: Re-prove RED and apply exact-source repair + env: + EXPECTED_HEAD: ${{ github.sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD" + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + test "$remote_head" = "$EXPECTED_HEAD" + + python - <<'PY' + from pathlib import Path + + target = Path("tests/test_noema_class_evidence_observation_contract.py") + text = target.read_text() + marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" + if marker not in text: + text = text.rstrip() + r''' + + +def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = f"Incidental spacing for {probe['probe_kind']}:{field}." + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + +def test_long_changed_line_uses_structural_exact_source_binding() -> None: + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = f"Bounded structural observation for {probe['probe_kind']}:{field}." + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) +''' + "\n" + target.write_text(text) + PY + + for spec in \ + tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker \ + tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding; do + set +e + python -m pytest -q "$spec" >/tmp/red.log 2>&1 + rc=$? + set -e + cat /tmp/red.log + test "$rc" -ne 0 + grep -q "1 failed" /tmp/red.log + done + + python - <<'PY' + from pathlib import Path + + gate = Path("scripts/ci/noema_review_gate.py") + text = gate.read_text() + start_marker = ' source_marker = source_excerpt if source_excerpt else ""\n' + end_marker = ' expected_claim_role = OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field]\n' + start = text.index(start_marker) + end = text.index(end_marker, start) + replacement = ''' source_is_blank = not source_excerpt.strip() + source_marker = "" if source_is_blank else source_excerpt + # Exact source identity is already established by equality against the + # trusted changed-line map. Repetition inside bounded prose is required + # only when the source can fit; blank/whitespace lines use . + if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) +''' + text = text[:start] + replacement + text[end:] + old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." + new_prompt = "For an empty or whitespace-only source line the observation must quote ; for a bounded nonblank source_excerpt it must quote the exact text. Longer nonblank lines remain exactly bound by source_excerpt equality while observation stays bounded and explains the claim." + if old_prompt in text: + text = text.replace(old_prompt, new_prompt, 1) + gate.write_text(text) + + docs = Path("docs/product-technical-gap-baseline.md") + dtext = docs.read_text() + heading = "## 2026-09-02 — Noema exact-source edge binding" + if heading not in dtext: + docs.write_text(dtext.rstrip() + "\n\n" + heading + "\n\nExact source identity remains the equality check between `class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only changed lines now require the explicit `` observation marker; nonblank changed lines longer than the bounded observation cap remain admissible because structural source equality, not impossible prose repetition, is authoritative. Regression fixtures cover both edge orderings.\n") + + changelog = Path("CHANGELOG.md") + ctext = changelog.read_text() + entry = "- Harden Noema exact-source evidence edges: whitespace-only changed lines require ``, while over-cap nonblank lines retain exact structural source binding without impossible bounded-prose repetition.\n" + if entry not in ctext: + heading = "## [Unreleased]\n" + if heading in ctext: + ctext = ctext.replace(heading, heading + entry, 1) + else: + ctext = entry + ctext + changelog.write_text(ctext) + PY + + git add scripts/ci/noema_review_gate.py tests/test_noema_class_evidence_observation_contract.py docs/product-technical-gap-baseline.md CHANGELOG.md + python -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py \ + tests/test_noema_observed_defect_corpus_current_main.py + python -m pytest -q tests/test_noema_*.py + python -m compileall -q scripts/ci + git diff --check + + git rm -f .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml + git rm -f .github/workflows/_temp_pr1641_edge_repair_v2.yml + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add -A + git diff --cached --check + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + test "$remote_head" = "$EXPECTED_HEAD" + git commit -m "fix(noema): harden exact-source evidence edges" + git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}" From f445324f11bc7f2841d55eaf8b36e0a6bebe6d8a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:32:06 +0900 Subject: [PATCH 38/59] ci: add fail-closed PR 1641 exact-source edge repair --- .../workflows/_temp_pr1641_edge_repair_v3.yml | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 .github/workflows/_temp_pr1641_edge_repair_v3.yml diff --git a/.github/workflows/_temp_pr1641_edge_repair_v3.yml b/.github/workflows/_temp_pr1641_edge_repair_v3.yml new file mode 100644 index 0000000000..6210c9b84c --- /dev/null +++ b/.github/workflows/_temp_pr1641_edge_repair_v3.yml @@ -0,0 +1,217 @@ +name: Temporary PR1641 exact-source edge repair v3 + +on: + push: + branches: + - fix/noema-observed-defect-corpus-current-main-20260902 + paths: + - .github/workflows/_temp_pr1641_edge_repair_v3.yml + +concurrency: + group: temp-pr1641-edge-v3-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' + runs-on: ubuntu-slim + timeout-minutes: 45 + env: + WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + steps: + - name: Checkout exact writer head + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14" + + - name: Install hash-locked review dependencies + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt + + - name: Revalidate exact writer head + env: + EXPECTED_HEAD: ${{ github.sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD" + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + test -n "$remote_head" + test "$remote_head" = "$EXPECTED_HEAD" + + - name: Add edge regressions and prove RED + run: | + set -euo pipefail + if ! grep -q '^def test_whitespace_only_changed_source_requires_explicit_blank_marker' tests/test_noema_class_evidence_observation_contract.py; then + cat >> tests/test_noema_class_evidence_observation_contract.py <<'PYTEST' + + +def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." + ) + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + +def test_long_changed_line_uses_structural_exact_source_binding() -> None: + """An over-cap changed line remains reviewable without impossible prose repetition.""" + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." + ) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) +PYTEST + fi + + for spec in \ + tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker \ + tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding; do + log="$(mktemp)" + set +e + python -m pytest -q "$spec" >"$log" 2>&1 + rc=$? + set -e + cat "$log" + test "$rc" -ne 0 + grep -q '1 failed' "$log" + done + + - name: Apply production and traceability repair + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + gate = Path("scripts/ci/noema_review_gate.py") + text = gate.read_text(encoding="utf-8") + old = ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''' + new = ''' source_is_blank = not source_excerpt.strip() + source_marker = "" if source_is_blank else source_excerpt + # Exact source identity is already established by equality against the + # trusted changed-line map. Repetition inside bounded prose is an + # anti-vacuity signal only when the source can fit; blank lines use + # an explicit structural marker rather than incidental whitespace. + if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''' + if text.count(old) != 1: + raise SystemExit(f"unexpected source quote guard count: {text.count(old)}") + text = text.replace(old, new, 1) + old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." + new_prompt = ( + "For an empty or whitespace-only source line the observation must quote ; " + "for a bounded nonblank source_excerpt it must quote the exact text. Longer nonblank " + "lines remain exactly bound by source_excerpt equality while observation stays bounded " + "and explains the claimed behavior." + ) + if old_prompt in text: + text = text.replace(old_prompt, new_prompt, 1) + gate.write_text(text, encoding="utf-8") + + docs = Path("docs/product-technical-gap-baseline.md") + dtext = docs.read_text(encoding="utf-8") + heading = "## 2026-09-02 — Noema exact-source edge binding" + if heading not in dtext: + dtext = dtext.rstrip() + "\n\n" + heading + "\n\n" + dtext += ( + "External exact-head review exposed two executable source-evidence edge defects: " + "whitespace-only changed lines could be admitted by incidental spacing, while a " + "nonblank changed line longer than the bounded observation field could become " + "structurally impossible to admit. Exact identity remains equality between " + "`class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only " + "source now requires the explicit `` observation marker; over-cap nonblank lines " + "remain exactly source-bound without requiring impossible prose repetition. Focused " + "regressions preserve both contracts.\n" + ) + docs.write_text(dtext, encoding="utf-8") + + changelog = Path("CHANGELOG.md") + ctext = changelog.read_text(encoding="utf-8") + entry = ( + "- Harden Noema exact-source evidence edges: whitespace-only changed lines require " + "``, while over-cap nonblank lines retain exact structural source binding without " + "impossible bounded-prose repetition.\n" + ) + if entry not in ctext: + marker = "## [Unreleased]\n" + ctext = ctext.replace(marker, marker + entry, 1) if marker in ctext else entry + "\n" + ctext + changelog.write_text(ctext, encoding="utf-8") + PY + + - name: Verify GREEN and remove superseded repair machinery + run: | + set -euo pipefail + python -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py \ + tests/test_noema_observed_defect_corpus_current_main.py + python -m pytest -q tests/test_noema_*.py + python -m compileall -q scripts/ci + git diff --check + git rm -f --ignore-unmatch \ + .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ + .github/workflows/_temp_pr1641_edge_repair_v2.yml \ + .github/repair-noema-observed-defect-corpus.py \ + .github/workflows/_source_pointer.yml + git diff --check + + - name: Publish verified successor + env: + EXPECTED_HEAD: ${{ github.sha }} + run: | + set -euo pipefail + test -n "${WORKFLOW_PUSH_TOKEN:-}" + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + test "$remote_head" = "$EXPECTED_HEAD" + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add -A + git diff --cached --check + test -n "$(git diff --cached --name-only)" + git commit -m "fix(noema): harden exact-source evidence edges" + git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}" From 168f0537b44c9484c064c0e578c90acfd272d3e4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:38:08 +0900 Subject: [PATCH 39/59] ci: add executable PR 1641 exact-source edge repair --- scripts/ci/temp_pr1641_edge_repair.py | 182 ++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 scripts/ci/temp_pr1641_edge_repair.py diff --git a/scripts/ci/temp_pr1641_edge_repair.py b/scripts/ci/temp_pr1641_edge_repair.py new file mode 100644 index 0000000000..c4cb1297af --- /dev/null +++ b/scripts/ci/temp_pr1641_edge_repair.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""One-shot exact-source edge repair for PR 1641; deletes itself after GREEN.""" +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +TEST = ROOT / "tests/test_noema_class_evidence_observation_contract.py" +GATE = ROOT / "scripts/ci/noema_review_gate.py" +DOCS = ROOT / "docs/product-technical-gap-baseline.md" +CHANGELOG = ROOT / "CHANGELOG.md" + +WHITESPACE_TEST = r''' + + +def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." + ) + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) +''' + +LONG_TEST = r''' + + +def test_long_changed_line_uses_structural_exact_source_binding() -> None: + """An over-cap changed line remains reviewable without impossible prose repetition.""" + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." + ) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) +''' + + +def run(*args: str, expect: int = 0) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True) + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + if result.returncode != expect: + raise SystemExit(f"command {args!r} returned {result.returncode}, expected {expect}") + return result + + +def append_regressions() -> None: + text = TEST.read_text(encoding="utf-8") + if "def test_whitespace_only_changed_source_requires_explicit_blank_marker" not in text: + text += WHITESPACE_TEST + if "def test_long_changed_line_uses_structural_exact_source_binding" not in text: + text += LONG_TEST + TEST.write_text(text, encoding="utf-8") + + +def prove_red() -> None: + specs = ( + "tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker", + "tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding", + ) + for spec in specs: + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", spec], + cwd=ROOT, + text=True, + capture_output=True, + ) + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + if result.returncode == 0 or "1 failed" not in (result.stdout + result.stderr): + raise SystemExit(f"expected focused RED regression did not fail exactly: {spec}") + + +def repair_source() -> None: + text = GATE.read_text(encoding="utf-8") + old = ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) +''' + new = ''' source_is_blank = not source_excerpt.strip() + source_marker = "" if source_is_blank else source_excerpt + # Exact source identity is already established by equality against the + # trusted changed-line map. Repetition inside bounded prose is an + # anti-vacuity signal only when the source can fit; blank lines use + # an explicit structural marker rather than incidental whitespace. + if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) +''' + if text.count(old) != 1: + raise SystemExit(f"unexpected source quote guard count: {text.count(old)}") + text = text.replace(old, new, 1) + old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." + new_prompt = ( + "For an empty or whitespace-only source line the observation must quote ; " + "for a bounded nonblank source_excerpt it must quote the exact text. Longer nonblank " + "lines remain exactly bound by source_excerpt equality while observation stays bounded " + "and explains the claimed behavior." + ) + if old_prompt in text: + text = text.replace(old_prompt, new_prompt, 1) + GATE.write_text(text, encoding="utf-8") + + +def update_traceability() -> None: + dtext = DOCS.read_text(encoding="utf-8") + heading = "## 2026-09-02 — Noema exact-source edge binding" + if heading not in dtext: + dtext = dtext.rstrip() + "\n\n" + heading + "\n\n" + dtext += ( + "External exact-head review exposed two executable source-evidence edge defects: " + "whitespace-only changed lines could be admitted by incidental spacing, while a " + "nonblank changed line longer than the bounded observation field could become " + "structurally impossible to admit. Exact identity remains equality between " + "`class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only " + "source now requires the explicit `` observation marker; over-cap nonblank lines " + "remain exactly source-bound without requiring impossible prose repetition. Focused " + "regressions preserve both contracts.\n" + ) + DOCS.write_text(dtext, encoding="utf-8") + + ctext = CHANGELOG.read_text(encoding="utf-8") + entry = ( + "- Harden Noema exact-source evidence edges: whitespace-only changed lines require " + "``, while over-cap nonblank lines retain exact structural source binding without " + "impossible bounded-prose repetition.\n" + ) + if entry not in ctext: + marker = "## [Unreleased]\n" + ctext = ctext.replace(marker, marker + entry, 1) if marker in ctext else entry + "\n" + ctext + CHANGELOG.write_text(ctext, encoding="utf-8") + + +def prove_green() -> None: + run(sys.executable, "-m", "pytest", "-q", "tests/test_noema_class_evidence_observation_contract.py", "tests/test_noema_observed_defect_corpus_current_main.py") + run(sys.executable, "-m", "pytest", "-q", "tests/test_noema_*.py") + run(sys.executable, "-m", "compileall", "-q", "scripts/ci") + run("git", "diff", "--check") + + +def main() -> None: + append_regressions() + prove_red() + repair_source() + update_traceability() + prove_green() + Path(__file__).unlink() + + +if __name__ == "__main__": + main() From cdcd2203c050b35377e68433d56e36f34208cf56 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:38:25 +0900 Subject: [PATCH 40/59] ci: make PR 1641 edge repair executable --- .../workflows/_temp_pr1641_edge_repair_v3.yml | 151 +----------------- 1 file changed, 3 insertions(+), 148 deletions(-) diff --git a/.github/workflows/_temp_pr1641_edge_repair_v3.yml b/.github/workflows/_temp_pr1641_edge_repair_v3.yml index 6210c9b84c..d7699a97b8 100644 --- a/.github/workflows/_temp_pr1641_edge_repair_v3.yml +++ b/.github/workflows/_temp_pr1641_edge_repair_v3.yml @@ -49,157 +49,12 @@ jobs: test -n "$remote_head" test "$remote_head" = "$EXPECTED_HEAD" - - name: Add edge regressions and prove RED + - name: Prove RED, repair owner source, and prove GREEN run: | set -euo pipefail - if ! grep -q '^def test_whitespace_only_changed_source_requires_explicit_blank_marker' tests/test_noema_class_evidence_observation_contract.py; then - cat >> tests/test_noema_class_evidence_observation_contract.py <<'PYTEST' + python scripts/ci/temp_pr1641_edge_repair.py - -def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: - """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" - source = " " - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." - ) - with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) - - -def test_long_changed_line_uses_structural_exact_source_binding() -> None: - """An over-cap changed line remains reviewable without impossible prose repetition.""" - source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." - ) - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) -PYTEST - fi - - for spec in \ - tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker \ - tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding; do - log="$(mktemp)" - set +e - python -m pytest -q "$spec" >"$log" 2>&1 - rc=$? - set -e - cat "$log" - test "$rc" -ne 0 - grep -q '1 failed' "$log" - done - - - name: Apply production and traceability repair - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - gate = Path("scripts/ci/noema_review_gate.py") - text = gate.read_text(encoding="utf-8") - old = ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - ''' - new = ''' source_is_blank = not source_excerpt.strip() - source_marker = "" if source_is_blank else source_excerpt - # Exact source identity is already established by equality against the - # trusted changed-line map. Repetition inside bounded prose is an - # anti-vacuity signal only when the source can fit; blank lines use - # an explicit structural marker rather than incidental whitespace. - if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - ''' - if text.count(old) != 1: - raise SystemExit(f"unexpected source quote guard count: {text.count(old)}") - text = text.replace(old, new, 1) - old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." - new_prompt = ( - "For an empty or whitespace-only source line the observation must quote ; " - "for a bounded nonblank source_excerpt it must quote the exact text. Longer nonblank " - "lines remain exactly bound by source_excerpt equality while observation stays bounded " - "and explains the claimed behavior." - ) - if old_prompt in text: - text = text.replace(old_prompt, new_prompt, 1) - gate.write_text(text, encoding="utf-8") - - docs = Path("docs/product-technical-gap-baseline.md") - dtext = docs.read_text(encoding="utf-8") - heading = "## 2026-09-02 — Noema exact-source edge binding" - if heading not in dtext: - dtext = dtext.rstrip() + "\n\n" + heading + "\n\n" - dtext += ( - "External exact-head review exposed two executable source-evidence edge defects: " - "whitespace-only changed lines could be admitted by incidental spacing, while a " - "nonblank changed line longer than the bounded observation field could become " - "structurally impossible to admit. Exact identity remains equality between " - "`class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only " - "source now requires the explicit `` observation marker; over-cap nonblank lines " - "remain exactly source-bound without requiring impossible prose repetition. Focused " - "regressions preserve both contracts.\n" - ) - docs.write_text(dtext, encoding="utf-8") - - changelog = Path("CHANGELOG.md") - ctext = changelog.read_text(encoding="utf-8") - entry = ( - "- Harden Noema exact-source evidence edges: whitespace-only changed lines require " - "``, while over-cap nonblank lines retain exact structural source binding without " - "impossible bounded-prose repetition.\n" - ) - if entry not in ctext: - marker = "## [Unreleased]\n" - ctext = ctext.replace(marker, marker + entry, 1) if marker in ctext else entry + "\n" + ctext - changelog.write_text(ctext, encoding="utf-8") - PY - - - name: Verify GREEN and remove superseded repair machinery - run: | - set -euo pipefail - python -m pytest -q \ - tests/test_noema_class_evidence_observation_contract.py \ - tests/test_noema_observed_defect_corpus_current_main.py - python -m pytest -q tests/test_noema_*.py - python -m compileall -q scripts/ci - git diff --check - git rm -f --ignore-unmatch \ - .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ - .github/workflows/_temp_pr1641_edge_repair_v2.yml \ - .github/repair-noema-observed-defect-corpus.py \ - .github/workflows/_source_pointer.yml - git diff --check - - - name: Publish verified successor + - name: Publish verified successor without workflow mutation env: EXPECTED_HEAD: ${{ github.sha }} run: | From 361d9fb3e97856179a98d7fbef4d1808e9269dbb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 12:02:39 +0900 Subject: [PATCH 41/59] ci(noema): execute and self-clean exact-source edge repair --- .../workflows/_temp_pr1641_edge_repair_v3.yml | 202 ++++++++++++++++-- 1 file changed, 187 insertions(+), 15 deletions(-) diff --git a/.github/workflows/_temp_pr1641_edge_repair_v3.yml b/.github/workflows/_temp_pr1641_edge_repair_v3.yml index d7699a97b8..dffcbdf8f0 100644 --- a/.github/workflows/_temp_pr1641_edge_repair_v3.yml +++ b/.github/workflows/_temp_pr1641_edge_repair_v3.yml @@ -20,10 +20,10 @@ jobs: runs-on: ubuntu-slim timeout-minutes: 45 env: - WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} steps: - name: Checkout exact writer head - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 with: ref: ${{ github.sha }} fetch-depth: 0 @@ -35,38 +35,210 @@ jobs: python-version: "3.14" - name: Install hash-locked review dependencies + shell: bash run: | set -euo pipefail - python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt + python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ + -r requirements-opencode-review-ci-hashes.txt - name: Revalidate exact writer head - env: - EXPECTED_HEAD: ${{ github.sha }} + shell: bash run: | set -euo pipefail - test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD" + test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" test -n "$remote_head" - test "$remote_head" = "$EXPECTED_HEAD" + test "$remote_head" = "${GITHUB_SHA}" - - name: Prove RED, repair owner source, and prove GREEN + - name: Materialize and prove exact-source edge REDs + shell: bash run: | set -euo pipefail - python scripts/ci/temp_pr1641_edge_repair.py + python - <<'PY' + from pathlib import Path - - name: Publish verified successor without workflow mutation - env: - EXPECTED_HEAD: ${{ github.sha }} + target = Path("tests/test_noema_class_evidence_observation_contract.py") + text = target.read_text(encoding="utf-8") + marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" + if marker not in text: + text = text.rstrip() + r''' + + +def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." + ) + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + +def test_long_changed_line_uses_structural_exact_source_binding() -> None: + """A source line longer than the prose cap remains reviewable through structural equality.""" + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{source} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." + ) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) +''' + "\n" + target.write_text(text, encoding="utf-8") + PY + + for spec in \ + tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker \ + tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding; do + log="$(mktemp)" + set +e + python -m pytest -q "$spec" >"$log" 2>&1 + rc=$? + set -e + cat "$log" + test "$rc" -ne 0 + grep -q "1 failed" "$log" + done + + - name: Apply smallest causal GREEN and traceability + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(path: str, old: str, new: str) -> None: + target = Path(path) + text = target.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"{path}: expected one exact replacement, found {count}") + target.write_text(text.replace(old, new, 1), encoding="utf-8") + + def append_once(path: str, marker: str, addition: str) -> None: + target = Path(path) + text = target.read_text(encoding="utf-8") + if marker not in text: + target.write_text(text.rstrip() + "\n\n" + addition.strip() + "\n", encoding="utf-8") + + gate = "scripts/ci/noema_review_gate.py" + replace_once( + gate, + ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''', + ''' source_is_blank = not source_excerpt.strip() + source_marker = "" if source_is_blank else source_excerpt + # Exact source identity is already proven by equality with the trusted + # changed-line map. Repetition in bounded prose is an extra anti-vacuity + # signal only when the nonblank source itself fits that bounded field. + if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''', + ) + replace_once( + gate, + "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior.", + "For an empty or whitespace-only line the observation must quote ; for a nonblank source_excerpt no longer than MAX_THREAD_BODY_CHARS it must quote the exact source_excerpt. Longer nonblank lines remain exactly bound by the separately validated source_excerpt field, while observation stays bounded and explains the claimed behavior.", + ) + + append_once( + "docs/product-technical-gap-baseline.md", + "## 2026-09-02 — Noema exact-source edge binding", + '''## 2026-09-02 — Noema exact-source edge binding + + External review exposed two deterministic evidence-edge failures in the observed-defect corpus. A whitespace-only changed line could satisfy the source-quote predicate through incidental spacing in otherwise generic prose, while a nonblank changed line longer than the bounded observation field could never be repeated and therefore made a valid formal verdict impossible. Exact source identity remains the equality check between `class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only source now requires the explicit `` marker; over-cap nonblank lines retain structural exact-source equality while bounded observations remain distinct and claim-role-bound. Regression fixtures preserve both cases without vendor-specific wording.''', + ) + append_once( + "docs/doctoring/noema-observed-defect-corpus-current-main.md", + "## Exact-source edge follow-up (2026-09-02)", + '''## Exact-source edge follow-up (2026-09-02) + + The corpus now includes a weak-oracle negative control for whitespace-only changed lines and a representability regression for changed lines longer than the observation cap. The deterministic gate treats source equality as authoritative identity, uses `` for blank/whitespace-only lines, and does not require an impossible copy of an over-cap source line into bounded prose.''', + ) + + changelog = Path("CHANGELOG.md") + text = changelog.read_text(encoding="utf-8") + entry = "- Harden Noema exact-source evidence edges: whitespace-only lines require ``, while over-cap nonblank lines retain structural exact-source binding without impossible bounded-prose repetition.\n" + if entry not in text: + if "## [Unreleased]\n" in text: + text = text.replace("## [Unreleased]\n", "## [Unreleased]\n" + entry, 1) + elif "## Unreleased\n" in text: + text = text.replace("## Unreleased\n", "## Unreleased\n" + entry, 1) + else: + text = entry + "\n" + text + changelog.write_text(text, encoding="utf-8") + PY + + - name: Verify focused and broader GREEN + shell: bash + run: | + set -euo pipefail + python -m compileall -q scripts/ci + python -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py \ + tests/test_noema_observed_defect_corpus_current_main.py + python -m pytest -q tests/test_noema_*.py + git diff --check + + - name: Remove every temporary PR1641 repair identity + shell: bash + run: | + set -euo pipefail + rm -f \ + .github/workflows/_temp_pr1641_edge_repair_v2.yml \ + .github/workflows/_temp_pr1641_edge_repair_v3.yml \ + .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ + scripts/ci/temp_pr1641_edge_repair.py + if git ls-files | grep -E '(^|/)_?temp_pr1641|temp-pr1641'; then + echo "::error::temporary PR1641 repair identity remains tracked" + exit 1 + fi + git diff --check + + - name: Publish verified successor with workflow-starting credential + shell: bash run: | set -euo pipefail - test -n "${WORKFLOW_PUSH_TOKEN:-}" + if [ -z "${WORKFLOW_PUSH_TOKEN:-}" ]; then + echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." + exit 1 + fi remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - test "$remote_head" = "$EXPECTED_HEAD" + test "$remote_head" = "${GITHUB_SHA}" git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com git add -A git diff --cached --check test -n "$(git diff --cached --name-only)" - git commit -m "fix(noema): harden exact-source evidence edges" + git commit -m "fix(noema): harden exact-source evidence edge cases" git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}" From 21368261952a46e880d151300567a550c9822806 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 13:00:10 +0900 Subject: [PATCH 42/59] chore(noema): remove superseded PR1641 repair workflow --- .../workflows/_temp_pr1641_edge_repair_v2.yml | 168 ------------------ 1 file changed, 168 deletions(-) delete mode 100644 .github/workflows/_temp_pr1641_edge_repair_v2.yml diff --git a/.github/workflows/_temp_pr1641_edge_repair_v2.yml b/.github/workflows/_temp_pr1641_edge_repair_v2.yml deleted file mode 100644 index 4fa9e60138..0000000000 --- a/.github/workflows/_temp_pr1641_edge_repair_v2.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: Temporary PR1641 robust edge repair - -on: - push: - branches: - - fix/noema-observed-defect-corpus-current-main-20260902 - -concurrency: - group: temp-pr1641-edge-v2-${{ github.repository }}-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - contents: write - -jobs: - repair: - runs-on: ubuntu-slim - timeout-minutes: 30 - steps: - - name: Checkout exact writer head - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: true - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 - with: - python-version: "3.14" - - - name: Install hash-locked test tooling - run: >- - python -m pip install --disable-pip-version-check --require-hashes - --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt - - - name: Re-prove RED and apply exact-source repair - env: - EXPECTED_HEAD: ${{ github.sha }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD" - remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - test "$remote_head" = "$EXPECTED_HEAD" - - python - <<'PY' - from pathlib import Path - - target = Path("tests/test_noema_class_evidence_observation_contract.py") - text = target.read_text() - marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" - if marker not in text: - text = text.rstrip() + r''' - - -def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: - source = " " - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = f"Incidental spacing for {probe['probe_kind']}:{field}." - with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) - - -def test_long_changed_line_uses_structural_exact_source_binding() -> None: - source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = f"Bounded structural observation for {probe['probe_kind']}:{field}." - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) -''' + "\n" - target.write_text(text) - PY - - for spec in \ - tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker \ - tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding; do - set +e - python -m pytest -q "$spec" >/tmp/red.log 2>&1 - rc=$? - set -e - cat /tmp/red.log - test "$rc" -ne 0 - grep -q "1 failed" /tmp/red.log - done - - python - <<'PY' - from pathlib import Path - - gate = Path("scripts/ci/noema_review_gate.py") - text = gate.read_text() - start_marker = ' source_marker = source_excerpt if source_excerpt else ""\n' - end_marker = ' expected_claim_role = OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field]\n' - start = text.index(start_marker) - end = text.index(end_marker, start) - replacement = ''' source_is_blank = not source_excerpt.strip() - source_marker = "" if source_is_blank else source_excerpt - # Exact source identity is already established by equality against the - # trusted changed-line map. Repetition inside bounded prose is required - # only when the source can fit; blank/whitespace lines use . - if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) -''' - text = text[:start] + replacement + text[end:] - old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." - new_prompt = "For an empty or whitespace-only source line the observation must quote ; for a bounded nonblank source_excerpt it must quote the exact text. Longer nonblank lines remain exactly bound by source_excerpt equality while observation stays bounded and explains the claim." - if old_prompt in text: - text = text.replace(old_prompt, new_prompt, 1) - gate.write_text(text) - - docs = Path("docs/product-technical-gap-baseline.md") - dtext = docs.read_text() - heading = "## 2026-09-02 — Noema exact-source edge binding" - if heading not in dtext: - docs.write_text(dtext.rstrip() + "\n\n" + heading + "\n\nExact source identity remains the equality check between `class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only changed lines now require the explicit `` observation marker; nonblank changed lines longer than the bounded observation cap remain admissible because structural source equality, not impossible prose repetition, is authoritative. Regression fixtures cover both edge orderings.\n") - - changelog = Path("CHANGELOG.md") - ctext = changelog.read_text() - entry = "- Harden Noema exact-source evidence edges: whitespace-only changed lines require ``, while over-cap nonblank lines retain exact structural source binding without impossible bounded-prose repetition.\n" - if entry not in ctext: - heading = "## [Unreleased]\n" - if heading in ctext: - ctext = ctext.replace(heading, heading + entry, 1) - else: - ctext = entry + ctext - changelog.write_text(ctext) - PY - - git add scripts/ci/noema_review_gate.py tests/test_noema_class_evidence_observation_contract.py docs/product-technical-gap-baseline.md CHANGELOG.md - python -m pytest -q \ - tests/test_noema_class_evidence_observation_contract.py \ - tests/test_noema_observed_defect_corpus_current_main.py - python -m pytest -q tests/test_noema_*.py - python -m compileall -q scripts/ci - git diff --check - - git rm -f .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml - git rm -f .github/workflows/_temp_pr1641_edge_repair_v2.yml - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git add -A - git diff --cached --check - remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - test "$remote_head" = "$EXPECTED_HEAD" - git commit -m "fix(noema): harden exact-source evidence edges" - git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}" From a78131a5ab6ad3cbc9608d5d5bca367957c476e4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 13:00:14 +0900 Subject: [PATCH 43/59] chore(noema): remove superseded PR1641 edge workflow --- .../workflows/_temp_pr1641_edge_repair_v3.yml | 244 ------------------ 1 file changed, 244 deletions(-) delete mode 100644 .github/workflows/_temp_pr1641_edge_repair_v3.yml diff --git a/.github/workflows/_temp_pr1641_edge_repair_v3.yml b/.github/workflows/_temp_pr1641_edge_repair_v3.yml deleted file mode 100644 index dffcbdf8f0..0000000000 --- a/.github/workflows/_temp_pr1641_edge_repair_v3.yml +++ /dev/null @@ -1,244 +0,0 @@ -name: Temporary PR1641 exact-source edge repair v3 - -on: - push: - branches: - - fix/noema-observed-defect-corpus-current-main-20260902 - paths: - - .github/workflows/_temp_pr1641_edge_repair_v3.yml - -concurrency: - group: temp-pr1641-edge-v3-${{ github.repository }}-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - contents: write - -jobs: - repair: - if: github.repository == 'ContextualWisdomLab/.github' - runs-on: ubuntu-slim - timeout-minutes: 45 - env: - WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} - steps: - - name: Checkout exact writer head - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 - with: - python-version: "3.14" - - - name: Install hash-locked review dependencies - shell: bash - run: | - set -euo pipefail - python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ - -r requirements-opencode-review-ci-hashes.txt - - - name: Revalidate exact writer head - shell: bash - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" - remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - test -n "$remote_head" - test "$remote_head" = "${GITHUB_SHA}" - - - name: Materialize and prove exact-source edge REDs - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - target = Path("tests/test_noema_class_evidence_observation_contract.py") - text = target.read_text(encoding="utf-8") - marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" - if marker not in text: - text = text.rstrip() + r''' - - -def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: - """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" - source = " " - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." - ) - with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) - - -def test_long_changed_line_uses_structural_exact_source_binding() -> None: - """A source line longer than the prose cap remains reviewable through structural equality.""" - source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." - ) - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) -''' + "\n" - target.write_text(text, encoding="utf-8") - PY - - for spec in \ - tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker \ - tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding; do - log="$(mktemp)" - set +e - python -m pytest -q "$spec" >"$log" 2>&1 - rc=$? - set -e - cat "$log" - test "$rc" -ne 0 - grep -q "1 failed" "$log" - done - - - name: Apply smallest causal GREEN and traceability - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - def replace_once(path: str, old: str, new: str) -> None: - target = Path(path) - text = target.read_text(encoding="utf-8") - count = text.count(old) - if count != 1: - raise SystemExit(f"{path}: expected one exact replacement, found {count}") - target.write_text(text.replace(old, new, 1), encoding="utf-8") - - def append_once(path: str, marker: str, addition: str) -> None: - target = Path(path) - text = target.read_text(encoding="utf-8") - if marker not in text: - target.write_text(text.rstrip() + "\n\n" + addition.strip() + "\n", encoding="utf-8") - - gate = "scripts/ci/noema_review_gate.py" - replace_once( - gate, - ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - ''', - ''' source_is_blank = not source_excerpt.strip() - source_marker = "" if source_is_blank else source_excerpt - # Exact source identity is already proven by equality with the trusted - # changed-line map. Repetition in bounded prose is an extra anti-vacuity - # signal only when the nonblank source itself fits that bounded field. - if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - ''', - ) - replace_once( - gate, - "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior.", - "For an empty or whitespace-only line the observation must quote ; for a nonblank source_excerpt no longer than MAX_THREAD_BODY_CHARS it must quote the exact source_excerpt. Longer nonblank lines remain exactly bound by the separately validated source_excerpt field, while observation stays bounded and explains the claimed behavior.", - ) - - append_once( - "docs/product-technical-gap-baseline.md", - "## 2026-09-02 — Noema exact-source edge binding", - '''## 2026-09-02 — Noema exact-source edge binding - - External review exposed two deterministic evidence-edge failures in the observed-defect corpus. A whitespace-only changed line could satisfy the source-quote predicate through incidental spacing in otherwise generic prose, while a nonblank changed line longer than the bounded observation field could never be repeated and therefore made a valid formal verdict impossible. Exact source identity remains the equality check between `class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only source now requires the explicit `` marker; over-cap nonblank lines retain structural exact-source equality while bounded observations remain distinct and claim-role-bound. Regression fixtures preserve both cases without vendor-specific wording.''', - ) - append_once( - "docs/doctoring/noema-observed-defect-corpus-current-main.md", - "## Exact-source edge follow-up (2026-09-02)", - '''## Exact-source edge follow-up (2026-09-02) - - The corpus now includes a weak-oracle negative control for whitespace-only changed lines and a representability regression for changed lines longer than the observation cap. The deterministic gate treats source equality as authoritative identity, uses `` for blank/whitespace-only lines, and does not require an impossible copy of an over-cap source line into bounded prose.''', - ) - - changelog = Path("CHANGELOG.md") - text = changelog.read_text(encoding="utf-8") - entry = "- Harden Noema exact-source evidence edges: whitespace-only lines require ``, while over-cap nonblank lines retain structural exact-source binding without impossible bounded-prose repetition.\n" - if entry not in text: - if "## [Unreleased]\n" in text: - text = text.replace("## [Unreleased]\n", "## [Unreleased]\n" + entry, 1) - elif "## Unreleased\n" in text: - text = text.replace("## Unreleased\n", "## Unreleased\n" + entry, 1) - else: - text = entry + "\n" + text - changelog.write_text(text, encoding="utf-8") - PY - - - name: Verify focused and broader GREEN - shell: bash - run: | - set -euo pipefail - python -m compileall -q scripts/ci - python -m pytest -q \ - tests/test_noema_class_evidence_observation_contract.py \ - tests/test_noema_observed_defect_corpus_current_main.py - python -m pytest -q tests/test_noema_*.py - git diff --check - - - name: Remove every temporary PR1641 repair identity - shell: bash - run: | - set -euo pipefail - rm -f \ - .github/workflows/_temp_pr1641_edge_repair_v2.yml \ - .github/workflows/_temp_pr1641_edge_repair_v3.yml \ - .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ - scripts/ci/temp_pr1641_edge_repair.py - if git ls-files | grep -E '(^|/)_?temp_pr1641|temp-pr1641'; then - echo "::error::temporary PR1641 repair identity remains tracked" - exit 1 - fi - git diff --check - - - name: Publish verified successor with workflow-starting credential - shell: bash - run: | - set -euo pipefail - if [ -z "${WORKFLOW_PUSH_TOKEN:-}" ]; then - echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." - exit 1 - fi - remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - test "$remote_head" = "${GITHUB_SHA}" - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git add -A - git diff --cached --check - test -n "$(git diff --cached --name-only)" - git commit -m "fix(noema): harden exact-source evidence edge cases" - git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}" From ec53e9c552ff2ecc47012a92b9b6235347372be9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 13:00:20 +0900 Subject: [PATCH 44/59] chore(noema): remove superseded PR1641 repair helper --- scripts/ci/temp_pr1641_edge_repair.py | 182 -------------------------- 1 file changed, 182 deletions(-) delete mode 100644 scripts/ci/temp_pr1641_edge_repair.py diff --git a/scripts/ci/temp_pr1641_edge_repair.py b/scripts/ci/temp_pr1641_edge_repair.py deleted file mode 100644 index c4cb1297af..0000000000 --- a/scripts/ci/temp_pr1641_edge_repair.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 -"""One-shot exact-source edge repair for PR 1641; deletes itself after GREEN.""" -from __future__ import annotations - -from pathlib import Path -import subprocess -import sys - -ROOT = Path(__file__).resolve().parents[2] -TEST = ROOT / "tests/test_noema_class_evidence_observation_contract.py" -GATE = ROOT / "scripts/ci/noema_review_gate.py" -DOCS = ROOT / "docs/product-technical-gap-baseline.md" -CHANGELOG = ROOT / "CHANGELOG.md" - -WHITESPACE_TEST = r''' - - -def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: - """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" - source = " " - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." - ) - with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) -''' - -LONG_TEST = r''' - - -def test_long_changed_line_uses_structural_exact_source_binding() -> None: - """An over-cap changed line remains reviewable without impossible prose repetition.""" - source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) - diff = f"""diff --git a/src/tool.py b/src/tool.py ---- a/src/tool.py -+++ b/src/tool.py -@@ -1 +1 @@ --old = 1 -+{source} -""" - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." - ) - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) -''' - - -def run(*args: str, expect: int = 0) -> subprocess.CompletedProcess[str]: - result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True) - sys.stdout.write(result.stdout) - sys.stderr.write(result.stderr) - if result.returncode != expect: - raise SystemExit(f"command {args!r} returned {result.returncode}, expected {expect}") - return result - - -def append_regressions() -> None: - text = TEST.read_text(encoding="utf-8") - if "def test_whitespace_only_changed_source_requires_explicit_blank_marker" not in text: - text += WHITESPACE_TEST - if "def test_long_changed_line_uses_structural_exact_source_binding" not in text: - text += LONG_TEST - TEST.write_text(text, encoding="utf-8") - - -def prove_red() -> None: - specs = ( - "tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker", - "tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding", - ) - for spec in specs: - result = subprocess.run( - [sys.executable, "-m", "pytest", "-q", spec], - cwd=ROOT, - text=True, - capture_output=True, - ) - sys.stdout.write(result.stdout) - sys.stderr.write(result.stderr) - if result.returncode == 0 or "1 failed" not in (result.stdout + result.stderr): - raise SystemExit(f"expected focused RED regression did not fail exactly: {spec}") - - -def repair_source() -> None: - text = GATE.read_text(encoding="utf-8") - old = ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) -''' - new = ''' source_is_blank = not source_excerpt.strip() - source_marker = "" if source_is_blank else source_excerpt - # Exact source identity is already established by equality against the - # trusted changed-line map. Repetition inside bounded prose is an - # anti-vacuity signal only when the source can fit; blank lines use - # an explicit structural marker rather than incidental whitespace. - if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) -''' - if text.count(old) != 1: - raise SystemExit(f"unexpected source quote guard count: {text.count(old)}") - text = text.replace(old, new, 1) - old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." - new_prompt = ( - "For an empty or whitespace-only source line the observation must quote ; " - "for a bounded nonblank source_excerpt it must quote the exact text. Longer nonblank " - "lines remain exactly bound by source_excerpt equality while observation stays bounded " - "and explains the claimed behavior." - ) - if old_prompt in text: - text = text.replace(old_prompt, new_prompt, 1) - GATE.write_text(text, encoding="utf-8") - - -def update_traceability() -> None: - dtext = DOCS.read_text(encoding="utf-8") - heading = "## 2026-09-02 — Noema exact-source edge binding" - if heading not in dtext: - dtext = dtext.rstrip() + "\n\n" + heading + "\n\n" - dtext += ( - "External exact-head review exposed two executable source-evidence edge defects: " - "whitespace-only changed lines could be admitted by incidental spacing, while a " - "nonblank changed line longer than the bounded observation field could become " - "structurally impossible to admit. Exact identity remains equality between " - "`class_evidence.source_excerpt` and the trusted changed-side diff map. Whitespace-only " - "source now requires the explicit `` observation marker; over-cap nonblank lines " - "remain exactly source-bound without requiring impossible prose repetition. Focused " - "regressions preserve both contracts.\n" - ) - DOCS.write_text(dtext, encoding="utf-8") - - ctext = CHANGELOG.read_text(encoding="utf-8") - entry = ( - "- Harden Noema exact-source evidence edges: whitespace-only changed lines require " - "``, while over-cap nonblank lines retain exact structural source binding without " - "impossible bounded-prose repetition.\n" - ) - if entry not in ctext: - marker = "## [Unreleased]\n" - ctext = ctext.replace(marker, marker + entry, 1) if marker in ctext else entry + "\n" + ctext - CHANGELOG.write_text(ctext, encoding="utf-8") - - -def prove_green() -> None: - run(sys.executable, "-m", "pytest", "-q", "tests/test_noema_class_evidence_observation_contract.py", "tests/test_noema_observed_defect_corpus_current_main.py") - run(sys.executable, "-m", "pytest", "-q", "tests/test_noema_*.py") - run(sys.executable, "-m", "compileall", "-q", "scripts/ci") - run("git", "diff", "--check") - - -def main() -> None: - append_regressions() - prove_red() - repair_source() - update_traceability() - prove_green() - Path(__file__).unlink() - - -if __name__ == "__main__": - main() From 70bc213836d02f96275efca790459573326060ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 13:02:03 +0900 Subject: [PATCH 45/59] ci(noema): retrigger exact-source edge repair --- .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml b/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml index 43ca0701bf..864b73356c 100644 --- a/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml +++ b/.github/workflows/_temp_pr1641_source_evidence_edge_repair.yml @@ -1,4 +1,5 @@ name: Temporary PR1641 source evidence edge repair +# retrigger after superseded temporary repair cleanup on: push: From 468d0ed6c0669333c763e4a815407dbb7fc1bc8b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 14:11:56 +0900 Subject: [PATCH 46/59] ci(noema): supersede flawed PR1641 source writer --- .../workflows/_temp_pr1641_final_green.yml | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 .github/workflows/_temp_pr1641_final_green.yml diff --git a/.github/workflows/_temp_pr1641_final_green.yml b/.github/workflows/_temp_pr1641_final_green.yml new file mode 100644 index 0000000000..33da6ea221 --- /dev/null +++ b/.github/workflows/_temp_pr1641_final_green.yml @@ -0,0 +1,397 @@ +name: Temporary PR1641 final review-quality GREEN + +on: + push: + branches: + - fix/noema-observed-defect-corpus-current-main-20260902 + paths: + - .github/workflows/_temp_pr1641_final_green.yml + +permissions: + contents: read + +concurrency: + group: temp-pr1641-source-evidence-edge-repair-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Checkout exact writer head without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install hash-locked review dependencies + shell: bash + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ + -r requirements-opencode-review-ci-hashes.txt + + - name: Revalidate exact writer head + shell: bash + run: | + set -euo pipefail + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + local_head="$(git rev-parse HEAD)" + if [ -z "$remote_head" ] || [ "$remote_head" != "$local_head" ]; then + echo "::error::writer head moved: local=$local_head remote=$remote_head" + exit 1 + fi + + - name: Materialize externally demonstrated RED regressions + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + import re + + target = Path("tests/test_noema_class_evidence_observation_contract.py") + text = target.read_text(encoding="utf-8") + + omission_pattern = re.compile( + r"def test_overlong_omission_marker_cannot_be_source_evidence\(\) -> None:\n.*?(?=\n\ndef test_overlong_class_observation_is_rejected_before_semantic_admission)", + re.S, + ) + replacement = r'''def test_literal_omission_marker_is_valid_real_source_evidence() -> None: + """A real source line matching the display marker must not be discarded.""" + marker = "[overlong changed line content omitted]" + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{marker} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = marker + witness["observation"] = f"{marker} is exact source evidence for {probe['probe_kind']}:{field}." + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + + def test_fetch_diff_synthetic_omission_never_becomes_changed_source(monkeypatch) -> None: + """Prompt truncation metadata cannot alias a genuine changed source line.""" + marker = "[overlong changed line content omitted]" + prefix = """diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +""" + raw = prefix + ("x" * (noema.MAX_DIFF_CHARS + 128)) + monkeypatch.setattr(noema, "run", lambda _args: raw) + bounded, truncated = noema.fetch_diff("owner/repo", 1) + assert truncated is True + assert marker in bounded + assert f"+{marker}" not in bounded + texts = noema.changed_diff_line_texts(bounded) + assert marker not in texts.values() + assert set(texts) == noema.changed_diff_locations(bounded) + ''' + text, count = omission_pattern.subn(replacement, text, count=1) + if count != 1: + raise SystemExit(f"expected one omission-marker regression to replace, found {count}") + + marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" + if marker not in text: + text = text.rstrip() + r''' + + + def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{source} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." + ) + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + + def test_long_changed_line_uses_structural_exact_source_binding() -> None: + """An over-cap exact source remains reviewable without impossible prose repetition.""" + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{source} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." + ) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + ''' + "\n" + target.write_text(text, encoding="utf-8") + PY + + - name: Prove the three source-evidence regressions are specifically RED + shell: bash + run: | + set -euo pipefail + specs=( + "tests/test_noema_class_evidence_observation_contract.py::test_literal_omission_marker_is_valid_real_source_evidence" + "tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker" + "tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding" + ) + for spec in "${specs[@]}"; do + log="$(mktemp)" + set +e + python -m pytest -q "$spec" >"$log" 2>&1 + status=$? + set -e + cat "$log" + if [ "$status" -eq 0 ]; then + echo "::error::expected RED regression was already GREEN: $spec" + exit 1 + fi + if ! grep -q "1 failed" "$log"; then + echo "::error::RED was not the intended test failure: $spec" + exit 1 + fi + done + + - name: Apply causal production and traceability repair + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + import re + + source = Path("scripts/ci/noema_review_gate.py") + text = source.read_text(encoding="utf-8") + + old_truncation = ''' if partial.startswith(("+", "-")) and ( + inside_hunk or not partial.startswith(("+++", "---")) + ): + complete += f"\\n{partial[0]}{marker}" + diff = complete + ''' + new_truncation = ''' if partial.startswith(("+", "-")) and ( + inside_hunk or not partial.startswith(("+++", "---")) + ): + # Keep truncation metadata outside the changed-line grammar. A real + # source line may literally equal the display marker, so prefixing + # the synthetic marker with + or - would make identity ambiguous. + complete += f"\\n{marker}" + diff = complete + ''' + if text.count(old_truncation) != 1: + raise SystemExit(f"fetch_diff truncation anchor count={text.count(old_truncation)}") + text = text.replace(old_truncation, new_truncation, 1) + + parser_pattern = re.compile( + r"def changed_diff_locations\(diff: str\) -> set\[tuple\[str, int, str\]\]:\n.*?(?=\n\ndef parse_diff_path)", + re.S, + ) + shared_parser = r'''def _changed_diff_evidence( + diff: str, + ) -> tuple[set[tuple[str, int, str]], dict[tuple[str, int, str], str]]: + """Parse changed coordinates and exact source text in one state machine.""" + locations: set[tuple[str, int, str]] = set() + texts: dict[tuple[str, int, str], str] = {} + old_path = new_path = "" + old_line = new_line = 0 + in_hunk = False + for raw_line in diff.splitlines(): + if raw_line.startswith("diff --git "): + old_path = new_path = "" + in_hunk = False + continue + if not in_hunk and raw_line.startswith("--- "): + old_path = parse_diff_path(raw_line[4:], "a/") + in_hunk = False + continue + if not in_hunk and raw_line.startswith("+++ "): + new_path = parse_diff_path(raw_line[4:], "b/") + in_hunk = False + continue + match = DIFF_HUNK_RE.match(raw_line) + if match: + old_line, new_line = map(int, match.groups()) + in_hunk = True + continue + if not in_hunk or raw_line.startswith(r"\ No newline"): + continue + if raw_line.startswith("+"): + if not new_path: + return set(), {} + location = (new_path, new_line, "RIGHT") + locations.add(location) + texts[location] = raw_line[1:] + new_line += 1 + elif raw_line.startswith("-"): + if not old_path: + return set(), {} + location = (old_path, old_line, "LEFT") + locations.add(location) + texts[location] = raw_line[1:] + old_line += 1 + else: + old_line += 1 + new_line += 1 + return locations, texts + + + def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: + """Return exact LEFT/RIGHT changed-line locations from a unified diff.""" + return _changed_diff_evidence(diff)[0] + + + def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: + """Return exact changed-side source text keyed by canonical diff location.""" + return _changed_diff_evidence(diff)[1] + ''' + text, count = parser_pattern.subn(shared_parser, text, count=1) + if count != 1: + raise SystemExit(f"changed-diff parser replacement count={count}") + + old_quote = ''' source_marker = source_excerpt if source_excerpt else "" + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''' + new_quote = ''' source_is_blank = not source_excerpt.strip() + source_marker = "" if source_is_blank else source_excerpt + # Exact identity is already established by equality against the trusted + # changed-line map. Repetition in bounded prose is an anti-vacuity signal + # only when the exact source can fit in the bounded observation. + if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: + if source_marker not in observation: + raise NoemaModelOutputError( + f"Noema adversarial probe {index} class_evidence.{field} observation " + "must quote the exact source_excerpt (or for an empty line)" + ) + ''' + if text.count(old_quote) != 1: + raise SystemExit(f"source quote anchor count={text.count(old_quote)}") + text = text.replace(old_quote, new_quote, 1) + + old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." + new_prompt = ( + "For an empty or whitespace-only line the observation must quote ; for a nonblank " + "source_excerpt no longer than MAX_THREAD_BODY_CHARS it must quote the exact source_excerpt. " + "Longer nonblank lines remain exactly bound by the separately validated source_excerpt field, " + "while observation stays bounded and explains the claimed behavior." + ) + if text.count(old_prompt) != 1: + raise SystemExit(f"prompt source-evidence anchor count={text.count(old_prompt)}") + text = text.replace(old_prompt, new_prompt, 1) + source.write_text(text, encoding="utf-8") + + baseline = Path("docs/product-technical-gap-baseline.md") + btext = baseline.read_text(encoding="utf-8") + marker = "## 2026-09-02 — Noema source-evidence parser convergence" + if marker not in btext: + btext = btext.rstrip() + "\n\n" + '''## 2026-09-02 — Noema source-evidence parser convergence + + External review demonstrated that the bounded-diff display marker and a genuine source line with the same text were conflated, while coordinate and source extraction duplicated one unified-diff state machine. The review gate now parses changed coordinates and source bytes together, keeps synthetic truncation metadata outside the +/- changed-line grammar, accepts a genuine marker-shaped source line, treats whitespace-only source as explicit ``, and uses structural exact-source equality for over-cap lines whose text cannot fit in bounded observation prose. The regressions preserve observable defect cases rather than vendor wording or benchmark claims. Candidate truth remains exact-head only until ordinary protected checks succeed. + '''.strip() + "\n" + baseline.write_text(btext, encoding="utf-8") + + changelog = Path("CHANGELOG.md") + ctext = changelog.read_text(encoding="utf-8") + entry = ( + "- Converge Noema changed-line coordinate/source parsing into one state machine; synthetic diff-truncation " + "metadata no longer aliases a genuine marker-shaped source line, whitespace-only source requires ``, " + "and over-cap exact source remains structurally reviewable without impossible prose repetition.\n" + ) + if entry not in ctext: + if "## [Unreleased]\n" in ctext: + ctext = ctext.replace("## [Unreleased]\n", "## [Unreleased]\n" + entry, 1) + elif "## Unreleased\n" in ctext: + ctext = ctext.replace("## Unreleased\n", "## Unreleased\n" + entry, 1) + else: + ctext = entry + "\n" + ctext + changelog.write_text(ctext, encoding="utf-8") + PY + + - name: Verify focused and broader GREEN contracts + shell: bash + run: | + set -euo pipefail + python -m compileall -q scripts/ci + python -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py \ + tests/test_noema_observed_defect_corpus_current_main.py + python -m pytest -q tests/test_noema_*.py + git diff --check + + - name: Remove all PR1641 temporary writer identities + shell: bash + run: | + set -euo pipefail + git rm -f \ + .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ + .github/workflows/_temp_pr1641_final_green.yml + if git ls-files | grep -E '(^|/)(_temp_pr1641|temp_pr1641)'; then + echo "::error::temporary PR1641 identity remains tracked in the publishable tree" + exit 1 + fi + git diff --check + + - name: Recheck remote writer head before publication + shell: bash + run: | + set -euo pipefail + git fetch origin "${GITHUB_REF_NAME}" + remote_head="$(git rev-parse "origin/${GITHUB_REF_NAME}")" + local_parent="$(git rev-parse HEAD)" + if [ "$remote_head" != "$local_parent" ]; then + echo "::error::writer branch moved before publish: local=$local_parent remote=$remote_head" + exit 1 + fi + + - name: Publish verified successor with workflow-starting credential only + shell: bash + env: + WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} + run: | + set -euo pipefail + if [ -z "${WORKFLOW_PUSH_TOKEN:-}" ]; then + echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." + exit 1 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + if git diff --cached --quiet; then + echo "::error::repair produced no publishable delta" + exit 1 + fi + git commit -m "fix(noema): converge exact-source evidence parsing" + git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push origin "HEAD:${GITHUB_REF_NAME}" From e4eeaa5c2e799eee3ec0d0ba7d00af5d81850284 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 14:17:18 +0900 Subject: [PATCH 47/59] ci(noema): replace indentation-fragile PR1641 writer --- .../workflows/_temp_pr1641_final_green.yml | 397 ------------------ 1 file changed, 397 deletions(-) delete mode 100644 .github/workflows/_temp_pr1641_final_green.yml diff --git a/.github/workflows/_temp_pr1641_final_green.yml b/.github/workflows/_temp_pr1641_final_green.yml deleted file mode 100644 index 33da6ea221..0000000000 --- a/.github/workflows/_temp_pr1641_final_green.yml +++ /dev/null @@ -1,397 +0,0 @@ -name: Temporary PR1641 final review-quality GREEN - -on: - push: - branches: - - fix/noema-observed-defect-corpus-current-main-20260902 - paths: - - .github/workflows/_temp_pr1641_final_green.yml - -permissions: - contents: read - -concurrency: - group: temp-pr1641-source-evidence-edge-repair-${{ github.repository }}-${{ github.ref_name }} - cancel-in-progress: true - -jobs: - repair: - if: github.repository == 'ContextualWisdomLab/.github' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - name: Checkout exact writer head without persisted credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.14" - - - name: Install hash-locked review dependencies - shell: bash - run: | - set -euo pipefail - python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ - -r requirements-opencode-review-ci-hashes.txt - - - name: Revalidate exact writer head - shell: bash - run: | - set -euo pipefail - remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" - local_head="$(git rev-parse HEAD)" - if [ -z "$remote_head" ] || [ "$remote_head" != "$local_head" ]; then - echo "::error::writer head moved: local=$local_head remote=$remote_head" - exit 1 - fi - - - name: Materialize externally demonstrated RED regressions - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - import re - - target = Path("tests/test_noema_class_evidence_observation_contract.py") - text = target.read_text(encoding="utf-8") - - omission_pattern = re.compile( - r"def test_overlong_omission_marker_cannot_be_source_evidence\(\) -> None:\n.*?(?=\n\ndef test_overlong_class_observation_is_rejected_before_semantic_admission)", - re.S, - ) - replacement = r'''def test_literal_omission_marker_is_valid_real_source_evidence() -> None: - """A real source line matching the display marker must not be discarded.""" - marker = "[overlong changed line content omitted]" - diff = f"""diff --git a/src/tool.py b/src/tool.py - --- a/src/tool.py - +++ b/src/tool.py - @@ -1 +1 @@ - -old = 1 - +{marker} - """ - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = marker - witness["observation"] = f"{marker} is exact source evidence for {probe['probe_kind']}:{field}." - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) - - - def test_fetch_diff_synthetic_omission_never_becomes_changed_source(monkeypatch) -> None: - """Prompt truncation metadata cannot alias a genuine changed source line.""" - marker = "[overlong changed line content omitted]" - prefix = """diff --git a/src/tool.py b/src/tool.py - --- a/src/tool.py - +++ b/src/tool.py - @@ -1 +1 @@ - -old = 1 - +""" - raw = prefix + ("x" * (noema.MAX_DIFF_CHARS + 128)) - monkeypatch.setattr(noema, "run", lambda _args: raw) - bounded, truncated = noema.fetch_diff("owner/repo", 1) - assert truncated is True - assert marker in bounded - assert f"+{marker}" not in bounded - texts = noema.changed_diff_line_texts(bounded) - assert marker not in texts.values() - assert set(texts) == noema.changed_diff_locations(bounded) - ''' - text, count = omission_pattern.subn(replacement, text, count=1) - if count != 1: - raise SystemExit(f"expected one omission-marker regression to replace, found {count}") - - marker = "def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None:" - if marker not in text: - text = text.rstrip() + r''' - - - def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: - """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" - source = " " - diff = f"""diff --git a/src/tool.py b/src/tool.py - --- a/src/tool.py - +++ b/src/tool.py - @@ -1 +1 @@ - -old = 1 - +{source} - """ - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." - ) - with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) - - - def test_long_changed_line_uses_structural_exact_source_binding() -> None: - """An over-cap exact source remains reviewable without impossible prose repetition.""" - source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) - diff = f"""diff --git a/src/tool.py b/src/tool.py - --- a/src/tool.py - +++ b/src/tool.py - @@ -1 +1 @@ - -old = 1 - +{source} - """ - verdict = _verdict(observations=True, source_excerpt=True) - for probe in verdict["adversarial_validation"]["probes"]: - for field, witness in probe["class_evidence"].items(): - witness["source_excerpt"] = source - witness["observation"] = ( - f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." - ) - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) - ''' + "\n" - target.write_text(text, encoding="utf-8") - PY - - - name: Prove the three source-evidence regressions are specifically RED - shell: bash - run: | - set -euo pipefail - specs=( - "tests/test_noema_class_evidence_observation_contract.py::test_literal_omission_marker_is_valid_real_source_evidence" - "tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker" - "tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding" - ) - for spec in "${specs[@]}"; do - log="$(mktemp)" - set +e - python -m pytest -q "$spec" >"$log" 2>&1 - status=$? - set -e - cat "$log" - if [ "$status" -eq 0 ]; then - echo "::error::expected RED regression was already GREEN: $spec" - exit 1 - fi - if ! grep -q "1 failed" "$log"; then - echo "::error::RED was not the intended test failure: $spec" - exit 1 - fi - done - - - name: Apply causal production and traceability repair - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - import re - - source = Path("scripts/ci/noema_review_gate.py") - text = source.read_text(encoding="utf-8") - - old_truncation = ''' if partial.startswith(("+", "-")) and ( - inside_hunk or not partial.startswith(("+++", "---")) - ): - complete += f"\\n{partial[0]}{marker}" - diff = complete - ''' - new_truncation = ''' if partial.startswith(("+", "-")) and ( - inside_hunk or not partial.startswith(("+++", "---")) - ): - # Keep truncation metadata outside the changed-line grammar. A real - # source line may literally equal the display marker, so prefixing - # the synthetic marker with + or - would make identity ambiguous. - complete += f"\\n{marker}" - diff = complete - ''' - if text.count(old_truncation) != 1: - raise SystemExit(f"fetch_diff truncation anchor count={text.count(old_truncation)}") - text = text.replace(old_truncation, new_truncation, 1) - - parser_pattern = re.compile( - r"def changed_diff_locations\(diff: str\) -> set\[tuple\[str, int, str\]\]:\n.*?(?=\n\ndef parse_diff_path)", - re.S, - ) - shared_parser = r'''def _changed_diff_evidence( - diff: str, - ) -> tuple[set[tuple[str, int, str]], dict[tuple[str, int, str], str]]: - """Parse changed coordinates and exact source text in one state machine.""" - locations: set[tuple[str, int, str]] = set() - texts: dict[tuple[str, int, str], str] = {} - old_path = new_path = "" - old_line = new_line = 0 - in_hunk = False - for raw_line in diff.splitlines(): - if raw_line.startswith("diff --git "): - old_path = new_path = "" - in_hunk = False - continue - if not in_hunk and raw_line.startswith("--- "): - old_path = parse_diff_path(raw_line[4:], "a/") - in_hunk = False - continue - if not in_hunk and raw_line.startswith("+++ "): - new_path = parse_diff_path(raw_line[4:], "b/") - in_hunk = False - continue - match = DIFF_HUNK_RE.match(raw_line) - if match: - old_line, new_line = map(int, match.groups()) - in_hunk = True - continue - if not in_hunk or raw_line.startswith(r"\ No newline"): - continue - if raw_line.startswith("+"): - if not new_path: - return set(), {} - location = (new_path, new_line, "RIGHT") - locations.add(location) - texts[location] = raw_line[1:] - new_line += 1 - elif raw_line.startswith("-"): - if not old_path: - return set(), {} - location = (old_path, old_line, "LEFT") - locations.add(location) - texts[location] = raw_line[1:] - old_line += 1 - else: - old_line += 1 - new_line += 1 - return locations, texts - - - def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: - """Return exact LEFT/RIGHT changed-line locations from a unified diff.""" - return _changed_diff_evidence(diff)[0] - - - def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: - """Return exact changed-side source text keyed by canonical diff location.""" - return _changed_diff_evidence(diff)[1] - ''' - text, count = parser_pattern.subn(shared_parser, text, count=1) - if count != 1: - raise SystemExit(f"changed-diff parser replacement count={count}") - - old_quote = ''' source_marker = source_excerpt if source_excerpt else "" - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - ''' - new_quote = ''' source_is_blank = not source_excerpt.strip() - source_marker = "" if source_is_blank else source_excerpt - # Exact identity is already established by equality against the trusted - # changed-line map. Repetition in bounded prose is an anti-vacuity signal - # only when the exact source can fit in the bounded observation. - if source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS: - if source_marker not in observation: - raise NoemaModelOutputError( - f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" - ) - ''' - if text.count(old_quote) != 1: - raise SystemExit(f"source quote anchor count={text.count(old_quote)}") - text = text.replace(old_quote, new_quote, 1) - - old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." - new_prompt = ( - "For an empty or whitespace-only line the observation must quote ; for a nonblank " - "source_excerpt no longer than MAX_THREAD_BODY_CHARS it must quote the exact source_excerpt. " - "Longer nonblank lines remain exactly bound by the separately validated source_excerpt field, " - "while observation stays bounded and explains the claimed behavior." - ) - if text.count(old_prompt) != 1: - raise SystemExit(f"prompt source-evidence anchor count={text.count(old_prompt)}") - text = text.replace(old_prompt, new_prompt, 1) - source.write_text(text, encoding="utf-8") - - baseline = Path("docs/product-technical-gap-baseline.md") - btext = baseline.read_text(encoding="utf-8") - marker = "## 2026-09-02 — Noema source-evidence parser convergence" - if marker not in btext: - btext = btext.rstrip() + "\n\n" + '''## 2026-09-02 — Noema source-evidence parser convergence - - External review demonstrated that the bounded-diff display marker and a genuine source line with the same text were conflated, while coordinate and source extraction duplicated one unified-diff state machine. The review gate now parses changed coordinates and source bytes together, keeps synthetic truncation metadata outside the +/- changed-line grammar, accepts a genuine marker-shaped source line, treats whitespace-only source as explicit ``, and uses structural exact-source equality for over-cap lines whose text cannot fit in bounded observation prose. The regressions preserve observable defect cases rather than vendor wording or benchmark claims. Candidate truth remains exact-head only until ordinary protected checks succeed. - '''.strip() + "\n" - baseline.write_text(btext, encoding="utf-8") - - changelog = Path("CHANGELOG.md") - ctext = changelog.read_text(encoding="utf-8") - entry = ( - "- Converge Noema changed-line coordinate/source parsing into one state machine; synthetic diff-truncation " - "metadata no longer aliases a genuine marker-shaped source line, whitespace-only source requires ``, " - "and over-cap exact source remains structurally reviewable without impossible prose repetition.\n" - ) - if entry not in ctext: - if "## [Unreleased]\n" in ctext: - ctext = ctext.replace("## [Unreleased]\n", "## [Unreleased]\n" + entry, 1) - elif "## Unreleased\n" in ctext: - ctext = ctext.replace("## Unreleased\n", "## Unreleased\n" + entry, 1) - else: - ctext = entry + "\n" + ctext - changelog.write_text(ctext, encoding="utf-8") - PY - - - name: Verify focused and broader GREEN contracts - shell: bash - run: | - set -euo pipefail - python -m compileall -q scripts/ci - python -m pytest -q \ - tests/test_noema_class_evidence_observation_contract.py \ - tests/test_noema_observed_defect_corpus_current_main.py - python -m pytest -q tests/test_noema_*.py - git diff --check - - - name: Remove all PR1641 temporary writer identities - shell: bash - run: | - set -euo pipefail - git rm -f \ - .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ - .github/workflows/_temp_pr1641_final_green.yml - if git ls-files | grep -E '(^|/)(_temp_pr1641|temp_pr1641)'; then - echo "::error::temporary PR1641 identity remains tracked in the publishable tree" - exit 1 - fi - git diff --check - - - name: Recheck remote writer head before publication - shell: bash - run: | - set -euo pipefail - git fetch origin "${GITHUB_REF_NAME}" - remote_head="$(git rev-parse "origin/${GITHUB_REF_NAME}")" - local_parent="$(git rev-parse HEAD)" - if [ "$remote_head" != "$local_parent" ]; then - echo "::error::writer branch moved before publish: local=$local_parent remote=$remote_head" - exit 1 - fi - - - name: Publish verified successor with workflow-starting credential only - shell: bash - env: - WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} - run: | - set -euo pipefail - if [ -z "${WORKFLOW_PUSH_TOKEN:-}" ]; then - echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." - exit 1 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - if git diff --cached --quiet; then - echo "::error::repair produced no publishable delta" - exit 1 - fi - git commit -m "fix(noema): converge exact-source evidence parsing" - git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push origin "HEAD:${GITHUB_REF_NAME}" From 2224c45d4a2f303b030509585d8887d845477aed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 14:18:23 +0900 Subject: [PATCH 48/59] ci(noema): run robust PR1641 exact-source GREEN repair --- .../workflows/_temp_pr1641_final_green.yml | 427 ++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 .github/workflows/_temp_pr1641_final_green.yml diff --git a/.github/workflows/_temp_pr1641_final_green.yml b/.github/workflows/_temp_pr1641_final_green.yml new file mode 100644 index 0000000000..17958d5745 --- /dev/null +++ b/.github/workflows/_temp_pr1641_final_green.yml @@ -0,0 +1,427 @@ +name: Temporary PR1641 final review-quality GREEN + +on: + push: + branches: + - fix/noema-observed-defect-corpus-current-main-20260902 + paths: + - .github/workflows/_temp_pr1641_final_green.yml + +permissions: + contents: read + +concurrency: + group: temp-pr1641-source-evidence-edge-repair-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Checkout exact writer head without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install hash-locked review dependencies + shell: bash + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ + -r requirements-opencode-review-ci-hashes.txt + + - name: Revalidate exact writer head + shell: bash + run: | + set -euo pipefail + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + local_head="$(git rev-parse HEAD)" + if [ -z "$remote_head" ] || [ "$remote_head" != "$local_head" ]; then + echo "::error::writer head moved: local=$local_head remote=$remote_head" + exit 1 + fi + + - name: Materialize externally demonstrated RED regressions + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + import re + import textwrap + + target = Path("tests/test_noema_class_evidence_observation_contract.py") + text = target.read_text(encoding="utf-8") + + old_marker_test = re.compile( + r"def test_overlong_omission_marker_cannot_be_source_evidence\(\) -> None:\n.*?(?=\n\ndef test_overlong_class_observation_is_rejected_before_semantic_admission)", + re.S, + ) + new_marker_tests = textwrap.dedent(r''' + def test_literal_omission_marker_is_valid_real_source_evidence() -> None: + """A real source line matching the display marker must not be discarded.""" + marker = "[overlong changed line content omitted]" + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{marker} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = marker + witness["observation"] = f"{marker} is exact source evidence for {probe['probe_kind']}:{field}." + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + + def test_fetch_diff_synthetic_omission_never_becomes_changed_source(monkeypatch) -> None: + """Prompt truncation metadata cannot alias a genuine changed source line.""" + marker = "[overlong changed line content omitted]" + prefix = """diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +""" + raw = prefix + ("x" * (noema.MAX_DIFF_CHARS + 128)) + monkeypatch.setattr(noema, "run", lambda _args: raw) + bounded, truncated = noema.fetch_diff("owner/repo", 1) + assert truncated is True + assert marker in bounded + assert f"+{marker}" not in bounded + texts = noema.changed_diff_line_texts(bounded) + assert marker not in texts.values() + assert set(texts) == noema.changed_diff_locations(bounded) + ''').strip() + text, count = old_marker_test.subn(new_marker_tests, text, count=1) + if count != 1: + raise SystemExit(f"expected one omission-marker test replacement, found {count}") + + parser_test = re.compile( + r"def test_changed_diff_line_texts_covers_context_markers_and_no_newline_marker\(\) -> None:\n.*?(?=\n\ndef test_changed_diff_line_texts_fails_closed_when_hunk_paths_are_missing)", + re.S, + ) + new_parser_test = textwrap.dedent(r''' + def test_changed_diff_line_texts_preserves_literal_marker_and_no_newline_marker() -> None: + """A literal marker is real source; only Git's no-newline metadata is skipped.""" + marker = "[overlong changed line content omitted]" + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1,3 +1,3 @@ + context + -{marker} + +{marker} + -old + +new + \\ No newline at end of file + """ + texts = noema.changed_diff_line_texts(diff) + assert texts == {{ + ("src/tool.py", 2, "LEFT"): marker, + ("src/tool.py", 2, "RIGHT"): marker, + ("src/tool.py", 3, "LEFT"): "old", + ("src/tool.py", 3, "RIGHT"): "new", + }} + assert set(texts) == noema.changed_diff_locations(diff) + ''').strip() + text, count = parser_test.subn(new_parser_test, text, count=1) + if count != 1: + raise SystemExit(f"expected one changed-diff parser test replacement, found {count}") + + if "def test_whitespace_only_changed_source_requires_explicit_blank_marker()" not in text: + text += "\n\n" + textwrap.dedent(r''' + def test_whitespace_only_changed_source_requires_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy the quote guard via incidental spacing.""" + source = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{source} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Incidental spacing is not a source quote for {probe['probe_kind']}:{field}." + ) + with pytest.raises(noema.NoemaModelOutputError, match="quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + + def test_long_changed_line_uses_structural_exact_source_binding() -> None: + """An over-cap exact source remains reviewable without impossible prose repetition.""" + source = "x" * (noema.MAX_THREAD_BODY_CHARS + 64) + diff = f"""diff --git a/src/tool.py b/src/tool.py + --- a/src/tool.py + +++ b/src/tool.py + @@ -1 +1 @@ + -old = 1 + +{source} + """ + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = source + witness["observation"] = ( + f"Bounded structural observation for {probe['probe_kind']}:{field} at the exact cited line." + ) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + ''').strip() + "\n" + + target.write_text(text, encoding="utf-8") + PY + + - name: Prove source-evidence regressions are specifically RED + shell: bash + run: | + set -euo pipefail + specs=( + "tests/test_noema_class_evidence_observation_contract.py::test_literal_omission_marker_is_valid_real_source_evidence" + "tests/test_noema_class_evidence_observation_contract.py::test_fetch_diff_synthetic_omission_never_becomes_changed_source" + "tests/test_noema_class_evidence_observation_contract.py::test_whitespace_only_changed_source_requires_explicit_blank_marker" + "tests/test_noema_class_evidence_observation_contract.py::test_long_changed_line_uses_structural_exact_source_binding" + ) + for spec in "${specs[@]}"; do + log="$(mktemp)" + set +e + python -m pytest -q "$spec" >"$log" 2>&1 + status=$? + set -e + cat "$log" + if [ "$status" -eq 0 ]; then + echo "::error::expected RED regression was already GREEN: $spec" + exit 1 + fi + if ! grep -q "1 failed" "$log"; then + echo "::error::RED was not the intended test failure: $spec" + exit 1 + fi + done + + - name: Apply causal production and traceability repair + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + import re + import textwrap + + source = Path("scripts/ci/noema_review_gate.py") + text = source.read_text(encoding="utf-8") + + old = ' complete += f"\\n{partial[0]}{marker}"' + new = textwrap.dedent(''' + # Synthetic truncation metadata stays outside the +/- changed-line + # grammar so a genuine source line with identical text remains distinct. + complete += f"\\n{marker}" + ''').strip("\n") + if text.count(old) != 1: + raise SystemExit(f"fetch_diff marker anchor count={text.count(old)}") + text = text.replace(old, new, 1) + + parser_pattern = re.compile( + r"def changed_diff_locations\(diff: str\) -> set\[tuple\[str, int, str\]\]:\n.*?(?=\n\ndef parse_diff_path)", + re.S, + ) + shared_parser = textwrap.dedent(r''' + def _changed_diff_evidence( + diff: str, + ) -> tuple[set[tuple[str, int, str]], dict[tuple[str, int, str], str]]: + """Parse changed coordinates and exact source text in one state machine.""" + locations: set[tuple[str, int, str]] = set() + texts: dict[tuple[str, int, str], str] = {} + old_path = new_path = "" + old_line = new_line = 0 + in_hunk = False + for raw_line in diff.splitlines(): + if raw_line.startswith("diff --git "): + old_path = new_path = "" + in_hunk = False + continue + if not in_hunk and raw_line.startswith("--- "): + old_path = parse_diff_path(raw_line[4:], "a/") + in_hunk = False + continue + if not in_hunk and raw_line.startswith("+++ "): + new_path = parse_diff_path(raw_line[4:], "b/") + in_hunk = False + continue + match = DIFF_HUNK_RE.match(raw_line) + if match: + old_line, new_line = map(int, match.groups()) + in_hunk = True + continue + if not in_hunk or raw_line.startswith(r"\ No newline"): + continue + if raw_line.startswith("+"): + if not new_path: + return set(), {} + location = (new_path, new_line, "RIGHT") + locations.add(location) + texts[location] = raw_line[1:] + new_line += 1 + elif raw_line.startswith("-"): + if not old_path: + return set(), {} + location = (old_path, old_line, "LEFT") + locations.add(location) + texts[location] = raw_line[1:] + old_line += 1 + else: + old_line += 1 + new_line += 1 + return locations, texts + + + def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: + """Return exact LEFT/RIGHT changed-line locations from a unified diff.""" + return _changed_diff_evidence(diff)[0] + + + def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: + """Return exact changed-side source text keyed by canonical diff location.""" + return _changed_diff_evidence(diff)[1] + ''').strip() + text, count = parser_pattern.subn(shared_parser, text, count=1) + if count != 1: + raise SystemExit(f"changed-diff parser replacement count={count}") + + old = ' source_marker = source_excerpt if source_excerpt else ""' + new = ( + ' source_is_blank = not source_excerpt.strip()\n' + ' source_marker = "" if source_is_blank else source_excerpt' + ) + if text.count(old) != 1: + raise SystemExit(f"source marker anchor count={text.count(old)}") + text = text.replace(old, new, 1) + + old = ' if source_marker not in observation:' + new = ( + ' if (source_is_blank or len(source_excerpt) <= MAX_THREAD_BODY_CHARS) and '\ + 'source_marker not in observation:' + ) + if text.count(old) != 1: + raise SystemExit(f"source quote predicate anchor count={text.count(old)}") + text = text.replace(old, new, 1) + + old_prompt = "The observation must quote that exact source_excerpt (or ) and explain the claimed behavior." + new_prompt = ( + "For an empty or whitespace-only line the observation must quote ; for a nonblank " + "source_excerpt no longer than MAX_THREAD_BODY_CHARS it must quote the exact source_excerpt. " + "Longer nonblank lines remain exactly bound by the separately validated source_excerpt field, " + "while observation stays bounded and explains the claimed behavior." + ) + if text.count(old_prompt) != 1: + raise SystemExit(f"prompt source-evidence anchor count={text.count(old_prompt)}") + text = text.replace(old_prompt, new_prompt, 1) + source.write_text(text, encoding="utf-8") + + additions = { + "docs/product-technical-gap-baseline.md": textwrap.dedent(''' + ## 2026-09-02 — Noema source-evidence parser convergence + + External review demonstrated that bounded-diff display metadata and a genuine source line with identical text were conflated, while coordinate and source extraction duplicated one unified-diff state machine. The review gate now parses changed coordinates and exact source together, keeps synthetic truncation metadata outside the +/- changed-line grammar, accepts genuine marker-shaped source, treats whitespace-only source as explicit ``, and uses structural exact-source equality for over-cap lines that cannot fit in bounded observation prose. These are executable false-positive/false-negative fixtures, not vendor wording or benchmark claims. Candidate truth remains exact-head only until protected checks succeed. + '''), + "docs/doctoring/noema-observed-defect-corpus-current-main.md": textwrap.dedent(''' + ## 2026-09-02 exact-source edge corpus + + The corpus now distinguishes synthetic truncation metadata from literal source with the same visible text, exercises whitespace-only and over-cap source excerpts, and asserts coordinate/source parsers cannot diverge because both views come from one state machine. This preserves exact changed-line evidence without lexical causal heuristics. + '''), + } + for path, addition in additions.items(): + target = Path(path) + body = target.read_text(encoding="utf-8") + heading = addition.strip().splitlines()[0] + if heading not in body: + target.write_text(body.rstrip() + "\n\n" + addition.strip() + "\n", encoding="utf-8") + + changelog = Path("CHANGELOG.md") + body = changelog.read_text(encoding="utf-8") + entry = ( + "- Converge Noema changed-line coordinate/source parsing into one state machine; synthetic diff-truncation " + "metadata no longer aliases a genuine marker-shaped source line, whitespace-only source requires ``, " + "and over-cap exact source stays structurally reviewable without impossible prose repetition.\n" + ) + if entry not in body: + if "## [Unreleased]\n" in body: + body = body.replace("## [Unreleased]\n", "## [Unreleased]\n" + entry, 1) + elif "## Unreleased\n" in body: + body = body.replace("## Unreleased\n", "## Unreleased\n" + entry, 1) + else: + body = entry + "\n" + body + changelog.write_text(body, encoding="utf-8") + PY + + - name: Verify focused and broader GREEN contracts + shell: bash + run: | + set -euo pipefail + python -m compileall -q scripts/ci + python -m pytest -q \ + tests/test_noema_class_evidence_observation_contract.py \ + tests/test_noema_observed_defect_corpus_current_main.py + python -m pytest -q tests/test_noema_*.py + git diff --check + + - name: Remove all PR1641 temporary writer identities + shell: bash + run: | + set -euo pipefail + git rm -f \ + .github/workflows/_temp_pr1641_source_evidence_edge_repair.yml \ + .github/workflows/_temp_pr1641_final_green.yml + if git ls-files | grep -E '(^|/)(_temp_pr1641|temp_pr1641)'; then + echo "::error::temporary PR1641 identity remains tracked in the publishable tree" + exit 1 + fi + git diff --check + + - name: Recheck remote writer head before publication + shell: bash + run: | + set -euo pipefail + git fetch origin "${GITHUB_REF_NAME}" + remote_head="$(git rev-parse "origin/${GITHUB_REF_NAME}")" + local_parent="$(git rev-parse HEAD)" + if [ "$remote_head" != "$local_parent" ]; then + echo "::error::writer branch moved before publish: local=$local_parent remote=$remote_head" + exit 1 + fi + + - name: Publish verified successor with workflow-starting credential only + shell: bash + env: + WORKFLOW_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} + run: | + set -euo pipefail + if [ -z "${WORKFLOW_PUSH_TOKEN:-}" ]; then + echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." + exit 1 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + if git diff --cached --quiet; then + echo "::error::repair produced no publishable delta" + exit 1 + fi + git commit -m "fix(noema): converge exact-source evidence parsing" + git remote set-url origin "https://x-access-token:${WORKFLOW_PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push origin "HEAD:${GITHUB_REF_NAME}" From d73afbd8591e70066043683d4b5486e33e98a4d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 14:23:10 +0900 Subject: [PATCH 49/59] ci(noema): keep broad source-evidence regression suite causal --- .github/workflows/_temp_pr1641_final_green.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/_temp_pr1641_final_green.yml b/.github/workflows/_temp_pr1641_final_green.yml index 17958d5745..52021e4e7f 100644 --- a/.github/workflows/_temp_pr1641_final_green.yml +++ b/.github/workflows/_temp_pr1641_final_green.yml @@ -332,6 +332,21 @@ jobs: text = text.replace(old_prompt, new_prompt, 1) source.write_text(text, encoding="utf-8") + broad = Path("tests/test_noema_review_gate.py") + broad_text = broad.read_text(encoding="utf-8") + old_assertions = ( + ' assert diff.endswith("+[overlong changed line content omitted]")\n' + ' assert ("a.py", 1, "RIGHT") in noema.changed_diff_locations(diff)' + ) + new_assertions = ( + ' assert diff.endswith("[overlong changed line content omitted]")\n' + ' assert ("a.py", 1, "RIGHT") not in noema.changed_diff_locations(diff)' + ) + if broad_text.count(old_assertions) != 2: + raise SystemExit(f"legacy truncation assertion count={broad_text.count(old_assertions)}") + broad_text = broad_text.replace(old_assertions, new_assertions) + broad.write_text(broad_text, encoding="utf-8") + additions = { "docs/product-technical-gap-baseline.md": textwrap.dedent(''' ## 2026-09-02 — Noema source-evidence parser convergence From ba2ee0afd51fe92a2d36b3ee8fc0bc7e3c82c3da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 14:34:04 +0900 Subject: [PATCH 50/59] fix(ci): make PR1641 GREEN writer executable --- .github/workflows/_temp_pr1641_final_green.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_temp_pr1641_final_green.yml b/.github/workflows/_temp_pr1641_final_green.yml index 52021e4e7f..a5b435444a 100644 --- a/.github/workflows/_temp_pr1641_final_green.yml +++ b/.github/workflows/_temp_pr1641_final_green.yml @@ -128,12 +128,12 @@ jobs: \\ No newline at end of file """ texts = noema.changed_diff_line_texts(diff) - assert texts == {{ + assert texts == { ("src/tool.py", 2, "LEFT"): marker, ("src/tool.py", 2, "RIGHT"): marker, ("src/tool.py", 3, "LEFT"): "old", ("src/tool.py", 3, "RIGHT"): "new", - }} + } assert set(texts) == noema.changed_diff_locations(diff) ''').strip() text, count = parser_test.subn(new_parser_test, text, count=1) @@ -226,11 +226,11 @@ jobs: text = source.read_text(encoding="utf-8") old = ' complete += f"\\n{partial[0]}{marker}"' - new = textwrap.dedent(''' - # Synthetic truncation metadata stays outside the +/- changed-line - # grammar so a genuine source line with identical text remains distinct. - complete += f"\\n{marker}" - ''').strip("\n") + new = ( + ' # Synthetic truncation metadata stays outside the +/- changed-line\n' + ' # grammar so a genuine source line with identical text remains distinct.\n' + ' complete += f"\\n{marker}"' + ) if text.count(old) != 1: raise SystemExit(f"fetch_diff marker anchor count={text.count(old)}") text = text.replace(old, new, 1) From df0f735f42adbb44d45f3c3a4e503e400b47ed79 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 14:41:00 +0900 Subject: [PATCH 51/59] fix(noema): expose structured failure kind Signed-off-by: Seongho Bae --- scripts/ci/noema_review_gate.py | 4 ++++ tests/test_noema_review_gate.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 5ab7e830f3..70e6f6fccc 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1346,10 +1346,13 @@ def _extract_http_error_telemetry(exc: urllib.error.HTTPError) -> dict[str, str return {} telemetry: dict[str, str | int] = {} model = _safe_model_identifier(detail.get("model")) + failure_kind = _safe_model_identifier(detail.get("failure_kind")) terminal_reason = _safe_model_identifier(detail.get("terminal_reason")) attempts = detail.get("attempts") if model is not None: telemetry["served_model"] = model + if failure_kind is not None: + telemetry["failure_kind"] = failure_kind if terminal_reason is not None: telemetry["terminal_reason"] = terminal_reason if isinstance(attempts, list) and attempts and len(attempts) <= 64: @@ -1383,6 +1386,7 @@ def _format_gateway_error_telemetry(telemetry: dict[str, str | int]) -> str: "upstream_phase", "attempt_number", "upstream_status", + "failure_kind", "terminal_reason", ) return " ".join( diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 6422ae5012..b401abb464 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1597,6 +1597,7 @@ def test_call_llm_reports_only_safe_model_from_bounded_http_error(monkeypatch, c "error": { "detail": { "model": "github_models/deepseek-v3", + "failure_kind": "structured_output_exhausted", "terminal_reason": "eligible_candidates_exhausted", "attempts": [{ "provider_name": "nvidia_nim", @@ -1634,6 +1635,7 @@ def open(self, request): assert "upstream_phase=connecting" in output assert "attempt_number=2" in output assert "upstream_status=503" in output + assert "failure_kind=structured_output_exhausted" in output assert "terminal_reason=eligible_candidates_exhausted" in output assert secret not in output assert secret not in diagnostic From 2a412b84c7c7e3693135f838ea4785912fc6d6a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 19:48:14 +0900 Subject: [PATCH 52/59] test(noema): verify failure-kind log boundaries Preserve PR #1898 runtime behavior and extend the existing HTTP error test across malformed values and length/control-character boundaries. Both the public annotation and raised diagnostic must obey the same field contract. The exact original head passed 123 tests with hash-locked tooling. Removing only failure-kind extraction/formatting reproduced 3 failures and 13 passing rejection cases; restoring it passed all 138 focused tests. No provider call, review approval, PR closure, or gate change is claimed. Co-Authored-By: Codex Signed-off-by: Seongho Bae --- tests/test_noema_review_gate.py | 39 +++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index b401abb464..eec58bb5f3 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1587,8 +1587,31 @@ def test_allowed_locations_json_truncates_at_the_byte_budget(): assert 0 < len(envelope["locations"]) < len(locations) -def test_call_llm_reports_only_safe_model_from_bounded_http_error(monkeypatch, capsys): - """A gateway HTTP error exposes only its canonical safe model identifier.""" +@pytest.mark.parametrize( + ("failure_kind", "expected_failure_kind"), + [ + pytest.param("structured_output_exhausted", "structured_output_exhausted", id="canonical"), + pytest.param(" upstream_error ", "upstream_error", id="trimmed"), + pytest.param("x" * 200, "x" * 200, id="maximum-length"), + pytest.param("x" * 201, None, id="overlength"), + pytest.param(None, None, id="null"), + pytest.param(True, None, id="boolean"), + pytest.param(1, None, id="integer"), + pytest.param({}, None, id="object"), + pytest.param([], None, id="array"), + pytest.param("", None, id="empty"), + pytest.param(" ", None, id="whitespace"), + pytest.param("upstream\n::error::injected", None, id="newline-command"), + pytest.param("upstream\rerror", None, id="carriage-return"), + pytest.param("upstream\x1b[31m", None, id="terminal-escape"), + pytest.param("upstream\ud800", None, id="surrogate"), + pytest.param("upstream=secret", None, id="field-injection"), + ], +) +def test_call_llm_reports_only_safe_model_from_bounded_http_error( + monkeypatch, capsys, failure_kind, expected_failure_kind +): + """A failed gateway call emits only bounded scalar receipt fields on both surfaces.""" monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") secret = "never-print-this-error-detail" @@ -1597,7 +1620,7 @@ def test_call_llm_reports_only_safe_model_from_bounded_http_error(monkeypatch, c "error": { "detail": { "model": "github_models/deepseek-v3", - "failure_kind": "structured_output_exhausted", + "failure_kind": failure_kind, "terminal_reason": "eligible_candidates_exhausted", "attempts": [{ "provider_name": "nvidia_nim", @@ -1635,7 +1658,15 @@ def open(self, request): assert "upstream_phase=connecting" in output assert "attempt_number=2" in output assert "upstream_status=503" in output - assert "failure_kind=structured_output_exhausted" in output + if expected_failure_kind is None: + assert "failure_kind=" not in output + assert "failure_kind=" not in diagnostic + else: + assert f"failure_kind={expected_failure_kind}" in output + assert f"failure_kind={expected_failure_kind}" in diagnostic + assert output.count("::warning::") == 1 + assert "::error::injected" not in output + assert "::error::injected" not in diagnostic assert "terminal_reason=eligible_candidates_exhausted" in output assert secret not in output assert secret not in diagnostic From 49ae54e789c0a6951b3212e2182e4c64d0348a81 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 20:02:23 +0900 Subject: [PATCH 53/59] fix(noema): preserve canonical gateway error code Keep the existing failure_kind delta. Preserve bounded error.code from the protected gateway envelope in both failed-call diagnostics; do not infer failure cause from the HTTP exception label. The expanded field regression reproduced four failures before the fix; focused Noema and declared-pip environment regression: 159 passed. Co-Authored-By: Codex Signed-off-by: Seongho Bae --- scripts/ci/noema_review_gate.py | 8 +++-- tests/test_noema_review_gate.py | 60 +++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 70e6f6fccc..37e82d3886 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1323,8 +1323,8 @@ def _extract_http_error_telemetry(exc: urllib.error.HTTPError) -> dict[str, str """Read bounded, allowlisted gateway failure telemetry without raw diagnostics. The response body is never returned or logged. Only the canonical - ``error.detail`` receipt fields are allowed; malformed, oversized, or - unexpected envelopes fail closed to no telemetry. + ``error.code`` and allowlisted ``error.detail`` fields are allowed. + Malformed, oversized, or unexpected envelopes fail closed to no telemetry. """ try: raw_bytes = exc.read(MAX_HTTP_ERROR_BODY_BYTES + 1) @@ -1346,11 +1346,14 @@ def _extract_http_error_telemetry(exc: urllib.error.HTTPError) -> dict[str, str return {} telemetry: dict[str, str | int] = {} model = _safe_model_identifier(detail.get("model")) + error_code = _safe_model_identifier(error.get("code")) failure_kind = _safe_model_identifier(detail.get("failure_kind")) terminal_reason = _safe_model_identifier(detail.get("terminal_reason")) attempts = detail.get("attempts") if model is not None: telemetry["served_model"] = model + if error_code is not None: + telemetry["error_code"] = error_code if failure_kind is not None: telemetry["failure_kind"] = failure_kind if terminal_reason is not None: @@ -1386,6 +1389,7 @@ def _format_gateway_error_telemetry(telemetry: dict[str, str | int]) -> str: "upstream_phase", "attempt_number", "upstream_status", + "error_code", "failure_kind", "terminal_reason", ) diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 81e7effa41..c9f7d8b58b 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1587,10 +1587,12 @@ def test_allowed_locations_json_truncates_at_the_byte_budget(): assert 0 < len(envelope["locations"]) < len(locations) +@pytest.mark.parametrize("receipt_field", ["failure_kind", "error_code"]) @pytest.mark.parametrize( - ("failure_kind", "expected_failure_kind"), + ("field_value", "expected_value"), [ pytest.param("structured_output_exhausted", "structured_output_exhausted", id="canonical"), + pytest.param("invalid_structured_output", "invalid_structured_output", id="protected-code"), pytest.param(" upstream_error ", "upstream_error", id="trimmed"), pytest.param("x" * 200, "x" * 200, id="maximum-length"), pytest.param("x" * 201, None, id="overlength"), @@ -1609,33 +1611,36 @@ def test_allowed_locations_json_truncates_at_the_byte_budget(): ], ) def test_call_llm_reports_only_safe_model_from_bounded_http_error( - monkeypatch, capsys, failure_kind, expected_failure_kind + monkeypatch, capsys, receipt_field, field_value, expected_value ): """A failed gateway call emits only bounded scalar receipt fields on both surfaces.""" monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") secret = "never-print-this-error-detail" - body = json.dumps( - { - "error": { - "detail": { - "model": "github_models/deepseek-v3", - "failure_kind": failure_kind, - "terminal_reason": "eligible_candidates_exhausted", - "attempts": [{ - "provider_name": "nvidia_nim", - "phase": "connecting", - "attempt_number": 2, - "provider_status": 503, - "secret": secret, - }], + payload = { + "error": { + "detail": { + "model": "github_models/deepseek-v3", + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [{ + "provider_name": "nvidia_nim", + "phase": "connecting", + "attempt_number": 2, + "provider_status": 503, "secret": secret, - }, - "message": secret, + }], + "secret": secret, }, - "arbitrary": secret, - } - ).encode() + "message": secret, + }, + "arbitrary": secret, + } + if receipt_field == "error_code": + # Current protected gateway errors have a code even without a failure kind. + payload["error"]["code"] = field_value + else: + payload["error"]["detail"]["failure_kind"] = field_value + body = json.dumps(payload).encode() class Opener: def open(self, request): @@ -1658,12 +1663,15 @@ def open(self, request): assert "upstream_phase=connecting" in output assert "attempt_number=2" in output assert "upstream_status=503" in output - if expected_failure_kind is None: - assert "failure_kind=" not in output - assert "failure_kind=" not in diagnostic + if expected_value is None: + assert f"{receipt_field}=" not in output + assert f"{receipt_field}=" not in diagnostic else: - assert f"failure_kind={expected_failure_kind}" in output - assert f"failure_kind={expected_failure_kind}" in diagnostic + assert f"{receipt_field}={expected_value}" in output + assert f"{receipt_field}={expected_value}" in diagnostic + absent_field = "failure_kind" if receipt_field == "error_code" else "error_code" + assert f"{absent_field}=" not in output + assert f"{absent_field}=" not in diagnostic assert output.count("::warning::") == 1 assert "::error::injected" not in output assert "::error::injected" not in diagnostic From 0db01c2615457430018a584be1394f57dfdd7038 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 20:12:15 +0900 Subject: [PATCH 54/59] test(noema): cover sparse gateway failure receipts Address the independent sparse-envelope finding while retaining the existing error telemetry implementation. Removing code extraction reproduced 8 failed/43 passed; restored field matrix 51 passed and focused Noema/edge/environment tests 176 passed. Remove one unused test import found by Ruff. Record the protected-source contract, unknown historical 502 cause, logging limits, separate verification stages, and APA logging guidance in the existing doctoring and gap baseline. Co-Authored-By: Codex Signed-off-by: Seongho Bae --- CHANGELOG.md | 5 ++ .../noema-repair-attempt-telemetry.md | 46 ++++++++++++++++++- docs/product-technical-gap-baseline.md | 14 ++++++ scripts/ci/noema_review_gate.py | 4 +- tests/test_noema_review_gate.py | 38 +++++++++++---- 5 files changed, 94 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06b3dba425..3930d22f92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ - Raised `hourly-review-repair.yml`'s discovery ceiling from 50 to 200 while rotating deterministic 50-PR deep-inspection windows by hourly run number. The scheduler hydrates only the selected window and stops immediately after its single dispatch, preserving access to newer PRs without quadrupling expensive review/check/comment work. See `docs/doctoring/hourly-review-repair-single-file-consolidation.md`'s 2026-09-03 follow-up. ## [Unreleased] +- Preserve the gateway's bounded error classification in failed Noema review + diagnostics, helping maintainers select the relevant investigation without + exposing free-form response bodies. Missing classifications remain unknown; + this does not resolve the historical gateway failure. See PR #1898 and + `docs/doctoring/noema-repair-attempt-telemetry.md`. - Include merge-scheduler entrypoint, core, and regression-test changes in the existing runtime-quality workflow's trigger and suite selector. Scheduler workflow edits retain queue checks and also select the full review-repair diff --git a/docs/doctoring/noema-repair-attempt-telemetry.md b/docs/doctoring/noema-repair-attempt-telemetry.md index ee4d681a59..c89176724c 100644 --- a/docs/doctoring/noema-repair-attempt-telemetry.md +++ b/docs/doctoring/noema-repair-attempt-telemetry.md @@ -14,7 +14,7 @@ The later review established a second ownership error: `contextual-orchestrator` Noema now sends exactly one structured-output request to the configured gateway. GitHub Actions fixes the model alias to `orchestrator/free`; the caller declares no provider, paid fallback, sampling temperature, or fixed inference timeout. `contextual-orchestrator` owns provider discovery, capability routing, structured-output repair, failover, and upstream completion. The repository remains responsible for deterministic local validation and exact-head publication. -Every gateway call emits exactly one passive Actions annotation. Success and failure annotations include caller attempt count, elapsed duration, active phase (`connecting`, `reading`, `decoding`, or `validating`), and a best-effort serving-model identifier. Serving-model text is secret-scrubbed, control-character-normalized, UTF-8 printable, and bounded before it can reach an annotation. Raw model output is never logged. +Every gateway call emits exactly one passive Actions annotation. Success and failure annotations include caller attempt count, elapsed duration, active phase (`connecting`, `reading`, `decoding`, `validating`, or `response_error`), and a best-effort serving-model identifier. The identifier validator trims exterior whitespace, then accepts only 1–200 ASCII characters in its restricted identifier alphabet. Invalid text is omitted, not repaired. This is format and length validation, not arbitrary secret detection; the gateway must supply non-sensitive identifiers. Raw model output is never logged. The local trailing-comma parser remains a deterministic syntax transform only. It may remove a genuine trailing comma after a complete JSON value, but missing-value forms such as `[,]`, `{,}`, `[1,,]`, and `{"a":,}` remain invalid. The transform emits no second attempt-level annotation and never bypasses semantic verdict validation. @@ -32,3 +32,47 @@ If the gateway cannot produce a valid structured verdict, Noema fails closed aft ## Verification The permanent contract test forbids `NOEMA_REPAIR_DEADLINE_SECONDS`, `_repair_wall_clock_deadline`, `NoemaRepairDeadlineExceeded`, `signal.setitimer`, retry-only parameters/recursion, and caller-specified `temperature`. Focused regressions prove one request on success and failure, one annotation per attempt, safe serving-model telemetry, strict missing-value rejection, accepted genuine trailing commas, and preserved exact changed-line diagnostics. + +## 2026-09-05 follow-up: retain the gateway's error classification + +Status: proposed consumer repair in [central PR #1898](https://github.com/ContextualWisdomLab/.github/pull/1898), not protected delivery or a resolved provider incident. This extends the evidence/control-plane requirement and G-02/G-03; ADR-0003's single-request ownership is unchanged. + +### Observed failure and source trace + +[Naruon #1244's failed job](https://github.com/ContextualWisdomLab/naruon/actions/runs/33933793278/job/101247827882) at head `50351e8cacc65b4124ba2145e00d41aeceef0775` reported HTTP 502, one caller attempt, `duration=1469.1s`, `phase=response_error`, and `served_model=deepseek-ai/deepseek-v4-flash-0731`. It did not preserve an error code or failure kind. The exception label `Noema gateway transport failed` therefore does not establish a network failure, nor does it establish structured-output exhaustion. That historical cause remains unknown. + +Protected contextual-orchestrator source at `a080297d2546bb61e89520d637cabc202db331ec` already maps `ProviderResponseError` to HTTP 502 and the literal `invalid_structured_output` in [`server.py:7978`](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/a080297d2546bb61e89520d637cabc202db331ec/contextual_orchestrator/server.py#L7978). [`_send_error`](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/a080297d2546bb61e89520d637cabc202db331ec/contextual_orchestrator/server.py#L8189) adds a request identifier; [`_error_payload`](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/a080297d2546bb61e89520d637cabc202db331ec/contextual_orchestrator/server.py#L686) places the classification in canonical `error.code`. This path has no `failure_kind`, model, or attempt list. Those source facts identify an observable envelope, not the cause of the earlier Naruon run or an immutable release. + +The original #1898 delta at `df0f735f42adbb44d45f3c3a4e503e400b47ed79` retains optional `error.detail.failure_kind`, but still discards `error.code`. Waiting only for contextual-orchestrator [#1004](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1004), whose proposed `36133c8ab85d44fc4be2356edbdd56d9fc09f0d8` adds `structured_output_exhausted`, would leave the existing protected-source envelope unclassified. Noema must not infer that kind from a status code, copy owner repair logic, or consume the proposed branch as a released dependency. + +### Chosen repair and security boundary + +The existing bounded reader and formatter now preserve both independent fields: `error.code` as `error_code`, and `error.detail.failure_kind` as `failure_kind`, when present and valid. The consumer repair is [commit `49ae54e789c0a6951b3212e2182e4c64d0348a81`](https://github.com/ContextualWisdomLab/.github/commit/49ae54e789c0a6951b3212e2182e4c64d0348a81). Both the single failure annotation and the raised diagnostic use the same extracted receipt. Missing fields remain absent; the failure still fails. No request, retry, provider selection, credential, timeout, or verdict-approval rule changes. + +The reader requires canonical mapping envelopes including `error.detail`, reads at most 16 KiB plus one oversize-detection byte, and rejects malformed or oversized bodies. Each new field reuses the existing identifier validator; neither free-form messages, request identifiers, arbitrary detail, nor flattened compatibility aliases are logged. Embedded CR/LF, terminal escape sequences, surrogates, delimiter injection, non-string values, and overlength identifiers cannot enter the new fields. A syntactically valid secret placed in an allowlisted field would not be detected by this validator: non-sensitive canonical classifications remain a producer obligation. + +This follows OWASP's advice to define log field types and lengths, validate data crossing trust zones, prevent log injection, and exclude credentials and sensitive payloads. It does not claim that a regular expression supplies complete redaction (OWASP Foundation, n.d.). + +### Reproduction, integration, and remaining delivery gates + +The existing failed-call regression covers 17 values for each independent field, plus those 17 values with the sparse current gateway envelope: 51 cases. It verifies annotation/exception output, absent sibling fields, one request/annotation, and exclusion of unrelated payload text. The sparse cases have a request identifier and compatibility aliases but no model, attempts, terminal reason, or failure kind. They address an independent static review's missing-fixture finding; they must preserve the code, report an unknown model, and exclude request-ID/message text. Unit-only HTTP doubles replace the external gateway; parsing, formatting, and `call_llm` execute normally. + +Removing the original four `failure_kind` lines produced 3 failures; restoring them produced 138 focused passes. Adding canonical-code assertions to the old implementation reproduced 4 failures and 30 passes, including the actual protected-source code. Before the sparse-fixture extension, the consumer fix plus focused Noema edge coverage and the environment regression produced **159 passed, zero failures/skips**. With the sparse fixture added, removing just the four new code-extraction/formatting lines reproduced **8 failed / 43 passed**; restoring the implementation produced **51 passed**. These are chronological receipts, not totals for the final candidate. + +The first broader run had **2897 passed, one skipped, 21 subtests passed, and one failure**: the task-local uv environment lacked `pip`, needed by `test_materialized_bounded_include_is_resolvable_by_pip`. The project already declares `pip==26.2.1`; installing that exact declared tool fixed the test without changing source, skipping it, or modifying a shared environment. The environment combines hash-locked review requirements and that separately declared pip pin; it is not claimed as an entirely hash-locked clean install. + +The original PR delta and validation commit were preserved by ordinary merges. Protected main `f250638827f8252b0d9e5cb2601f4d333f96162f` (merged prerequisite #1922) is integrated at `719c91b1f678de6da3029b8f5920d6a245520e2e`. A preliminary normal run returned **2923 passed, one skipped, 21 subtests passed** before the sparse-fixture follow-up. The existing LLVM 19 admission test was skipped because its reviewed tools are absent on this macOS host; that path remains unverified, not passed. The separate maintainer exception reported for #1922 is not authorization to bypass #1898's gates. Full normal and `GITHUB_ACTIONS=true` verification must finish on the final integrated candidate, followed by fresh current-head hosted checks and qualifying independent review. Capture exact head/base and final command results in #1898; do not transfer old-head passes. + +Run from the isolated repository root: + +```sh +.venv/bin/python -m pytest -q -W error tests/test_noema_review_gate.py tests/test_noema_model_output_edge_coverage.py +PATH="$PWD/.venv/bin:$PATH" .venv/bin/python -m pytest tests -q -W error -rs +GITHUB_ACTIONS=true PATH="$PWD/.venv/bin:$PATH" .venv/bin/python -m pytest tests -q -W error -rs +``` + +After protected delivery, confirm a real consumer run uses the exact released central revision and gateway contract. If it fails, retain the returned classification and investigate that owner path. Do not reroute to a paid model, repeat an active model request, weaken semantic validation, or declare the historical 502 repaired from unit evidence. Product runtime, real PostgreSQL, browser, release, and deployed-gateway verification are not covered by this consumer diagnostic test. + +### Reference + +OWASP Foundation. (n.d.). *Logging cheat sheet*. OWASP Cheat Sheet Series. Retrieved September 5, 2026, from https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..3fa65395c7 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,19 @@ # Product and Technical Gap Baseline +## 2026-09-05 Noema failure classification follow-up — G-02 / G-03 + +This dated follow-up does not refresh the historical inventory below or authorize a merge. Central [#1898](https://github.com/ContextualWisdomLab/.github/pull/1898) remains a proposed repair. Its consumer integration `719c91b1f678de6da3029b8f5920d6a245520e2e` preserves the original `df0f735f42adbb44d45f3c3a4e503e400b47ed79` delta and includes protected main `f250638827f8252b0d9e5cb2601f4d333f96162f` by normal merge. + +- **Product need:** a failed required review must give maintainers a usable next investigation while protecting customer/source content. This supports the PRD-05 quality-first routing outcome and evidence/control-plane TRD; it does not demonstrate retrieval, calendar, connector, or UI acceptance. +- **Observed gap:** Naruon #1244 at `50351e8cacc65b4124ba2145e00d41aeceef0775` failed in [job 101247827882](https://github.com/ContextualWisdomLab/naruon/actions/runs/33933793278/job/101247827882) after 1469.1 seconds and one request. Phase/model telemetry exists, but no preserved classification proves its cause. A transport exception label is not network-failure evidence. +- **Source contract:** protected contextual-orchestrator `a080297d2546bb61e89520d637cabc202db331ec` already returns canonical `error.code=invalid_structured_output` for a structured-response error, without `failure_kind`. Noema previously dropped that code. The proposed #1004 exhaustion kind is separate and remains unreleased evidence. +- **Action and ownership:** #1898 preserves canonical `error.code` and optional `error.detail.failure_kind` with the existing bounded identifier validator in both failure diagnostics. `.github` owns the consumer receipt; contextual-orchestrator owns classification, model validation/repair, discovery, and routing. No extra model request, mutable owner dependency, raw response logging, approval relaxation, or timeout is added. +- **Verification:** canonical-code RED reproduced 4 failed / 30 passed; repaired focused Noema/edge/environment checks returned 159 passed. The earlier full run's missing declared pip tool was repaired in the isolated environment. Integrated full normal/CI runs, hosted required checks, independent review, protected delivery, and an exact-revision consumer run remain separate gates. This entry does not mark G-02 or G-03 closed. + +The [doctoring record](doctoring/noema-repair-attempt-telemetry.md#2026-09-05-follow-up-retain-the-gateways-error-classification) contains exact producer/API links, the rejected alternatives, logging-security limits, APA 7 reference, executable commands, and the remaining owner investigation. The existing ownership flow is unchanged: workflow → gateway-owned routing/validation → one bounded receipt → deterministic local validation → exact-head publication. + +## Historical baseline + 작성 기준일: **2026-08-26 10:35 KST** 대상: **ContextualWisdomLab/.github** 중앙 거버넌스·자동화 레포지터리와 이를 소비하는 naruon 생태계 현재 보호된 `main`: `826b92394c63deb6981c3a8d16a724d71f85a0d7` diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 37e82d3886..01c1a90ff9 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1299,7 +1299,7 @@ def decode_llm_response_body(raw_bytes: bytes) -> str: def _extract_served_model(raw: str) -> str | None: - """Return a bounded, scrubbed, single-line UTF-8-printable serving model id.""" + """Return a bounded, format-checked ASCII serving-model identifier.""" try: data = json.loads(raw) except (json.JSONDecodeError, TypeError, ValueError): @@ -1310,7 +1310,7 @@ def _extract_served_model(raw: str) -> str | None: def _safe_model_identifier(value: Any) -> str | None: - """Accept only a conservative, bounded model identifier safe for public logs.""" + """Validate a bounded ASCII identifier's format, without detecting secrets.""" if not isinstance(value, str): return None candidate = value.strip() diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index c9f7d8b58b..726d7071de 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,6 +1,5 @@ import base64 import hashlib -import http.client import io import json import os @@ -1587,7 +1586,10 @@ def test_allowed_locations_json_truncates_at_the_byte_budget(): assert 0 < len(envelope["locations"]) < len(locations) -@pytest.mark.parametrize("receipt_field", ["failure_kind", "error_code"]) +@pytest.mark.parametrize( + ("receipt_field", "with_attempt_details"), + [("failure_kind", True), ("error_code", True), ("error_code", False)], +) @pytest.mark.parametrize( ("field_value", "expected_value"), [ @@ -1611,7 +1613,7 @@ def test_allowed_locations_json_truncates_at_the_byte_budget(): ], ) def test_call_llm_reports_only_safe_model_from_bounded_http_error( - monkeypatch, capsys, receipt_field, field_value, expected_value + monkeypatch, capsys, receipt_field, with_attempt_details, field_value, expected_value ): """A failed gateway call emits only bounded scalar receipt fields on both surfaces.""" monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") @@ -1635,15 +1637,25 @@ def test_call_llm_reports_only_safe_model_from_bounded_http_error( }, "arbitrary": secret, } + if not with_attempt_details: + # Mirror _send_error/_error_payload: no model, attempts, or failure kind. + payload["error"]["detail"] = {"request_id": secret} + payload.update( + error_code=field_value, + error_message=secret, + error_detail=payload["error"]["detail"], + ) if receipt_field == "error_code": # Current protected gateway errors have a code even without a failure kind. payload["error"]["code"] = field_value else: payload["error"]["detail"]["failure_kind"] = field_value body = json.dumps(payload).encode() + requests_seen = [] class Opener: def open(self, request): + requests_seen.append(request) raise noema.urllib.error.HTTPError( request.full_url, 502, "Bad Gateway", {}, io.BytesIO(body) ) @@ -1655,14 +1667,21 @@ def open(self, request): output = capsys.readouterr().out diagnostic = str(exc_info.value) + assert len(requests_seen) == 1 assert "phase=response_error" in output - assert "served_model=github_models/deepseek-v3" in output + serving_model = "github_models/deepseek-v3" if with_attempt_details else "unknown" + assert f"served_model={serving_model}" in output assert "phase=response_error" in diagnostic - assert "served_model=github_models/deepseek-v3" in diagnostic - assert "provider_name=nvidia_nim" in output - assert "upstream_phase=connecting" in output - assert "attempt_number=2" in output - assert "upstream_status=503" in output + assert f"served_model={serving_model}" in diagnostic + for receipt_fragment in ( + "provider_name=nvidia_nim", + "upstream_phase=connecting", + "attempt_number=2", + "upstream_status=503", + "terminal_reason=eligible_candidates_exhausted", + ): + assert (receipt_fragment in output) is with_attempt_details + assert (receipt_fragment in diagnostic) is with_attempt_details if expected_value is None: assert f"{receipt_field}=" not in output assert f"{receipt_field}=" not in diagnostic @@ -1675,7 +1694,6 @@ def open(self, request): assert output.count("::warning::") == 1 assert "::error::injected" not in output assert "::error::injected" not in diagnostic - assert "terminal_reason=eligible_candidates_exhausted" in output assert secret not in output assert secret not in diagnostic From fbe26022bcea836de247c156ec04c09fe785e5c9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 20:21:33 +0900 Subject: [PATCH 55/59] fix(noema): unify exact diff evidence parsing Resolve the remaining valid #1641 review findings. One unified-diff state machine now emits both exact source text and accepted coordinates. Bounded truncation drops the incomplete final line instead of manufacturing +/- source, so a genuine line equal to the historical omission marker remains reviewable. Focused RED reproduced both failures before the repair. GREEN: focused 3 passed; Noema 268 passed; full 2,922 passed, 1 skipped, 21 subtests; py_compile and diff checks clean. --- CHANGELOG.md | 2 +- ...ema-observed-defect-corpus-current-main.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- scripts/ci/noema_review_gate.py | 79 +++++-------------- ...ema_class_evidence_observation_contract.py | 12 +-- tests/test_noema_review_gate.py | 8 +- 6 files changed, 32 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aeaabca39..e4f4d73ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,7 +110,7 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] -- **Require source-bound observed defect classes in Noema formal reviews (#1641).** Canonical changed-line coordinates now reject JSON booleans, material reviews must cover distinct classes from the executable external-finding corpus, class witnesses bind to exact changed-side source text (including lexical-shape-independent blank/non-ASCII lines) with non-vacuous causal observations, while bounded-diff omission markers are rejected, and the prompt explicitly attacks workflow-event authority plus mutable-alias, TOCTOU, identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure shapes without fabricating benchmark claims. +- **Require source-bound observed defect classes in Noema formal reviews (#1641).** Canonical changed-line coordinates now reject JSON booleans, material reviews must cover distinct classes from the executable external-finding corpus, and class witnesses bind to exact changed-side source text (including lexical-shape-independent blank/non-ASCII lines) with non-vacuous causal observations. A single parser now owns both source text and coordinates; bounded truncation drops the incomplete line instead of synthesizing a changed-line marker, so genuine source equal to the old marker remains reviewable. The prompt explicitly attacks workflow-event authority plus mutable-alias, TOCTOU, identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure shapes without fabricating benchmark claims. - **Fail closed on fabricated Noema execution and external-source provenance (#1641).** Model-authored claims that runtime behavior, command output, toolchain help, or authoritative external documentation confirmed a conclusion now require an out-of-band typed receipt and an exact receipt citation. The isolated reviewer may still reason from changed source and recommend toolchain-specific verification; it cannot present that recommendation as executed evidence. This regression is grounded in `ConceptWeave#35@a31ae0c2`, where review `5120903874` claimed Cargo runtime/documentation confirmation although required Noema run `33938445009` executed no Cargo or documentation lookup step. - **Pin `opencode-review-dispatch.yml` off the starved floating `ubuntu-latest` image.** The 2026-09-01 floating-image fix (see that entry below) pinned `strix.yml`, diff --git a/docs/doctoring/noema-observed-defect-corpus-current-main.md b/docs/doctoring/noema-observed-defect-corpus-current-main.md index e6cf516ed4..edb5e2b8f9 100644 --- a/docs/doctoring/noema-observed-defect-corpus-current-main.md +++ b/docs/doctoring/noema-observed-defect-corpus-current-main.md @@ -10,7 +10,7 @@ JSON booleans are rejected as line coordinates even though Python considers `Tru This repair is a narrow current-main successor to the heavily diverged PR #1589 evidence lineage. It does not copy CodeRabbitAI or Devin wording and makes no superiority claim. -Exact-head follow-up also makes bounded-diff omission markers ineligible as source evidence. Short identifiers, symbol-only lines, blank changed lines, and non-ASCII source remain admissible through exact string equality rather than lexical guessing. +Exact-head follow-up removes synthetic bounded-diff omission lines from the diff grammar entirely: truncation drops the incomplete final line and carries the separate `truncated` control flag. A genuine source line equal to the historical marker remains admissible, as do short identifiers, symbol-only lines, blank changed lines, and non-ASCII source through exact string equality rather than lexical guessing. Coordinates and source text now come from one parser so future diff fixes cannot desynchronize their trust boundaries. The exact-head structural follow-up removes the fixed English relation-word list. Formal evidence now carries a schema-derived `claim_role` for each defect-class witness, while the deterministic gate verifies exact source identity, canonical coordinates, role identity, and distinct observations. Semantic causal adequacy remains a reviewer/evaluation responsibility; the validator does not pretend English keyword presence proves causality. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 0fa42887db..4e1b03885c 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2634,7 +2634,7 @@ Higgins, S. S., Crepalde, N., & Fernandes, L. (2021). Segmented multiplexity: A - **Regression evidence:** `tests/test_noema_observed_defect_corpus_current_main.py` is committed before the causal production change and covers boolean aliasing, malformed/unknown class labels, duplicate-class diversity, witness/source binding, a valid multi-class verdict, and rendered prompt coverage. - **Authority boundary:** no reviewer, provider, routing, merge, or repository-write authority is widened. The taxonomy is evaluation/admission evidence only. -- **Noema exact-source follow-up (PR #1641):** bounded-diff overlong-line omission markers are not admissible source evidence; short, symbol-only, blank, and non-ASCII changed lines use exact source equality, while arbitrary source-adjacent words do not satisfy causal evidence. +- **Noema exact-source follow-up (PR #1641):** bounded truncation no longer synthesizes a +/- omission line; it drops the incomplete line and carries the separate `truncated` flag. Genuine source equal to the historical marker remains admissible. One parser now emits both changed coordinates and exact source text, while short, symbol-only, blank, and non-ASCII changed lines use exact equality and arbitrary source-adjacent words do not satisfy causal evidence. - **Noema structural-causality follow-up (PR #1641):** removed fixed English relation-word admission. Each class witness now carries an exact schema-derived `claim_role` plus exact changed-line source text; deterministic validation stays language-neutral and semantic causality is tested through reviewer/evaluation regressions rather than guessed from keywords. diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 6d997ff55e..92bd2b4bec 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -584,29 +584,25 @@ def current_actor() -> str: def fetch_diff(repo: str, number: int) -> tuple[str, bool]: - """Fetch the PR diff and truncate it to the bounded LLM prompt size.""" + """Fetch the PR diff and truncate before an incomplete final line.""" diff = run(["gh", "api", f"repos/{repo}/pulls/{number}", "-H", "Accept: application/vnd.github.v3.diff"]) truncated = len(diff) > MAX_DIFF_CHARS if truncated: - marker = "[overlong changed line content omitted]" - bounded = diff[: MAX_DIFF_CHARS - len(marker) - 2] - complete, separator, partial = bounded.rpartition("\n") + bounded = diff[:MAX_DIFF_CHARS] + complete, separator, _partial = bounded.rpartition("\n") if not separator: - return diff[:MAX_DIFF_CHARS], truncated - last_hunk = max(complete.rfind("\n@@"), 0 if complete.startswith("@@") else -1) - last_file = max(complete.rfind("\ndiff --git "), 0 if complete.startswith("diff --git ") else -1) - inside_hunk = last_hunk > last_file - if partial.startswith(("+", "-")) and ( - inside_hunk or not partial.startswith(("+++", "---")) - ): - complete += f"\n{partial[0]}{marker}" + return bounded, truncated + # Do not synthesize a +/- line: such a marker is indistinguishable + # from genuine source with the same text and can become false exact + # changed-line evidence. The explicit ``truncated`` flag tells the + # model and deterministic gate that the bounded diff is incomplete. diff = complete return diff, truncated -def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: - """Return exact LEFT/RIGHT changed-line locations from a unified diff.""" - locations: set[tuple[str, int, str]] = set() +def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: + """Return exact changed-side source text from one unified-diff parser.""" + texts: dict[tuple[str, int, str], str] = {} old_path = new_path = "" old_line = new_line = 0 in_hunk = False @@ -630,59 +626,15 @@ def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: continue if not in_hunk or raw_line.startswith("\\ No newline"): continue - if raw_line.startswith("+"): - if not new_path: - return set() - locations.add((new_path, new_line, "RIGHT")) - new_line += 1 - elif raw_line.startswith("-"): - if not old_path: - return set() - locations.add((old_path, old_line, "LEFT")) - old_line += 1 - else: - old_line += 1 - new_line += 1 - return locations - - -def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: - """Return exact changed-side source text keyed by canonical diff location.""" - texts: dict[tuple[str, int, str], str] = {} - old_path = new_path = "" - old_line = new_line = 0 - in_hunk = False - for raw_line in diff.splitlines(): - if raw_line.startswith("diff --git "): - old_path = new_path = "" - in_hunk = False - continue - if not in_hunk and raw_line.startswith("--- "): - old_path = parse_diff_path(raw_line[4:], "a/") - continue - if not in_hunk and raw_line.startswith("+++ "): - new_path = parse_diff_path(raw_line[4:], "b/") - continue - match = DIFF_HUNK_RE.match(raw_line) - if match: - old_line, new_line = map(int, match.groups()) - in_hunk = True - continue - if not in_hunk or raw_line.startswith(r"\ No newline"): - continue if raw_line.startswith("+"): if not new_path: return {} - source_text = raw_line[1:] - if source_text != "[overlong changed line content omitted]": - texts[(new_path, new_line, "RIGHT")] = source_text + texts[(new_path, new_line, "RIGHT")] = raw_line[1:] new_line += 1 elif raw_line.startswith("-"): if not old_path: return {} - source_text = raw_line[1:] - if source_text != "[overlong changed line content omitted]": - texts[(old_path, old_line, "LEFT")] = source_text + texts[(old_path, old_line, "LEFT")] = raw_line[1:] old_line += 1 else: old_line += 1 @@ -690,6 +642,11 @@ def changed_diff_line_texts(diff: str) -> dict[tuple[str, int, str], str]: return texts +def changed_diff_locations(diff: str) -> set[tuple[str, int, str]]: + """Return coordinates from the exact-source parser to prevent drift.""" + return set(changed_diff_line_texts(diff)) + + def parse_diff_path(raw: str, prefix: str) -> str: """Decode a Git unified-diff path, including C-quoted UTF-8 paths.""" value = raw.split("\t", 1)[0] diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index d1bb158c8f..00892eb261 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -212,7 +212,7 @@ def test_canonical_changed_location_rejects_noncanonical_coordinates( def test_changed_diff_line_texts_covers_context_markers_and_no_newline_marker() -> None: - """Exact-source extraction skips omission markers while preserving neighboring changed text.""" + """A genuine marker-shaped source line remains exact review evidence.""" diff = """diff --git a/src/tool.py b/src/tool.py --- a/src/tool.py +++ b/src/tool.py @@ -225,9 +225,12 @@ def test_changed_diff_line_texts_covers_context_markers_and_no_newline_marker() \\ No newline at end of file """ assert noema.changed_diff_line_texts(diff) == { + ("src/tool.py", 2, "LEFT"): "[overlong changed line content omitted]", + ("src/tool.py", 2, "RIGHT"): "[overlong changed line content omitted]", ("src/tool.py", 3, "LEFT"): "old", ("src/tool.py", 3, "RIGHT"): "new", } + assert noema.changed_diff_locations(diff) == set(noema.changed_diff_line_texts(diff)) def test_changed_diff_line_texts_fails_closed_when_hunk_paths_are_missing() -> None: @@ -264,8 +267,8 @@ def test_blank_changed_source_uses_explicit_blank_marker() -> None: noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) -def test_overlong_omission_marker_cannot_be_source_evidence() -> None: - """A bounded-diff omission marker cannot be reintroduced as an exact source excerpt.""" +def test_literal_omission_marker_source_remains_reviewable() -> None: + """Literal source text must not alias synthetic prompt-truncation metadata.""" marker = "[overlong changed line content omitted]" diff = f"""diff --git a/src/tool.py b/src/tool.py --- a/src/tool.py @@ -279,8 +282,7 @@ def test_overlong_omission_marker_cannot_be_source_evidence() -> None: for field, witness in probe["class_evidence"].items(): witness["source_excerpt"] = marker witness["observation"] = f"{marker} is exact source evidence for {probe['probe_kind']}:{field}." - with pytest.raises(noema.NoemaModelOutputError, match="exact changed-line source_excerpt"): - noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) def test_overlong_class_observation_is_rejected_before_semantic_admission() -> None: diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 5fa23dec53..1ccbf727ba 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -870,16 +870,16 @@ def app_identity(args, **kwargs): monkeypatch.setattr(noema, "run", lambda *args, **kwargs: source) diff, truncated = noema.fetch_diff("owner/repo", 1) assert truncated - assert diff.endswith("+[overlong changed line content omitted]") - assert ("a.py", 1, "RIGHT") in noema.changed_diff_locations(diff) + assert "[overlong changed line content omitted]" not in diff + assert ("a.py", 1, "RIGHT") not in noema.changed_diff_locations(diff) assert len(diff) <= noema.MAX_DIFF_CHARS source = "diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n@@ -0,0 +1 @@\n+++" + "x" * noema.MAX_DIFF_CHARS monkeypatch.setattr(noema, "run", lambda *args, **kwargs: source) diff, truncated = noema.fetch_diff("owner/repo", 1) assert truncated - assert diff.endswith("+[overlong changed line content omitted]") - assert ("a.py", 1, "RIGHT") in noema.changed_diff_locations(diff) + assert "[overlong changed line content omitted]" not in diff + assert ("a.py", 1, "RIGHT") not in noema.changed_diff_locations(diff) assert noema.extract_json_object('{"decision":"approve"}') == {"decision": "approve"} assert noema.extract_json_object('prefix {"decision":"comment"} suffix') == {"decision": "comment"} From 294601776987407b2feac45eb949656dd579d655 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 20:32:23 +0900 Subject: [PATCH 56/59] fix(noema): close whitespace evidence provenance gaps --- ...ema-observed-defect-corpus-current-main.md | 2 +- scripts/ci/noema_review_gate.py | 4 +-- ...ema_class_evidence_observation_contract.py | 25 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/doctoring/noema-observed-defect-corpus-current-main.md b/docs/doctoring/noema-observed-defect-corpus-current-main.md index edb5e2b8f9..33c966ba68 100644 --- a/docs/doctoring/noema-observed-defect-corpus-current-main.md +++ b/docs/doctoring/noema-observed-defect-corpus-current-main.md @@ -2,7 +2,7 @@ The trusted Noema review gate treats externally demonstrated review misses as executable regression evidence, not as benchmark claims. Material source/test reviews must exercise at least two distinct observed defect classes and every admitted class witness remains bound to an exact changed-side source coordinate. -The current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are `{path,line,side,source_excerpt,observation}` records bound to the probe location. `source_excerpt` must equal the exact changed-side line, and `observation` must quote the exact source line (or ``) plus a causal/behavioral relation beyond taxonomy labels; ASCII token shape is not admission authority; repeated or differently worded generic labels do not satisfy the deterministic validator. +The current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are `{path,line,side,source_excerpt,claim_role,observation}` records bound to the probe location. `source_excerpt` must equal the exact changed-side line, and `observation` must quote the exact source line (or ``) plus a causal/behavioral relation beyond taxonomy labels; ASCII token shape is not admission authority; repeated or differently worded generic labels do not satisfy the deterministic validator. The model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. For CI/automation changes, the review prompt also requires checking whether the mutation credential can create the downstream events/checks the state machine depends on. diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 92bd2b4bec..33b368be6c 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -737,11 +737,11 @@ def _validate_observed_probe_class_evidence( f"Noema adversarial probe {index} class_evidence.{field} observation " f"exceeds {MAX_THREAD_BODY_CHARS} characters" ) - source_marker = source_excerpt if source_excerpt else "" + source_marker = source_excerpt if source_excerpt.strip() else "" if source_marker not in observation: raise NoemaModelOutputError( f"Noema adversarial probe {index} class_evidence.{field} observation " - "must quote the exact source_excerpt (or for an empty line)" + "must quote the exact source_excerpt (or for a blank line)" ) expected_claim_role = OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field] claim_role = source_ref.get("claim_role") diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index 00892eb261..f4cc5fbad8 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -267,6 +267,31 @@ def test_blank_changed_source_uses_explicit_blank_marker() -> None: noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) +def test_whitespace_only_changed_source_uses_explicit_blank_marker() -> None: + """Whitespace-only source cannot satisfy evidence through incidental prose spaces.""" + spaces = " " + diff = f"""diff --git a/src/tool.py b/src/tool.py +--- a/src/tool.py ++++ b/src/tool.py +@@ -1 +1 @@ +-old = 1 ++{spaces} +""" + verdict = _verdict(observations=True, source_excerpt=True) + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["source_excerpt"] = spaces + witness["observation"] = f"ordinary prose space is not evidence for {probe['probe_kind']}:{field}." + + with pytest.raises(noema.NoemaModelOutputError, match=r"must quote the exact source_excerpt"): + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + for probe in verdict["adversarial_validation"]["probes"]: + for field, witness in probe["class_evidence"].items(): + witness["observation"] = f" is exact source evidence for {probe['probe_kind']}:{field}." + noema.validate_substantive_verdict(verdict, diff, ["src/tool.py"]) + + def test_literal_omission_marker_source_remains_reviewable() -> None: """Literal source text must not alias synthetic prompt-truncation metadata.""" marker = "[overlong changed line content omitted]" From 43a1fdc93d998e19947cf157ac8df5c780e9887c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 22:30:31 +0900 Subject: [PATCH 57/59] fix(noema): align structured probe contracts --- CHANGELOG.md | 1 + ...ema-observed-defect-corpus-current-main.md | 2 + docs/product-technical-gap-baseline.md | 2 + scripts/ci/noema_review_gate.py | 84 ++++++++++++---- ...ema_class_evidence_observation_contract.py | 98 +++++++++++++++++++ tests/test_noema_repair_attempt_telemetry.py | 11 ++- 6 files changed, 177 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f4d73ca0..c04d2787f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- **Keep Noema's strict output schema and deterministic probe validator identical (#1641).** Each structured probe now declares its closed `probe_kind` together with the exact required `class_evidence` witness roles and source receipt fields. Nested `anyOf` variants preserve strict OpenAI-compatible required/additional-property semantics, so a realistic verdict cannot be rejected merely because the outbound schema and local admission contract disagree. The single-request invalid-location regression now reaches and asserts the intended changed-side rejection instead of passing on an earlier status mismatch. - **Require source-bound observed defect classes in Noema formal reviews (#1641).** Canonical changed-line coordinates now reject JSON booleans, material reviews must cover distinct classes from the executable external-finding corpus, and class witnesses bind to exact changed-side source text (including lexical-shape-independent blank/non-ASCII lines) with non-vacuous causal observations. A single parser now owns both source text and coordinates; bounded truncation drops the incomplete line instead of synthesizing a changed-line marker, so genuine source equal to the old marker remains reviewable. The prompt explicitly attacks workflow-event authority plus mutable-alias, TOCTOU, identity, oracle, contract, authority, dependency-context, coercion, and state-machine failure shapes without fabricating benchmark claims. - **Fail closed on fabricated Noema execution and external-source provenance (#1641).** Model-authored claims that runtime behavior, command output, toolchain help, or authoritative external documentation confirmed a conclusion now require an out-of-band typed receipt and an exact receipt citation. The isolated reviewer may still reason from changed source and recommend toolchain-specific verification; it cannot present that recommendation as executed evidence. This regression is grounded in `ConceptWeave#35@a31ae0c2`, where review `5120903874` claimed Cargo runtime/documentation confirmation although required Noema run `33938445009` executed no Cargo or documentation lookup step. - **Pin `opencode-review-dispatch.yml` off the starved floating `ubuntu-latest` image.** diff --git a/docs/doctoring/noema-observed-defect-corpus-current-main.md b/docs/doctoring/noema-observed-defect-corpus-current-main.md index 33c966ba68..9186be47a9 100644 --- a/docs/doctoring/noema-observed-defect-corpus-current-main.md +++ b/docs/doctoring/noema-observed-defect-corpus-current-main.md @@ -4,6 +4,8 @@ The trusted Noema review gate treats externally demonstrated review misses as ex The current closed taxonomy is: `mutable_alias`, `time_of_check_time_of_use`, `execution_identity`, `coercion_boundary`, `test_oracle`, `cross_contract`, `authority_boundary`, `dependency_context`, and `state_machine_race`. Each class has class-specific witness keys. Witness values are `{path,line,side,source_excerpt,claim_role,observation}` records bound to the probe location. `source_excerpt` must equal the exact changed-side line, and `observation` must quote the exact source line (or ``) plus a causal/behavioral relation beyond taxonomy labels; ASCII token shape is not admission authority; repeated or differently worded generic labels do not satisfy the deterministic validator. +The outbound strict structured-output schema and the local validator share that same closed contract. Every probe is one nested `anyOf` variant that correlates a single `probe_kind` with exactly its required `class_evidence` keys; every witness field is required and unknown fields are rejected. Only the containing `adversarial_validation` value is nullable for a non-formal comment. This follows the strict structured-output rule that object properties are required (nullable when truly optional) and prevents the gateway from accepting a probe shape that deterministic admission must reject. + The model is explicitly asked to attack mutable/immutability escapes, changing getters/TOCTOU, request or tenant identity confusion, weak/vacuous oracles, cross-contract contradictions, authority overreach, missing causal dependency context, and reliability/security state-machine races. A falsified hypothesis is valid evidence and must not be promoted into a finding merely to satisfy taxonomy diversity. For CI/automation changes, the review prompt also requires checking whether the mutation credential can create the downstream events/checks the state machine depends on. JSON booleans are rejected as line coordinates even though Python considers `True == 1`: changed-line evidence requires `type(line) is int` and a positive value. Production review calls always provide the complete changed-path manifest, which activates the observed taxonomy; direct validator unit tests may omit that manifest to exercise lower-level generic schema boundaries independently. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 4e1b03885c..113373a73e 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2638,6 +2638,8 @@ Higgins, S. S., Crepalde, N., & Fernandes, L. (2021). Segmented multiplexity: A - **Noema structural-causality follow-up (PR #1641):** removed fixed English relation-word admission. Each class witness now carries an exact schema-derived `claim_role` plus exact changed-line source text; deterministic validation stays language-neutral and semantic causality is tested through reviewer/evaluation regressions rather than guessed from keywords. +- **Noema strict-schema parity follow-up (PR #1641):** the outbound response schema now correlates every observed `probe_kind` with the exact required class-witness object that production validates. A realistic verdict is applied to both contracts in one regression, and invalid changed-line telemetry reaches the intended coordinate rejection before asserting the one-request boundary. + ### 2026-09-05 — Noema executed-evidence provenance boundary (#1641) - **Observed RED:** `ConceptualWisdomLab/ConceptWeave#35@a31ae0c2df920f2794f7ddb456795b04797ab472` received CHANGES_REQUESTED review `5120903874`, which stated that Cargo CLI documentation and runtime behavior confirmed `cargo generate-lockfile --locked` was unsupported. Required Noema run `33938445009`, job `101256294197`, used trusted workflow source `8272e4f95c253ab067592460cc9288581bf3a422`; its model phase invoked only the isolated Noema gateway client. No Cargo command, help lookup, or official-document retrieval step executed. Cargo 1.98.0's actual help is contrary evidence, but this central repair does not hard-code a Cargo verdict or remove the consumer lockfile guard. diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 33b368be6c..8cc4e2fa88 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -133,27 +133,71 @@ }, "required": ["path", "line", "side", "analysis"], } -_NOEMA_PROBE_SCHEMA: dict[str, Any] = { - "type": "object", - "additionalProperties": False, - "properties": { +_NOEMA_PROBE_BASE_PROPERTIES: dict[str, Any] = { + "path": {"type": "string"}, + "line": {"type": "integer"}, + "side": {"type": "string", "enum": ["LEFT", "RIGHT"]}, + "hypothesis": {"type": "string"}, + "attack_or_counterexample": {"type": "string"}, + "evidence": {"type": "string"}, + "outcome": {"type": "string", "enum": ["falsified", "confirmed"]}, +} + + +def _noema_class_evidence_witness_schema(claim_role: str) -> dict[str, Any]: + """Return one strict changed-source witness schema for a taxonomy role.""" + properties = { "path": {"type": "string"}, "line": {"type": "integer"}, "side": {"type": "string", "enum": ["LEFT", "RIGHT"]}, - "hypothesis": {"type": "string"}, - "attack_or_counterexample": {"type": "string"}, - "evidence": {"type": "string"}, - "outcome": {"type": "string", "enum": ["falsified", "confirmed"]}, - }, - "required": [ - "path", - "line", - "side", - "hypothesis", - "attack_or_counterexample", - "evidence", - "outcome", - ], + "source_excerpt": {"type": "string"}, + "claim_role": {"type": "string", "enum": [claim_role]}, + "observation": {"type": "string"}, + } + return { + "type": "object", + "additionalProperties": False, + "properties": properties, + "required": list(properties), + } + + +def _noema_observed_probe_schema(probe_kind: str) -> dict[str, Any]: + """Return a strict probe variant correlated with its exact evidence roles.""" + evidence_properties = { + field: _noema_class_evidence_witness_schema( + OBSERVED_REVIEW_PROBE_CLAIM_ROLES[probe_kind][field] + ) + for field in OBSERVED_REVIEW_PROBE_EVIDENCE_FIELDS[probe_kind] + } + properties = { + **_NOEMA_PROBE_BASE_PROPERTIES, + "probe_kind": {"type": "string", "enum": [probe_kind]}, + "class_evidence": { + "type": "object", + "additionalProperties": False, + "properties": evidence_properties, + "required": list(evidence_properties), + }, + } + return { + "type": "object", + "additionalProperties": False, + "properties": properties, + "required": list(properties), + } + + +# Keep the kind and its exact class-evidence key set in the same ``anyOf`` +# branch. Independent enums would let the gateway admit a mismatched pair that +# the deterministic validator must reject. All formal probe fields are required +# rather than nullable; only their containing adversarial-validation object may +# be null for a non-formal comment verdict. +_NOEMA_PROBE_SCHEMA: dict[str, Any] = { + "anyOf": [ + _noema_observed_probe_schema(probe_kind) + for probe_kind in sorted(OBSERVED_REVIEW_PROBE_KINDS) + ] } _NOEMA_FINDING_SCHEMA: dict[str, Any] = { "type": "object", @@ -173,8 +217,8 @@ def _noema_verdict_json_schema(required_probes: int) -> dict[str, Any]: ``required_probes`` must come from ``_required_probe_count(diff, changed_paths)`` -- the same call ``validate_substantive_verdict`` uses -- so the gateway-enforced structural floor and the Python-side backstop - can never silently diverge. The static per-field schemas above are safe - to share by reference here since nothing in this module mutates them. + can never silently diverge. The probe schema also correlates each closed + taxonomy kind with the exact witness roles enforced by the local validator. """ return { "type": "object", diff --git a/tests/test_noema_class_evidence_observation_contract.py b/tests/test_noema_class_evidence_observation_contract.py index f4cc5fbad8..42fde191d4 100644 --- a/tests/test_noema_class_evidence_observation_contract.py +++ b/tests/test_noema_class_evidence_observation_contract.py @@ -16,6 +16,73 @@ """ +def _assert_matches_declared_schema(value: object, schema: dict[str, object]) -> None: + """Apply the strict-output JSON Schema subset used by the Noema contract.""" + variants = schema.get("anyOf") + if isinstance(variants, list): + failures: list[str] = [] + for variant in variants: + try: + _assert_matches_declared_schema(value, variant) + except AssertionError as exc: + failures.append(str(exc)) + else: + return + raise AssertionError("no anyOf variant admitted the verdict: " + "; ".join(failures)) + + expected_type = schema.get("type") + allowed_types = expected_type if isinstance(expected_type, list) else [expected_type] + if value is None: + actual_type = "null" + elif isinstance(value, dict): + actual_type = "object" + elif isinstance(value, list): + actual_type = "array" + elif type(value) is int: + actual_type = "integer" + elif isinstance(value, str): + actual_type = "string" + else: + actual_type = type(value).__name__ + assert actual_type in allowed_types, f"expected {allowed_types}, got {actual_type}" + + if "enum" in schema: + assert value in schema["enum"] + if actual_type == "object": + properties = schema.get("properties") + assert isinstance(properties, dict) + required = schema.get("required") + assert isinstance(required, list) + assert set(required) == set(properties), "strict objects require every property" + assert set(value) == set(properties), "required/additional properties diverged" + for key, child_schema in properties.items(): + _assert_matches_declared_schema(value[key], child_schema) + elif actual_type == "array": + assert len(value) >= int(schema.get("minItems", 0)) + for item in value: + _assert_matches_declared_schema(item, schema["items"]) + + +def _assert_strict_object_contract(schema: dict[str, object]) -> None: + """Require every nested object variant to use the strict SDK shape.""" + variants = schema.get("anyOf") + if isinstance(variants, list): + for variant in variants: + _assert_strict_object_contract(variant) + return + schema_type = schema.get("type") + allowed_types = schema_type if isinstance(schema_type, list) else [schema_type] + if "object" in allowed_types: + properties = schema.get("properties") + assert isinstance(properties, dict) + assert schema.get("additionalProperties") is False + assert set(schema.get("required", [])) == set(properties) + for child_schema in properties.values(): + _assert_strict_object_contract(child_schema) + if "array" in allowed_types: + _assert_strict_object_contract(schema["items"]) + + def _location() -> dict[str, object]: """Return the single exact changed-side location used by this fixture.""" return {"path": "src/tool.py", "line": 1, "side": "RIGHT"} @@ -194,6 +261,37 @@ def test_distinct_source_bound_class_observations_are_accepted() -> None: ) +def test_outbound_strict_schema_and_local_validator_admit_the_same_verdict() -> None: + """The exact structured-output receipt cannot contradict local admission.""" + verdict = _verdict(observations=True, source_excerpt=True) + schema = noema._noema_verdict_json_schema(required_probes=2) + + _assert_matches_declared_schema(verdict, schema) + noema.validate_substantive_verdict(verdict, DIFF, ["src/tool.py"]) + + +def test_every_outbound_probe_variant_is_strict_and_taxonomy_complete() -> None: + """Nested unions remain SDK-compatible and cover the closed local taxonomy.""" + response_format = noema._noema_verdict_response_format(required_probes=2) + assert response_format["json_schema"]["strict"] is True + schema = response_format["json_schema"]["schema"] + assert schema["type"] == "object" + _assert_strict_object_contract(schema) + + probe_variants = schema["properties"]["adversarial_validation"]["properties"]["probes"][ + "items" + ]["anyOf"] + assert { + variant["properties"]["probe_kind"]["enum"][0] + for variant in probe_variants + } == noema.OBSERVED_REVIEW_PROBE_KINDS + assert schema["properties"]["adversarial_validation"]["type"] == ["object", "null"] + assert all( + variant["properties"]["class_evidence"]["type"] == "object" + for variant in probe_variants + ) + + @pytest.mark.parametrize( ("record", "message"), [ diff --git a/tests/test_noema_repair_attempt_telemetry.py b/tests/test_noema_repair_attempt_telemetry.py index 162fd4591d..45abf99520 100644 --- a/tests/test_noema_repair_attempt_telemetry.py +++ b/tests/test_noema_repair_attempt_telemetry.py @@ -226,6 +226,12 @@ def test_malformed_verdict_json_is_not_retried(monkeypatch) -> None: def test_rejected_changed_line_verdict_is_not_retried(monkeypatch) -> None: verdict = _verdict() verdict["decision"] = "request_changes" + verdict["adversarial_validation"]["status"] = "failed" + probe = verdict["adversarial_validation"]["probes"][0] + probe["line"] = 99 + probe["outcome"] = "confirmed" + for witness in probe["class_evidence"].values(): + witness["line"] = 99 verdict["findings"] = [{ "severity": "high", "file": "README.md", @@ -235,6 +241,9 @@ def test_rejected_changed_line_verdict_is_not_retried(monkeypatch) -> None: }] raw = json.dumps({"model": "provider/model", "choices": [{"message": {"content": json.dumps(verdict)}}]}).encode() calls, kwargs = _invoke_once(monkeypatch, raw=raw) - with pytest.raises(gate.NoemaModelOutputError, match="caller attempts=1"): + with pytest.raises( + gate.NoemaModelOutputError, + match=r"adversarial probe entry 1/1 .*line=99.*not an exact changed-side line.*caller attempts=1", + ): gate.call_llm(**kwargs) assert len(calls) == 1 From 9df1ea4c03521aa69e9ba6b48fa4d940d858fb0c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 22:35:15 +0900 Subject: [PATCH 58/59] test(noema): cover malformed provenance containers --- ...ema_observed_defect_corpus_current_main.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_noema_observed_defect_corpus_current_main.py b/tests/test_noema_observed_defect_corpus_current_main.py index baf0874b98..91fadc500e 100644 --- a/tests/test_noema_observed_defect_corpus_current_main.py +++ b/tests/test_noema_observed_defect_corpus_current_main.py @@ -201,6 +201,48 @@ def test_trusted_receipt_must_be_typed_and_explicitly_cited() -> None: ) +def test_evidence_statement_collection_ignores_non_prose_container_values() -> None: + """Malformed optional containers cannot become provenance claim text.""" + verdict = { + "summary": " ", + "reviewed_lines": [None, {"analysis": "reviewed source"}], + "adversarial_validation": { + "residual_risk": None, + "probes": [ + None, + { + "hypothesis": "hypothesis", + "attack_or_counterexample": "attack", + "evidence": "evidence", + "class_evidence": None, + }, + { + "hypothesis": "second hypothesis", + "attack_or_counterexample": "second attack", + "evidence": "second evidence", + "class_evidence": { + "malformed": None, + "valid": {"observation": "observed source"}, + }, + }, + ], + }, + "findings": [None, {"message": "finding"}], + } + + assert noema._model_evidence_statements(verdict) == [ + "reviewed source", + "hypothesis", + "attack", + "evidence", + "second hypothesis", + "second attack", + "second evidence", + "observed source", + "finding", + ] + + def test_noema_prompt_names_every_observed_defect_class(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/v1/chat/completions") monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") From ad48dd65c7d0d8b6e0d37f0315302b9c6e138899 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 17:23:11 +0900 Subject: [PATCH 59/59] =?UTF-8?q?fix(noema):=20=EC=9D=91=EB=8B=B5=20?= =?UTF-8?q?=EC=A0=95=EB=A6=AC=20=EC=98=A4=EB=A5=98=EC=99=80=20=EC=9B=90?= =?UTF-8?q?=EB=9E=98=20=ED=86=B5=EC=8B=A0=20=EC=8B=A4=ED=8C=A8=EB=A5=BC=20?= =?UTF-8?q?=EB=B6=84=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 정리 중 일반 예외가 원래 실패를 덮지 않게 하되 사용자 중단은 전파한다. 직접 리다이렉트 검사는 자신이 받은 응답을 닫는다. Commit-Message-Assisted-by: Codex (OpenAI) Signed-off-by: Seongho Bae --- CHANGELOG.md | 3 +++ .../noema-observed-defect-corpus-current-main.md | 8 ++++++++ scripts/ci/noema_review_gate.py | 6 +++--- tests/test_noema_review_gate.py | 13 +++++++++---- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 187a4432aa..ae7549aed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,9 @@ - Raised `hourly-review-repair.yml`'s discovery ceiling from 50 to 200 while rotating deterministic 50-PR deep-inspection windows by hourly run number. The scheduler hydrates only the selected window and stops immediately after its single dispatch, preserving access to newer PRs without quadrupling expensive review/check/comment work. See `docs/doctoring/hourly-review-repair-single-file-consolidation.md`'s 2026-09-03 follow-up. ## [Unreleased] +- Failed Noema reviews retain the original network failure even when response + cleanup also fails, while process cancellation still stops the review. Direct + redirect-rejection tests now release their responses explicitly. - Include merge-scheduler entrypoint, core, and regression-test changes in the existing runtime-quality workflow's trigger and suite selector. Scheduler workflow edits retain queue checks and also select the full review-repair diff --git a/docs/doctoring/noema-observed-defect-corpus-current-main.md b/docs/doctoring/noema-observed-defect-corpus-current-main.md index 9186be47a9..efef8db6a5 100644 --- a/docs/doctoring/noema-observed-defect-corpus-current-main.md +++ b/docs/doctoring/noema-observed-defect-corpus-current-main.md @@ -17,3 +17,11 @@ Exact-head follow-up removes synthetic bounded-diff omission lines from the diff The exact-head structural follow-up removes the fixed English relation-word list. Formal evidence now carries a schema-derived `claim_role` for each defect-class witness, while the deterministic gate verifies exact source identity, canonical coordinates, role identity, and distinct observations. Semantic causal adequacy remains a reviewer/evaluation responsibility; the validator does not pretend English keyword presence proves causality. Workflow-local bootstrap or generated commits are not accepted as final review/check proof merely because their source transaction verified locally. The merge candidate must be a workflow-starting successor writer head produced through ordinary owner-side mutation, with the required review and quality checks observed on that exact unchanged head before merge. + +Failed HTTP responses belong to the requesting transport. After bounded telemetry +extraction, close the response there; a secondary cleanup exception must not replace +the original typed transport failure. Process cancellation still propagates. Tests +that invoke a redirect handler directly own the resulting HTTPError and must close +it themselves rather than relying on garbage collection. Run the Noema regression +tests with `-W error`; tests in other HTTP consumers do not become passing evidence +merely because this transport was repaired. diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index a5b42ea7c1..586187d541 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -20,6 +20,7 @@ import urllib.parse import urllib.request from collections.abc import Sequence +from contextlib import suppress from typing import Any from scripts.ci.opencode_review_normalize_output import changed_file_is_material @@ -1979,10 +1980,9 @@ def call_llm( # HTTPError owns its response body. Telemetry reads only a # bounded allowlisted prefix, then this caller must release the # socket/file even when decoding or schema inspection fails. - try: + # Keep the primary transport error; process cancellation still propagates. + with suppress(Exception): exc.close() - except (OSError, ValueError, http.client.HTTPException): - pass model_value = gateway_telemetry.get("served_model") served_model = model_value if isinstance(model_value, str) else None elapsed = time.monotonic() - attempt_started diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 7811de37da..5c44090de5 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1679,8 +1679,9 @@ def open(self, _request): assert body.closed +@pytest.mark.parametrize("close_error_type", [OSError, ValueError, RuntimeError, KeyboardInterrupt]) def test_call_llm_preserves_typed_transport_failure_when_http_error_close_fails( - monkeypatch, + monkeypatch, close_error_type, ): """A cleanup error cannot mask the original bounded gateway failure.""" monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") @@ -1688,7 +1689,8 @@ def test_call_llm_preserves_typed_transport_failure_when_http_error_close_fails( class CloseFailsHTTPError(noema.urllib.error.HTTPError): def close(self): - raise OSError("cleanup failed") + super().close() + raise close_error_type("cleanup failed") error = CloseFailsHTTPError( "https://llm.example.test/chat", @@ -1704,7 +1706,9 @@ def open(self, _request): monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) - with pytest.raises(noema.NoemaTransportError, match="HTTP Error 502"): + expected_error = KeyboardInterrupt if close_error_type is KeyboardInterrupt else noema.NoemaTransportError + expected_message = "cleanup failed" if close_error_type is KeyboardInterrupt else "HTTP Error 502" + with pytest.raises(expected_error, match=expected_message): noema.call_llm("owner/repo", 1, make_pr(), "diff", False, "head") @@ -1812,7 +1816,7 @@ def test_noema_redirect_handler_rejects_redirects(): handler = noema.NoRedirectHandler() request = noema.urllib.request.Request("https://llm.example.test/chat") - with pytest.raises(noema.urllib.error.HTTPError): + with pytest.raises(noema.urllib.error.HTTPError) as error_info: handler.redirect_request( request, fp=None, @@ -1821,6 +1825,7 @@ def test_noema_redirect_handler_rejects_redirects(): headers={}, newurl="http://169.254.169.254/latest/meta-data/", ) + error_info.value.close() def test_call_llm_rejects_control_character_scheme_evasion(monkeypatch):