From 0102bcc8095203d141988e9bd0d19f0f588700cb Mon Sep 17 00:00:00 2001 From: Gadi Evron Date: Mon, 20 Jul 2026 22:37:30 +0300 Subject: [PATCH 01/44] fix(tests): stop a test evicting core modules and billing a real API call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_F3_verdict_taxonomy_shared_constant.py deleted every core.* entry from sys.modules without restoring it. Any test collected afterwards that bound `import core.X as m` and monkeypatched m.attr was silently defeated: the code under test re-imported and got a DIFFERENT module object, so the patch applied to an orphan. That is why test_enhance_limit bypassed its stub and issued a live Anthropic API call during an offline run — a billing hazard, not merely untidy. The eviction is now save/restore-wrapped, with a regression lock. The same file defaulted its root to an absolute path inside another machine session's scratchpad. Where that directory still exists the test does not error — it silently asserts against a stale, different tree, which is why it reported core/reporter.py as missing a constant it has always imported. Seven test files carried machine-specific absolute paths; all now derive from __file__. tests/test_no_hardcoded_paths.py guards every test file against recurrence, with a self-test so the matcher cannot silently rot. test_F4_entry_root_additive_v3 required an out-of-tree pre-#165 checkout and a .patch file that was never committed, so it has failed since the day it landed — as a hard assert, not a skip. Now skipped with a precise reason and replaced by forward-looking tests asserting the current tree seeds the F4 entry points (APIRouter, aiohttp RouteTableDef, Starlette websocket_route, Django CBV), which had zero coverage: it was the only test naming those markers and it aborted before reaching any assertion. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test_F1_receiver_type_contract.py | 3 +- .../test_F2_shared_segment_exclusion.py | 2 +- .../python/test_F4_entry_root_additive_v3.py | 60 +++++++++++- .../zig/test_zig_alias_CORRECT_setunion.py | 3 +- ...t_zig_alias_index_recursion_depth_bound.py | 3 +- ...est_F3_verdict_taxonomy_shared_constant.py | 80 +++++++++++++--- .../tests/test_json_corrector_output_cap.py | 3 +- .../tests/test_no_hardcoded_paths.py | 94 +++++++++++++++++++ ...est_verifier_casing_ingestion_normalize.py | 3 +- 9 files changed, 228 insertions(+), 23 deletions(-) create mode 100644 libs/openant-core/tests/test_no_hardcoded_paths.py diff --git a/libs/openant-core/tests/conformance/test_F1_receiver_type_contract.py b/libs/openant-core/tests/conformance/test_F1_receiver_type_contract.py index 5bbbd279..9283a25c 100644 --- a/libs/openant-core/tests/conformance/test_F1_receiver_type_contract.py +++ b/libs/openant-core/tests/conformance/test_F1_receiver_type_contract.py @@ -28,8 +28,7 @@ IMPL_CORE = os.environ.get( "IMPL_CORE", - "/private/tmp/claude-501/-Users-gadievron-Documents-ClaudeNew-OpenAnt-new-bugs-2/" - "e77a0496-1f59-4f65-80e9-fa508d40fa3c/scratchpad/impl-core", + str(Path(__file__).resolve().parent.parent.parent), ) diff --git a/libs/openant-core/tests/conformance/test_F2_shared_segment_exclusion.py b/libs/openant-core/tests/conformance/test_F2_shared_segment_exclusion.py index d2a94f8c..dc47ea45 100644 --- a/libs/openant-core/tests/conformance/test_F2_shared_segment_exclusion.py +++ b/libs/openant-core/tests/conformance/test_F2_shared_segment_exclusion.py @@ -41,7 +41,7 @@ _CORE_ROOT = os.environ.get( "OPENANT_CORE_ROOT", - "/Users/gadievron/Documents/ClaudeNew/OpenAnt/new-bugs-2/OpenAnt/libs/openant-core", + str(Path(__file__).resolve().parent.parent.parent), ) if _CORE_ROOT not in sys.path: sys.path.insert(0, _CORE_ROOT) diff --git a/libs/openant-core/tests/parsers/python/test_F4_entry_root_additive_v3.py b/libs/openant-core/tests/parsers/python/test_F4_entry_root_additive_v3.py index 3309e74c..40fa43eb 100644 --- a/libs/openant-core/tests/parsers/python/test_F4_entry_root_additive_v3.py +++ b/libs/openant-core/tests/parsers/python/test_F4_entry_root_additive_v3.py @@ -39,10 +39,16 @@ import shutil import subprocess import sys + +import pytest import tempfile from pathlib import Path HERE = Path(__file__).resolve().parent +# This repo's own openant-core — tests/parsers/python/ -> openant-core. +# Used by the forward-looking tests, which assert against the CURRENT tree and +# so need no external checkout. +CORE_ROOT = HERE.parent.parent.parent PATCH = HERE / "F4-entry-root-additive-v3.patch" # Pristine core: fixes/ and OpenAnt/ are siblings under new-bugs-2/. Allow an # explicit override so the test is relocatable. @@ -213,8 +219,24 @@ def _build_patched_core() -> Path: def test_strict_superset_and_new_seeds(): - assert PRISTINE_CORE.is_dir(), f"pristine core not found: {PRISTINE_CORE}" - assert PATCH.is_file(), f"patch not found: {PATCH}" + # This is a development-time RED/GREEN harness: it compares a PRE-#165 + # checkout against the same tree with the F4 patch applied. Commit 858f5d6 + # merged that patch, and the .patch file was never committed, so neither + # precondition can be satisfied from inside this repo — it has failed on + # every run since the day it landed. + # + # Skipped rather than left red so it stops masquerading as a signal. The + # coverage it was meant to provide now lives in + # test_f4_seeds_are_produced_by_the_current_core below, which asserts the + # same seeds forward against the current tree and needs no external + # checkout. + if not PRISTINE_CORE.is_dir() or not PATCH.is_file(): + pytest.skip( + "pristine-vs-patched harness needs an out-of-tree pre-#165 checkout " + f"({PRISTINE_CORE}) and a patch file ({PATCH}) that is not committed; " + "forward-looking coverage lives in " + "test_f4_seeds_are_produced_by_the_current_core" + ) repo = _write_fixture_repo() patched_core = _build_patched_core() @@ -265,3 +287,37 @@ def test_strict_superset_and_new_seeds(): sys.exit(2) print("PASS test_strict_superset_and_new_seeds") print("all tests PASSED") + + +def test_f4_seeds_are_produced_by_the_current_core(): + """Forward-looking replacement for the pristine-vs-patched comparison. + + Asserts that the CURRENT tree seeds the F4 entry-point patterns — + custom ``APIRouter`` instances, aiohttp ``RouteTableDef``, Starlette + ``websocket_route``, and Django class-based-view dispatch methods — against + the same fixture repo the original harness used. + + This is the only coverage of those patterns in the suite. The original test + could not provide it: it aborts on an unsatisfiable precondition before + reaching a single assertion. + """ + repo = _write_fixture_repo() + seeds = _run_pipeline(CORE_ROOT, repo) + + missing = NEW_EXPECTED - seeds + assert not missing, ( + "current core does not seed the F4 entry points: " + f"{sorted(missing)}\nseeded: {sorted(seeds)}" + ) + + +def test_cbv_helper_is_never_seeded_by_the_current_core(): + """``get_queryset`` is a CBV helper, not an HTTP dispatch method. + + Seeding it would inflate the reachability root set with non-entry points. + """ + repo = _write_fixture_repo() + seeds = _run_pipeline(CORE_ROOT, repo) + + leaked = NEVER_SEEDED & seeds + assert not leaked, f"non-entry-point helper(s) seeded: {sorted(leaked)}" diff --git a/libs/openant-core/tests/parsers/zig/test_zig_alias_CORRECT_setunion.py b/libs/openant-core/tests/parsers/zig/test_zig_alias_CORRECT_setunion.py index 043c6777..f44a7352 100644 --- a/libs/openant-core/tests/parsers/zig/test_zig_alias_CORRECT_setunion.py +++ b/libs/openant-core/tests/parsers/zig/test_zig_alias_CORRECT_setunion.py @@ -23,6 +23,7 @@ import importlib.util import os import sys +from pathlib import Path import pytest @@ -30,7 +31,7 @@ # (`utilities.file_io`, `tree_sitter_zig`). _CORE_ROOT = os.environ.get( "OPENANT_CORE_ROOT", - "/Users/gadievron/Documents/ClaudeNew/OpenAnt/new-bugs-2/OpenAnt/libs/openant-core", + str(Path(__file__).resolve().parent.parent.parent.parent), ) if _CORE_ROOT not in sys.path: sys.path.insert(0, _CORE_ROOT) diff --git a/libs/openant-core/tests/parsers/zig/test_zig_alias_index_recursion_depth_bound.py b/libs/openant-core/tests/parsers/zig/test_zig_alias_index_recursion_depth_bound.py index c91835b8..f5012236 100644 --- a/libs/openant-core/tests/parsers/zig/test_zig_alias_index_recursion_depth_bound.py +++ b/libs/openant-core/tests/parsers/zig/test_zig_alias_index_recursion_depth_bound.py @@ -36,12 +36,13 @@ import importlib.util import os import sys +from pathlib import Path # openant-core root must be importable for the module's own imports # (`utilities.file_io`, `tree_sitter_zig`). _CORE_ROOT = os.environ.get( "OPENANT_CORE_ROOT", - "/Users/gadievron/Documents/ClaudeNew/OpenAnt/new-bugs-2/OpenAnt/libs/openant-core", + str(Path(__file__).resolve().parent.parent.parent.parent), ) if _CORE_ROOT not in sys.path: sys.path.insert(0, _CORE_ROOT) diff --git a/libs/openant-core/tests/test_F3_verdict_taxonomy_shared_constant.py b/libs/openant-core/tests/test_F3_verdict_taxonomy_shared_constant.py index 9f0a0abb..abdfe980 100644 --- a/libs/openant-core/tests/test_F3_verdict_taxonomy_shared_constant.py +++ b/libs/openant-core/tests/test_F3_verdict_taxonomy_shared_constant.py @@ -16,13 +16,14 @@ GREEN on the PATCHED tree: the module exists and the invariant holds, and the behavior change (bypassable/error now disclosure-eligible) is asserted. -The target tree is chosen via the OPENANT_ROOT env var (falls back to the -scratchpad impl-core copy). Run: +The target tree is chosen via the OPENANT_ROOT env var (defaults to this +repo's own openant-core). Run: OPENANT_ROOT=/path/to/patched/tree pytest F3-...test.py """ import importlib +from contextlib import contextmanager import os import re import sys @@ -30,23 +31,55 @@ import pytest -_DEFAULT_ROOT = ( - "/private/tmp/claude-501/" - "-Users-gadievron-Documents-ClaudeNew-OpenAnt-new-bugs-2/" - "e77a0496-1f59-4f65-80e9-fa508d40fa3c/scratchpad/impl-core" -) +# Default to THIS repo's core. The previous default was an absolute path into +# another machine-session's scratchpad; where that directory still happened to +# exist, the whole suite silently asserted against a stale, different tree +# (its core/ had no verdict_taxonomy at all, so the consumer check reported +# reporter.py as missing DISCLOSURE_ELIGIBLE when in fact it imports it). +_DEFAULT_ROOT = str(Path(__file__).resolve().parent.parent) ROOT = Path(os.environ.get("OPENANT_ROOT", _DEFAULT_ROOT)).resolve() +@contextmanager +def _isolated_core_namespace(): + """Import ``core.*`` from ROOT without leaking the eviction into the session. + + The eviction itself is necessary: pointing OPENANT_ROOT at a different tree + must re-import rather than reuse a cached copy. What was missing is putting + ``sys.modules`` and ``sys.path`` BACK afterwards. + + Leaking them is not a tidiness issue. Any test collected after this one that + bound ``import core.X as m`` at collection time and then monkeypatched + ``m.attr`` was silently defeated: the code under test re-imported ``core.X`` + and got a DIFFERENT module object, so the patch applied to an orphan. That + is what made ``test_enhance_limit`` bypass its stub and issue a live + Anthropic API call during an offline test run. + """ + saved_modules = { + name: module + for name, module in sys.modules.items() + if name == "core" or name.startswith("core.") + } + saved_path = list(sys.path) + try: + if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + for name in list(sys.modules): + if name == "core" or name.startswith("core."): + del sys.modules[name] + yield + finally: + for name in list(sys.modules): + if name == "core" or name.startswith("core."): + del sys.modules[name] + sys.modules.update(saved_modules) + sys.path[:] = saved_path + + def _load_taxonomy(): """Import core.verdict_taxonomy from the target tree in a clean namespace.""" - if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - # Drop any previously-imported copy so a different OPENANT_ROOT re-imports. - for name in list(sys.modules): - if name == "core" or name.startswith("core."): - del sys.modules[name] - return importlib.import_module("core.verdict_taxonomy") + with _isolated_core_namespace(): + return importlib.import_module("core.verdict_taxonomy") # --------------------------------------------------------------------------- @@ -157,3 +190,22 @@ def test_disclosure_consumers_reference_the_constant(): if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v"])) + + +def test_loading_taxonomy_does_not_leak_module_eviction(): + """The eviction must not outlive the helper. + + Regression lock for a cross-test hazard: leaking it silently defeated + monkeypatches in every test collected afterwards, which made + test_enhance_limit bypass its stub and issue a live API call. + """ + import core.parser_adapter as before + + _load_taxonomy() + + import core.parser_adapter as after + assert after is before, ( + "core.parser_adapter was re-imported as a different object after " + "_load_taxonomy() — the sys.modules eviction leaked" + ) + assert "core.parser_adapter" in sys.modules diff --git a/libs/openant-core/tests/test_json_corrector_output_cap.py b/libs/openant-core/tests/test_json_corrector_output_cap.py index 249e876b..7f91aece 100644 --- a/libs/openant-core/tests/test_json_corrector_output_cap.py +++ b/libs/openant-core/tests/test_json_corrector_output_cap.py @@ -29,11 +29,12 @@ import json import os import sys +from pathlib import Path _CORE_ROOT = os.environ.get( "OPENANT_CORE_ROOT", - "/Users/gadievron/Documents/ClaudeNew/OpenAnt/new-bugs-2/OpenAnt/libs/openant-core", + str(Path(__file__).resolve().parent.parent), ) if _CORE_ROOT not in sys.path: sys.path.insert(0, _CORE_ROOT) diff --git a/libs/openant-core/tests/test_no_hardcoded_paths.py b/libs/openant-core/tests/test_no_hardcoded_paths.py new file mode 100644 index 00000000..13e3bcc0 --- /dev/null +++ b/libs/openant-core/tests/test_no_hardcoded_paths.py @@ -0,0 +1,94 @@ +"""No test may hardcode an absolute path to somebody else's machine. + +Several tests were committed with a fallback like:: + + ROOT = os.environ.get("OPENANT_ROOT", "/Users//.../openant-core") + +which is worse than a missing file. When that directory happens to exist on the +machine running the suite — a stale scratchpad from an earlier session, say — +the test does not error. It silently asserts against a DIFFERENT, older tree +and reports its findings as if they were about this one. That is exactly how +`test_F3_verdict_taxonomy_shared_constant` came to report `core/reporter.py` as +missing a constant it has imported all along. + +The fallback must be derived from ``__file__`` so it always points at the tree +the test actually lives in. +""" + +import re +from pathlib import Path + +import pytest + +TESTS_ROOT = Path(__file__).parent + +# Absolute paths that belong to a specific machine/session rather than to the +# repository. A repo-relative path or a tmp_path fixture is always fine. +_MACHINE_PATH = re.compile( + r"""["'](?:/Users/[^"']+|/home/[^"']+|/private/tmp/[^"']+|/tmp/claude-[^"']+)["']""" +) + +# Lines that merely SHOW a command in a docstring are documentation, not +# behaviour. Only executable references matter. +_DOCSTRING_HINT = re.compile(r"^\s*(#|>>>|\$|PY=|\w+=)") + + +def _python_test_files(): + # This file is exempt from its own rule: it must contain examples of the + # forbidden pattern in order to verify the matcher still detects them + # (see test_the_guard_actually_matches_the_bad_pattern). + return sorted( + p for p in TESTS_ROOT.rglob("test_*.py") if p.name != Path(__file__).name + ) + + +def _offending_lines(path: Path) -> list[str]: + """Executable lines in *path* embedding a machine-specific absolute path.""" + offenders = [] + in_docstring = False + quote = None + + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = line.strip() + + # Track triple-quoted blocks so docstring examples are exempt. + if in_docstring: + if quote in stripped: + in_docstring = False + continue + for q in ('"""', "'''"): + if stripped.startswith(q): + # A one-line docstring opens and closes on the same line. + if stripped.count(q) == 1: + in_docstring, quote = True, q + break + + if in_docstring or _DOCSTRING_HINT.match(stripped): + continue + if _MACHINE_PATH.search(line): + offenders.append(f"{path.relative_to(TESTS_ROOT)}:{lineno}: {stripped[:110]}") + + return offenders + + +@pytest.mark.parametrize( + "test_file", _python_test_files(), ids=lambda p: str(p.relative_to(TESTS_ROOT)) +) +def test_no_machine_specific_absolute_paths(test_file): + offenders = _offending_lines(test_file) + assert not offenders, ( + "test hardcodes a machine-specific absolute path:\n " + + "\n ".join(offenders) + + "\n\nDerive it from __file__ instead, e.g.\n" + " _DEFAULT_ROOT = str(Path(__file__).resolve().parent.parent)\n" + "A stale path that still exists on disk makes the test assert against " + "the WRONG tree instead of failing loudly." + ) + + +def test_the_guard_actually_matches_the_bad_pattern(): + """Guard against the guard silently going stale.""" + assert _MACHINE_PATH.search('ROOT = "/Users/someone/repo/core"') + assert _MACHINE_PATH.search('X = "/private/tmp/claude-501/scratch/impl-core"') + assert not _MACHINE_PATH.search('ROOT = Path(__file__).parent.parent') + assert not _MACHINE_PATH.search('p = tmp_path / "dataset.json"') diff --git a/libs/openant-core/tests/test_verifier_casing_ingestion_normalize.py b/libs/openant-core/tests/test_verifier_casing_ingestion_normalize.py index e4e1014e..20817ab9 100644 --- a/libs/openant-core/tests/test_verifier_casing_ingestion_normalize.py +++ b/libs/openant-core/tests/test_verifier_casing_ingestion_normalize.py @@ -32,11 +32,12 @@ import os import sys +from pathlib import Path _CORE_ROOT = os.environ.get( "OPENANT_CORE_ROOT", - "/Users/gadievron/Documents/ClaudeNew/OpenAnt/new-bugs-2/OpenAnt/libs/openant-core", + str(Path(__file__).resolve().parent.parent), ) if _CORE_ROOT not in sys.path: sys.path.insert(0, _CORE_ROOT) From f2af16c93784060ea8557f756289e4e59718a460 Mon Sep 17 00:00:00 2001 From: Gadi Evron Date: Mon, 20 Jul 2026 22:41:56 +0300 Subject: [PATCH 02/44] feat: multi-language scanning and repository-supplied threat models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two features that share the same integration points (core/scanner.py, openant/cli.py, prompts/) and so land together; splitting them further would require hunk-level surgery on those files for no reviewer benefit. === Multi-language === detect_language() counted source files and returned max(counts), discarding the rest — a 60% Go / 40% TypeScript repo was scanned as Go and the absence of the TypeScript was never reported anywhere. config/languages.json is now the single source of truth. Four places independently described the supported set and had drifted: the config, the parser dispatch chain, argparse choices (twice), and Go flag help in three commands — scan.go, parse.go and README.md had all silently fallen behind when Zig was added. Everything now derives from the config, with consistency tests pinning each description to it. Resolution searches an env override then upward from module and CWD and DEGRADES rather than raising: supported_languages() runs during argparse construction, so a missing config previously took down `openant --help` entirely. parse_repository_multi() parses each selected language into //, because every parser writes the SAME seven flat filenames into whatever directory it is handed — two languages sharing one leaves exactly one survivor. dataset_merge.py merges them so every later stage still runs ONCE. Call graphs are NOT merged: no parser emits cross-language edges, so a union would assert connectivity that does not exist; call_graphs.json indexes them, built by probing the filesystem rather than a hardcoded list. Partial success is data, not exceptions — one broken toolchain must not cost every other language. The catch list is narrow so KeyboardInterrupt and MemoryError still abort. Sequential by design: cost tracking deltas against a process-global tracker, the Python parser mutates sys.path in-process, and six concurrent parsers on a monorepo is a realistic OOM. Excluded languages are reported loudly — stderr, step report, and JSON envelope. Measured, the default threshold silently dropped a PHP file holding a real path traversal to save 0.13s; for a security scanner a silent skip is a silently missed vulnerability class. === Threat models === A repository can commit OPENANT.THREATMODEL.md declaring its own free-form classification, components, attacker profiles, three-level input trust, what is and is not a vulnerability, and impact. When present it replaces the built-in application context entirely. The built-in model had four application types and ONE hardcoded attacker ("an attacker on the internet with a browser and nothing else"), with a single boolean deciding whether local-only findings counted. A deployment orchestrator whose real threat is "a developer with commit access to a watched manifest repo, no shell on the host" could not be expressed at all. Declared profiles now replace that persona in BOTH analysis stages, each with explicit capabilities and — load-bearing — explicit CANNOT limits. load_threat_model returns None only when the file is ABSENT; a malformed file RAISES, and the scanner calls it outside the context step's warn-and-continue handler on purpose. Degrading a malformed model into a default web_app context would silently apply the wrong security model to every finding. The file is attacker-authored when scanning third-party code, so it is guarded before opening: lstat rejects symlinks and non-regular files (a FIFO hung the scanner indefinitely — confirmed) and caps size at 1 MiB. `openant threat-model ` generates one; --validate-only makes no LLM call. context/THREAT_MODEL_AUTHORITY_DESIGN.md records the unsolved risk: the danger is not prompt injection but that the audited repository is granted authority over its own threat model. === Bugs found by adversarial review and fixed here === - Python and Ruby multi-file units were analysed WHOLESALE. Parsers emit the boundary marker in each language's comment syntax (`#` for Python/Ruby) but all four consumers matched `//` literally, so the split never fired and the model was handed the entire concatenation as the target function with the "do NOT analyze" context section silently dropped. Verified by isolating control: 3 function definitions inside the ANALYZE-ONLY block before, 1 after. - Entry-point seeds computed across ALL languages were passed into EACH per-language reachability filter, defeating its empty-seed blackout guard — every unit of any language lacking its own entry points was silently dropped while the scan reported success. - `--languages python` parsed the dominant language instead. - `--languages go` on a repo with no Go silently scanned Python. - Code fences were keyed by the scan-wide language, so a .ts file in a "javascript" scan was fenced as ```javascript. - The dynamic tester defaulted to "Python" for any unmapped language; it now resolves per finding and SKIPS untemplated ones, with SKIPPED registered as a first-class status so skipped findings are not invisible. === Documentation === OPENANT_THREATMODEL_TEMPLATE.md listed four mitigations for the accepted prompt-injection gap in the present tense. None existed — worse than an undocumented gap, because a reviewer would approve the risky configuration on the strength of controls that are not there. They are now an unchecked TODO list. internal/languages/registry.go claimed a shared golden fixture pinning the two detectors together; no such fixture exists. LanguageSelection.excluded documented itself as surfaced while being discarded; it is now genuinely surfaced. internal notes.md records the request, plan, result and lessons. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 1 + apps/openant-cli/cmd/init.go | 129 +--- apps/openant-cli/cmd/parse.go | 3 +- apps/openant-cli/cmd/scan.go | 39 +- .../internal/languages/registry.go | 235 ++++++ .../internal/languages/registry_test.go | 169 +++++ config/languages.json | 63 ++ libs/openant-core/CLAUDE.md | 2 +- libs/openant-core/CURRENT_IMPLEMENTATION.md | 2 +- libs/openant-core/DOCUMENTATION.md | 4 +- libs/openant-core/OPENANT.md | 4 +- libs/openant-core/PIPELINE_MANUAL.md | 2 +- libs/openant-core/README.md | 2 +- .../context/OPENANT_THREATMODEL_TEMPLATE.md | 439 +++++++++++ .../context/THREAT_MODEL_AUTHORITY_DESIGN.md | 176 +++++ .../context/application_context.py | 50 +- libs/openant-core/context/threat_model.py | 699 ++++++++++++++++++ .../context/threat_model_agent.py | 202 +++++ libs/openant-core/core/dataset_merge.py | 228 ++++++ libs/openant-core/core/file_boundary.py | 108 +++ libs/openant-core/core/language_registry.py | 259 +++++++ libs/openant-core/core/language_selection.py | 199 +++++ libs/openant-core/core/parser_adapter.py | 652 ++++++++-------- libs/openant-core/core/reporter.py | 30 +- libs/openant-core/core/scanner.py | 292 +++++++- libs/openant-core/core/schemas.py | 60 ++ libs/openant-core/openant/cli.py | 355 ++++++++- .../prompts/threat_model_render.py | 152 ++++ .../prompts/verification_prompts.py | 48 +- .../prompts/vulnerability_analysis.py | 35 +- libs/openant-core/pytest.ini | 2 + .../fixtures/sample_multilang_repo/app.py | 23 + .../fixtures/sample_multilang_repo/db.py | 25 + .../fixtures/sample_multilang_repo/utils.py | 11 + .../sample_multilang_repo/web/src/app.js | 24 + .../sample_multilang_repo/web/src/db.js | 29 + .../sample_multilang_repo/web/src/utils.js | 12 + .../test_application_context_backcompat.py | 136 ++++ .../tests/test_cli_multilang_flags.py | 152 ++++ libs/openant-core/tests/test_dataset_merge.py | 339 +++++++++ .../tests/test_detect_languages.py | 121 +++ .../tests/test_dynamic_tester_language.py | 160 ++++ libs/openant-core/tests/test_file_boundary.py | 137 ++++ .../tests/test_language_registry.py | 349 +++++++++ .../test_language_registry_resolution.py | 79 ++ .../tests/test_language_selection.py | 157 ++++ .../test_multilang_critical_regressions.py | 129 ++++ libs/openant-core/tests/test_parse_multi.py | 301 ++++++++ .../tests/test_parser_adapter_timeout.py | 23 +- .../tests/test_parser_registry.py | 150 ++++ .../openant-core/tests/test_reporter_fence.py | 75 ++ .../tests/test_scanner_multilang.py | 184 +++++ .../tests/test_scanner_refilter_loop.py | 124 ++++ .../test_scanner_refilter_loop_executes.py | 145 ++++ .../tests/test_scanner_refilter_multilang.py | 97 +++ .../test_scanner_threat_model_integration.py | 157 ++++ .../tests/test_schemas_multilang.py | 113 +++ .../tests/test_threat_model_agent.py | 200 +++++ .../tests/test_threat_model_hardening.py | 134 ++++ .../tests/test_threat_model_prompts.py | 218 ++++++ .../tests/test_threat_model_schema.py | 444 +++++++++++ .../test_threshold_exclusions_are_loud.py | 119 +++ .../utilities/agentic_enhancer/agent.py | 6 +- .../utilities/dynamic_tester/__init__.py | 46 +- .../utilities/dynamic_tester/__main__.py | 2 +- .../utilities/dynamic_tester/models.py | 7 +- .../utilities/dynamic_tester/reporter.py | 4 +- .../dynamic_tester/test_generator.py | 44 +- libs/openant-core/validate_dataset_schema.py | 4 +- 69 files changed, 8525 insertions(+), 596 deletions(-) create mode 100644 apps/openant-cli/internal/languages/registry.go create mode 100644 apps/openant-cli/internal/languages/registry_test.go create mode 100644 libs/openant-core/context/OPENANT_THREATMODEL_TEMPLATE.md create mode 100644 libs/openant-core/context/THREAT_MODEL_AUTHORITY_DESIGN.md create mode 100644 libs/openant-core/context/threat_model.py create mode 100644 libs/openant-core/context/threat_model_agent.py create mode 100644 libs/openant-core/core/dataset_merge.py create mode 100644 libs/openant-core/core/file_boundary.py create mode 100644 libs/openant-core/core/language_registry.py create mode 100644 libs/openant-core/core/language_selection.py create mode 100644 libs/openant-core/prompts/threat_model_render.py create mode 100644 libs/openant-core/tests/fixtures/sample_multilang_repo/app.py create mode 100644 libs/openant-core/tests/fixtures/sample_multilang_repo/db.py create mode 100644 libs/openant-core/tests/fixtures/sample_multilang_repo/utils.py create mode 100644 libs/openant-core/tests/fixtures/sample_multilang_repo/web/src/app.js create mode 100644 libs/openant-core/tests/fixtures/sample_multilang_repo/web/src/db.js create mode 100644 libs/openant-core/tests/fixtures/sample_multilang_repo/web/src/utils.js create mode 100644 libs/openant-core/tests/test_application_context_backcompat.py create mode 100644 libs/openant-core/tests/test_cli_multilang_flags.py create mode 100644 libs/openant-core/tests/test_dataset_merge.py create mode 100644 libs/openant-core/tests/test_detect_languages.py create mode 100644 libs/openant-core/tests/test_dynamic_tester_language.py create mode 100644 libs/openant-core/tests/test_file_boundary.py create mode 100644 libs/openant-core/tests/test_language_registry.py create mode 100644 libs/openant-core/tests/test_language_registry_resolution.py create mode 100644 libs/openant-core/tests/test_language_selection.py create mode 100644 libs/openant-core/tests/test_multilang_critical_regressions.py create mode 100644 libs/openant-core/tests/test_parse_multi.py create mode 100644 libs/openant-core/tests/test_parser_registry.py create mode 100644 libs/openant-core/tests/test_reporter_fence.py create mode 100644 libs/openant-core/tests/test_scanner_multilang.py create mode 100644 libs/openant-core/tests/test_scanner_refilter_loop.py create mode 100644 libs/openant-core/tests/test_scanner_refilter_loop_executes.py create mode 100644 libs/openant-core/tests/test_scanner_refilter_multilang.py create mode 100644 libs/openant-core/tests/test_scanner_threat_model_integration.py create mode 100644 libs/openant-core/tests/test_schemas_multilang.py create mode 100644 libs/openant-core/tests/test_threat_model_agent.py create mode 100644 libs/openant-core/tests/test_threat_model_hardening.py create mode 100644 libs/openant-core/tests/test_threat_model_prompts.py create mode 100644 libs/openant-core/tests/test_threat_model_schema.py create mode 100644 libs/openant-core/tests/test_threshold_exclusions_are_loud.py diff --git a/README.md b/README.md index 860589cc..fda64b42 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ To submit your repo for scanning: - C/C++ (beta) - PHP (beta) - Ruby (beta) +- Zig (beta) ## Credits diff --git a/apps/openant-cli/cmd/init.go b/apps/openant-cli/cmd/init.go index aa0eb0f1..8235cf6d 100644 --- a/apps/openant-cli/cmd/init.go +++ b/apps/openant-cli/cmd/init.go @@ -1,9 +1,7 @@ package cmd import ( - "encoding/json" "fmt" - "io/fs" "os" "os/exec" "path/filepath" @@ -11,6 +9,7 @@ import ( "github.com/knostic/open-ant-cli/internal/config" "github.com/knostic/open-ant-cli/internal/git" + "github.com/knostic/open-ant-cli/internal/languages" "github.com/knostic/open-ant-cli/internal/output" "github.com/spf13/cobra" ) @@ -47,7 +46,7 @@ var ( ) func init() { - initCmd.Flags().StringVarP(&initLanguage, "language", "l", "", "Language to analyze: python, javascript, go, c, ruby, php, zig, auto (auto = experimental dominance heuristic; see #61)") + initCmd.Flags().StringVarP(&initLanguage, "language", "l", "", languages.FlagHelp()) initCmd.Flags().StringVar(&initCommit, "commit", "", "Specific commit SHA (default: HEAD)") initCmd.Flags().StringVar(&initName, "name", "", "Override project name (default: derived from URL/path)") initCmd.Flags().BoolVar(&initFull, "full", false, "Force full scan (rejects --incremental/--diff-base/--pr)") @@ -136,7 +135,7 @@ func runInit(cmd *cobra.Command, args []string) { // Auto-detect language if not specified if initLanguage == "" || initLanguage == "auto" { fmt.Fprintf(os.Stderr, "Auto-detecting language...\n") - detected, err := detectLanguage(repoPath) + detected, err := languages.DetectLanguage(repoPath) if err != nil { output.PrintError(fmt.Sprintf("Language auto-detection failed: %s\nSpecify manually with -l/--language", err)) os.Exit(1) @@ -242,128 +241,6 @@ func runInit(cmd *cobra.Command, args []string) { fmt.Println() } -// languagesConfig is the structure of config/languages.json. -type languagesConfig struct { - SkipDirs []string `json:"skip_dirs"` - Extensions map[string]string `json:"extensions"` -} - -// findLanguagesConfig locates config/languages.json by walking up from the -// executable path and then the current working directory. -func findLanguagesConfig() (string, error) { - rel := filepath.Join("config", "languages.json") - - // Strategy 1: walk up from the executable. - if exePath, err := os.Executable(); err == nil { - exePath, _ = filepath.EvalSymlinks(exePath) - dir := filepath.Dir(exePath) - for range 6 { - candidate := filepath.Join(dir, rel) - if info, err := os.Stat(candidate); err == nil && !info.IsDir() { - return candidate, nil - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - } - - // Strategy 2: walk up from CWD. - if cwd, err := os.Getwd(); err == nil { - dir := cwd - for range 6 { - candidate := filepath.Join(dir, rel) - if info, err := os.Stat(candidate); err == nil && !info.IsDir() { - return candidate, nil - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - } - - return "", fmt.Errorf("could not find config/languages.json from executable or working directory") -} - -// loadLanguagesConfig loads the shared language detection config. -func loadLanguagesConfig() (*languagesConfig, error) { - path, err := findLanguagesConfig() - if err != nil { - return nil, err - } - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read %s: %w", path, err) - } - var cfg languagesConfig - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", path, err) - } - return &cfg, nil -} - -// detectLanguage walks a repository and returns the dominant language by file count. -// Extension mappings and skip directories are loaded from config/languages.json -// (shared with libs/openant-core/core/parser_adapter.py::detect_language()). -func detectLanguage(repoPath string) (string, error) { - cfg, err := loadLanguagesConfig() - if err != nil { - return "", fmt.Errorf("failed to load language config: %w", err) - } - - skipDirs := make(map[string]bool, len(cfg.SkipDirs)) - for _, d := range cfg.SkipDirs { - skipDirs[d] = true - } - - counts := make(map[string]int) - - err = filepath.WalkDir(repoPath, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return nil // skip inaccessible paths - } - if d.IsDir() { - if skipDirs[d.Name()] { - return filepath.SkipDir - } - return nil - } - - ext := strings.ToLower(filepath.Ext(d.Name())) - if lang, ok := cfg.Extensions[ext]; ok { - counts[lang]++ - } - return nil - }) - if err != nil { - return "", fmt.Errorf("failed to walk repository: %w", err) - } - - // Find the dominant language - bestLang := "" - bestCount := 0 - for lang, count := range counts { - if count > bestCount { - bestCount = count - bestLang = lang - } - } - - if bestLang == "" { - return "", fmt.Errorf( - "no supported source files found in %s. "+ - "Supported languages: Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, Zig", - repoPath, - ) - } - - return bestLang, nil -} - // resolveLocalCommit determines the commit SHA to record for a LOCAL git repo. // openant references local repos in place and never checks them out (unlike the // remote path, which runs `git checkout`), so the recorded commit MUST reflect diff --git a/apps/openant-cli/cmd/parse.go b/apps/openant-cli/cmd/parse.go index 78fa838a..f348132f 100644 --- a/apps/openant-cli/cmd/parse.go +++ b/apps/openant-cli/cmd/parse.go @@ -4,6 +4,7 @@ import ( "os" "strings" + "github.com/knostic/open-ant-cli/internal/languages" "github.com/knostic/open-ant-cli/internal/output" "github.com/knostic/open-ant-cli/internal/python" "github.com/spf13/cobra" @@ -34,7 +35,7 @@ var ( func init() { parseCmd.Flags().StringVarP(&parseOutput, "output", "o", "", "Output directory (default: project scan dir)") - parseCmd.Flags().StringVarP(&parseLanguage, "language", "l", "", "Language: python, javascript, go, c, ruby, php, auto") + parseCmd.Flags().StringVarP(&parseLanguage, "language", "l", "", languages.FlagHelp()) parseCmd.Flags().StringVar(&parseLevel, "level", "reachable", "Processing level: all, reachable, codeql, exploitable") parseCmd.Flags().StringVar(&parseDiffBase, "diff-base", "", "Incremental mode: tag units overlapping diff vs this ref") parseCmd.Flags().IntVar(&parsePR, "pr", 0, "Incremental mode against a GitHub PR number (mutex with --diff-base)") diff --git a/apps/openant-cli/cmd/scan.go b/apps/openant-cli/cmd/scan.go index 8883cde1..9108e50b 100644 --- a/apps/openant-cli/cmd/scan.go +++ b/apps/openant-cli/cmd/scan.go @@ -7,6 +7,7 @@ import ( "github.com/knostic/open-ant-cli/internal/checkpoint" "github.com/knostic/open-ant-cli/internal/config" "github.com/knostic/open-ant-cli/internal/git" + "github.com/knostic/open-ant-cli/internal/languages" "github.com/knostic/open-ant-cli/internal/output" "github.com/knostic/open-ant-cli/internal/python" "github.com/spf13/cobra" @@ -33,24 +34,24 @@ A final scan.report.json aggregates all step reports.`, } var ( - scanOutput string - scanLanguage string - scanLevel string - scanVerify bool - scanNoContext bool - scanNoEnhance bool - scanEnhanceMode string - scanNoReport bool - scanSkipDynamicTest bool - scanLimit int - scanLLMConfig string - scanWorkers int - scanBackoff int - scanFull bool - scanIncremental bool - scanDiffBase string - scanPR int - scanDiffScope string + scanOutput string + scanLanguage string + scanLevel string + scanVerify bool + scanNoContext bool + scanNoEnhance bool + scanEnhanceMode string + scanNoReport bool + scanSkipDynamicTest bool + scanLimit int + scanLLMConfig string + scanWorkers int + scanBackoff int + scanFull bool + scanIncremental bool + scanDiffBase string + scanPR int + scanDiffScope string scanLLMReachability bool scanLLMReachabilityMaxCodeBytes int ) @@ -64,7 +65,7 @@ func init() { // same knobs. func registerScanFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&scanOutput, "output", "o", "", "Output directory (default: project scan dir or temp dir)") - cmd.Flags().StringVarP(&scanLanguage, "language", "l", "", "Language: python, javascript, go, c, ruby, php, auto") + cmd.Flags().StringVarP(&scanLanguage, "language", "l", "", languages.FlagHelp()) cmd.Flags().StringVar(&scanLevel, "level", "reachable", "Processing level: all, reachable, codeql, exploitable") cmd.Flags().BoolVar(&scanVerify, "verify", false, "Enable Stage 2 attacker simulation") cmd.Flags().BoolVar(&scanNoContext, "no-context", false, "Skip application context generation") diff --git a/apps/openant-cli/internal/languages/registry.go b/apps/openant-cli/internal/languages/registry.go new file mode 100644 index 00000000..31951bf0 --- /dev/null +++ b/apps/openant-cli/internal/languages/registry.go @@ -0,0 +1,235 @@ +// Package languages is the Go-side reader for config/languages.json, the +// single source of truth for which languages OpenAnt supports. +// +// This package exists so that flag help text is DERIVED from config rather +// than hardcoded. Previously each of cmd/init.go, cmd/scan.go and cmd/parse.go +// carried its own literal list, and scan.go/parse.go silently fell behind when +// Zig was added — nothing failed, so nobody noticed. +// +// The Python side reads the same file via libs/openant-core/core/language_registry.py. +// +// NOTE: the two detectors are NOT yet pinned to each other by a shared +// fixture. Each has its own tests over its own temp trees, so a semantic +// divergence (skip-dir pruning, case-folding, tie-breaking) would not be +// caught. A cross-language golden fixture is the missing control here. +package languages + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// parserSpec mirrors the per-language "parser" object in config/languages.json. +type parserSpec struct { + Mode string `json:"mode"` + Script string `json:"script"` + Bootstrap string `json:"bootstrap"` +} + +// languageSpec mirrors one entry of the "languages" object. +type languageSpec struct { + Extensions []string `json:"extensions"` + Parser parserSpec `json:"parser"` + DockerTemplate *string `json:"docker_template"` + Enabled bool `json:"enabled"` +} + +// Config is the parsed form of config/languages.json. +// +// SkipDirs and Extensions are the legacy flat maps, kept byte-compatible +// because both this reader and the Python one consume them. Languages is the +// richer per-language block; a Python-side consistency test asserts the flat +// Extensions map stays exactly the union of the per-language lists. +type Config struct { + SkipDirs []string `json:"skip_dirs"` + Extensions map[string]string `json:"extensions"` + Languages map[string]languageSpec `json:"languages"` +} + +// FindConfig locates config/languages.json by walking up from the executable +// path and then the current working directory. +func FindConfig() (string, error) { + rel := filepath.Join("config", "languages.json") + + // Strategy 1: walk up from the executable. + if exePath, err := os.Executable(); err == nil { + exePath, _ = filepath.EvalSymlinks(exePath) + dir := filepath.Dir(exePath) + for range 6 { + candidate := filepath.Join(dir, rel) + if info, err := os.Stat(candidate); err == nil && !info.IsDir() { + return candidate, nil + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + } + + // Strategy 2: walk up from CWD. + if cwd, err := os.Getwd(); err == nil { + dir := cwd + for range 6 { + candidate := filepath.Join(dir, rel) + if info, err := os.Stat(candidate); err == nil && !info.IsDir() { + return candidate, nil + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + } + + return "", fmt.Errorf("could not find config/languages.json from executable or working directory") +} + +// Load reads and parses the shared language config. +func Load() (*Config, error) { + path, err := FindConfig() + if err != nil { + return nil, err + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", path, err) + } + var cfg Config + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", path, err) + } + return &cfg, nil +} + +// Supported returns the enabled language names, sorted. +func Supported() ([]string, error) { + cfg, err := Load() + if err != nil { + return nil, err + } + names := make([]string, 0, len(cfg.Languages)) + for name, spec := range cfg.Languages { + if spec.Enabled { + names = append(names, name) + } + } + sort.Strings(names) + return names, nil +} + +// FlagHelp renders the --language flag help string from config. +// +// Every cobra command that exposes --language must call this rather than +// writing its own list. If the config cannot be read we degrade to a generic +// string instead of failing: flag registration happens during init and a hard +// error there would make the whole CLI unusable over a config problem that +// only affects help text. +func FlagHelp() string { + names, err := Supported() + if err != nil || len(names) == 0 { + return "Language to analyze (see config/languages.json), or auto to detect" + } + return fmt.Sprintf( + "Language: %s, auto (auto = experimental dominance heuristic; see #61)", + strings.Join(names, ", "), + ) +} + +// DetectLanguages walks a repository and returns the source-file count per +// language. +// +// This is the multi-language primitive; DetectLanguage wraps it for the +// single-language callers. Directories named in skip_dirs are PRUNED (not just +// filtered), which the Python implementation mirrors via os.walk so both sides +// agree on what "skip" means. +func DetectLanguages(repoPath string) (map[string]int, error) { + cfg, err := Load() + if err != nil { + return nil, fmt.Errorf("failed to load language config: %w", err) + } + + skipDirs := make(map[string]bool, len(cfg.SkipDirs)) + for _, d := range cfg.SkipDirs { + skipDirs[d] = true + } + + counts := make(map[string]int) + + err = filepath.WalkDir(repoPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // skip inaccessible paths + } + if d.IsDir() { + if skipDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + + ext := strings.ToLower(filepath.Ext(d.Name())) + if lang, ok := cfg.Extensions[ext]; ok { + counts[lang]++ + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk repository: %w", err) + } + + return counts, nil +} + +// Ranked returns languages ordered by descending file count, ties broken +// alphabetically. +// +// The tie-break is not cosmetic. Go map iteration order is randomized, so the +// previous "keep the first strictly-greater count" loop returned an ARBITRARY +// winner on a tie — and could disagree with Python's max() on the same repo, +// across runs. Sorting makes both sides deterministic and identical. +func Ranked(counts map[string]int) []string { + names := make([]string, 0, len(counts)) + for name := range counts { + names = append(names, name) + } + sort.Slice(names, func(i, j int) bool { + if counts[names[i]] != counts[names[j]] { + return counts[names[i]] > counts[names[j]] + } + return names[i] < names[j] + }) + return names +} + +// DetectLanguage returns the dominant language by file count. +// +// Behaviour is preserved from the original cmd/init.go implementation, except +// that ties are now resolved deterministically (see Ranked). +func DetectLanguage(repoPath string) (string, error) { + counts, err := DetectLanguages(repoPath) + if err != nil { + return "", err + } + + ranked := Ranked(counts) + if len(ranked) == 0 { + supported, sErr := Supported() + list := "Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, Zig" + if sErr == nil && len(supported) > 0 { + list = strings.Join(supported, ", ") + } + return "", fmt.Errorf( + "no supported source files found in %s. Supported languages: %s", + repoPath, list, + ) + } + + return ranked[0], nil +} diff --git a/apps/openant-cli/internal/languages/registry_test.go b/apps/openant-cli/internal/languages/registry_test.go new file mode 100644 index 00000000..2d0f4982 --- /dev/null +++ b/apps/openant-cli/internal/languages/registry_test.go @@ -0,0 +1,169 @@ +package languages + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The Go detector had no tests at all before this file — cmd/ has no +// init_test.go. That is how the tie-break nondeterminism below survived. + +func writeTree(t *testing.T, root string, files []string) { + t.Helper() + for _, rel := range files { + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(path, []byte("x"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } +} + +func TestSupportedMatchesConfig(t *testing.T) { + got, err := Supported() + if err != nil { + t.Fatalf("Supported() error: %v", err) + } + want := []string{"c", "go", "javascript", "php", "python", "ruby", "zig"} + if len(got) != len(want) { + t.Fatalf("Supported() = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Supported() = %v, want %v", got, want) + } + } +} + +func TestFlagHelpIsDerivedFromConfig(t *testing.T) { + help := FlagHelp() + for _, lang := range []string{"python", "javascript", "go", "c", "ruby", "php", "zig"} { + if !strings.Contains(help, lang) { + t.Errorf("FlagHelp() omits %q: %s", lang, help) + } + } + if !strings.Contains(help, "auto") { + t.Errorf("FlagHelp() omits auto: %s", help) + } +} + +func TestDetectLanguagesCountsPerLanguage(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, []string{ + "a.py", "b.py", "c.py", + "x.ts", "y.js", + "main.go", + }) + + counts, err := DetectLanguages(dir) + if err != nil { + t.Fatalf("DetectLanguages error: %v", err) + } + want := map[string]int{"python": 3, "javascript": 2, "go": 1} + if len(counts) != len(want) { + t.Fatalf("counts = %v, want %v", counts, want) + } + for lang, n := range want { + if counts[lang] != n { + t.Errorf("counts[%s] = %d, want %d", lang, counts[lang], n) + } + } +} + +func TestSkipDirsArePruned(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, []string{ + "app.py", + "node_modules/pkg/index.js", + "node_modules/pkg/deep/nested/more.js", + "vendor/lib.go", + ".git/hooks/thing.py", + }) + + counts, err := DetectLanguages(dir) + if err != nil { + t.Fatalf("DetectLanguages error: %v", err) + } + if counts["javascript"] != 0 { + t.Errorf("node_modules not pruned: javascript=%d", counts["javascript"]) + } + if counts["go"] != 0 { + t.Errorf("vendor not pruned: go=%d", counts["go"]) + } + if counts["python"] != 1 { + t.Errorf("python = %d, want 1 (.git must be pruned)", counts["python"]) + } +} + +// TestTieBreakIsDeterministic fails on the pre-refactor implementation. +// +// The old loop kept the first strictly-greater count while iterating a Go map, +// whose order is randomized per run. On a tie it returned an arbitrary winner — +// and could disagree with the Python detector on the same repo. +func TestTieBreakIsDeterministic(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, []string{"a.py", "b.py", "x.js", "y.js"}) + + first, err := DetectLanguage(dir) + if err != nil { + t.Fatalf("DetectLanguage error: %v", err) + } + for i := 0; i < 50; i++ { + got, err := DetectLanguage(dir) + if err != nil { + t.Fatalf("DetectLanguage error on run %d: %v", i, err) + } + if got != first { + t.Fatalf("nondeterministic tie-break: run 0 = %q, run %d = %q", first, i, got) + } + } + // Alphabetical on a tie, matching the Python side. + if first != "javascript" { + t.Errorf("tie between javascript and python resolved to %q, want %q", first, "javascript") + } +} + +func TestDetectLanguagePicksDominant(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, []string{"a.py", "b.py", "c.py", "x.js"}) + + got, err := DetectLanguage(dir) + if err != nil { + t.Fatalf("DetectLanguage error: %v", err) + } + if got != "python" { + t.Errorf("DetectLanguage = %q, want python", got) + } +} + +func TestEmptyRepoErrors(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, []string{"README.md", "Makefile"}) + + if _, err := DetectLanguage(dir); err == nil { + t.Fatal("expected an error for a repo with no supported source files") + } +} + +func TestRankedOrdersByCountThenName(t *testing.T) { + got := Ranked(map[string]int{"go": 2, "python": 5, "zig": 2, "c": 9}) + want := []string{"c", "python", "go", "zig"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("Ranked = %v, want %v", got, want) + } + } +} + +func TestUnreadableDirIsTolerated(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, []string{"a.py"}) + // A walk error on one entry must not abort the whole detection. + if _, err := DetectLanguages(filepath.Join(dir, "does-not-exist")); err != nil { + t.Fatalf("walking a missing dir should degrade, got: %v", err) + } +} diff --git a/config/languages.json b/config/languages.json index 7a99dded..8cfd5ce3 100644 --- a/config/languages.json +++ b/config/languages.json @@ -30,5 +30,68 @@ ".rake": "ruby", ".php": "php", ".zig": "zig" + }, + "languages": { + "python": { + "extensions": [".py"], + "parser": {"mode": "inprocess"}, + "fence": "python", + "docker_template": "python", + "enabled": true + }, + "javascript": { + "extensions": [".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs"], + "parser": { + "mode": "subprocess", + "script": "parsers/javascript/test_pipeline.py", + "bootstrap": "npm" + }, + "fence": {".ts": "typescript", ".tsx": "typescript", "*": "javascript"}, + "docker_template": "node", + "enabled": true + }, + "go": { + "extensions": [".go"], + "parser": {"mode": "subprocess", "script": "parsers/go/test_pipeline.py"}, + "fence": "go", + "docker_template": "go", + "enabled": true + }, + "c": { + "extensions": [".c", ".h", ".cpp", ".hpp", ".cc", ".cxx", ".hxx", ".hh"], + "parser": {"mode": "subprocess", "script": "parsers/c/test_pipeline.py"}, + "fence": { + ".cpp": "cpp", + ".hpp": "cpp", + ".cc": "cpp", + ".cxx": "cpp", + ".hxx": "cpp", + ".hh": "cpp", + "*": "c" + }, + "docker_template": null, + "enabled": true + }, + "ruby": { + "extensions": [".rb", ".rake"], + "parser": {"mode": "subprocess", "script": "parsers/ruby/test_pipeline.py"}, + "fence": "ruby", + "docker_template": "ruby", + "enabled": true + }, + "php": { + "extensions": [".php"], + "parser": {"mode": "subprocess", "script": "parsers/php/test_pipeline.py"}, + "fence": "php", + "docker_template": "php", + "enabled": true + }, + "zig": { + "extensions": [".zig"], + "parser": {"mode": "subprocess", "script": "parsers/zig/test_pipeline.py"}, + "fence": "zig", + "docker_template": null, + "enabled": true + } } } diff --git a/libs/openant-core/CLAUDE.md b/libs/openant-core/CLAUDE.md index 3c616653..439db55e 100644 --- a/libs/openant-core/CLAUDE.md +++ b/libs/openant-core/CLAUDE.md @@ -21,7 +21,7 @@ The symlink automatically picks up the new binary. Running `make install` would # Project Context -This is OpenAnt, a two-stage SAST tool using Claude for vulnerability analysis. Supports Python, JavaScript/TypeScript, and Go codebases with 4-level cost optimization. +This is OpenAnt, a two-stage SAST tool using Claude for vulnerability analysis. Supports Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, and Zig codebases with 4-level cost optimization. **Key files to read after context reset:** - `DOCUMENTATION.md` - **Start here** - Index of all documentation diff --git a/libs/openant-core/CURRENT_IMPLEMENTATION.md b/libs/openant-core/CURRENT_IMPLEMENTATION.md index f2524c3a..3102a484 100644 --- a/libs/openant-core/CURRENT_IMPLEMENTATION.md +++ b/libs/openant-core/CURRENT_IMPLEMENTATION.md @@ -121,7 +121,7 @@ prompts/ prompt_selector.py - Routes to vulnerability_analysis prompt ``` -**Note:** Both Stage 1 and Stage 2 prompts are language-agnostic - the same prompt is used for Python, JavaScript, and Go. +**Note:** Both Stage 1 and Stage 2 prompts are language-agnostic - the same prompt is used for every supported language. **Stage 1 Prompt Format:** ``` diff --git a/libs/openant-core/DOCUMENTATION.md b/libs/openant-core/DOCUMENTATION.md index 5f1f4346..b94bb0a3 100644 --- a/libs/openant-core/DOCUMENTATION.md +++ b/libs/openant-core/DOCUMENTATION.md @@ -58,9 +58,9 @@ OpenAnt documentation is organized into three tiers based on audience and purpos ### Key Facts About the Codebase - **8-Step Pipeline:** Parse → Generate Units → Entry-Point Filter → Application Context → Context Enhancement → Stage 1 Detection → Stage 2 Verification → Dynamic Testing -- **Language-Agnostic Prompts:** The same prompts are used for Python, JavaScript, and Go +- **Language-Agnostic Prompts:** The same prompts are used for every supported language - **Two-Stage Analysis:** Stage 1 detects vulnerabilities, Stage 2 uses attacker simulation to verify exploitability -- **Supported Languages:** Python, JavaScript/TypeScript, Go +- **Supported Languages:** Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, and Zig ### File Naming Conventions diff --git a/libs/openant-core/OPENANT.md b/libs/openant-core/OPENANT.md index 6b44bc3a..8bf2c263 100644 --- a/libs/openant-core/OPENANT.md +++ b/libs/openant-core/OPENANT.md @@ -1,6 +1,6 @@ # OpenAnt Architecture Documentation -OpenAnt is an LLM-powered Static Application Security Testing (SAST) tool that uses a two-stage pipeline for vulnerability analysis with 4-level cost optimization. Supports Python, JavaScript/TypeScript, and Go. +OpenAnt is an LLM-powered Static Application Security Testing (SAST) tool that uses a two-stage pipeline for vulnerability analysis with 4-level cost optimization. Supports Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, and Zig. ## Table of Contents @@ -176,7 +176,7 @@ Five categories capture the spectrum of security states: | `prompts/verification_prompts.py` | Stage 2 attacker simulation prompt | | `prompts/prompt_selector.py` | Routes to vulnerability_analysis prompt | -**Note:** Both Stage 1 and Stage 2 prompts are language-agnostic - the same prompt is used for Python, JavaScript/TypeScript, and Go. +**Note:** Both Stage 1 and Stage 2 prompts are language-agnostic - the same prompt is used for every supported language. ### Dynamic Tester diff --git a/libs/openant-core/PIPELINE_MANUAL.md b/libs/openant-core/PIPELINE_MANUAL.md index fe77b78f..5c70ac3d 100644 --- a/libs/openant-core/PIPELINE_MANUAL.md +++ b/libs/openant-core/PIPELINE_MANUAL.md @@ -36,7 +36,7 @@ OpenAnt is a vulnerability analysis tool using Claude. The name "two-stage" refe | 7 | **Stage 2: Verification** | No | Attacker simulation to confirm exploitability | | 8 | **Dynamic Testing** | No | Docker-isolated exploit testing (requires Docker) | -**Supported Languages:** Python, JavaScript/TypeScript, Go +**Supported Languages:** Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, and Zig **Two-Stage Analysis:** - **Stage 1** asks: "Is this code vulnerable?" diff --git a/libs/openant-core/README.md b/libs/openant-core/README.md index 9d466edf..71680264 100644 --- a/libs/openant-core/README.md +++ b/libs/openant-core/README.md @@ -2,7 +2,7 @@ **LLM-Powered Static Application Security Testing** -OpenAnt uses Claude to analyze code for security vulnerabilities through a two-stage pipeline: detection followed by verification. Features 4-level cost optimization with CodeQL integration. Supports Python, JavaScript/TypeScript, and Go. +OpenAnt uses Claude to analyze code for security vulnerabilities through a two-stage pipeline: detection followed by verification. Features 4-level cost optimization with CodeQL integration. Supports Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, and Zig. --- diff --git a/libs/openant-core/context/OPENANT_THREATMODEL_TEMPLATE.md b/libs/openant-core/context/OPENANT_THREATMODEL_TEMPLATE.md new file mode 100644 index 00000000..50b75c3d --- /dev/null +++ b/libs/openant-core/context/OPENANT_THREATMODEL_TEMPLATE.md @@ -0,0 +1,439 @@ +# OPENANT.THREATMODEL.md Template + +This file provides a **custom threat model** for OpenAnt vulnerability analysis. +Place it, named exactly `OPENANT.THREATMODEL.md`, in your repository root. + +It is the alternative to `OPENANT.md` / the built-in four-type classifier. Use it when +your application's attacker model does not fit `web_app` / `cli_tool` / `library` / +`agent_framework` — which is most non-trivial infrastructure. + +**`OPENANT.THREATMODEL.md` is deliberately NOT one of the `MANUAL_OVERRIDE_FILES`.** +It is consumed by its own code path (`context/threat_model.py`), so the built-in +application-type path and the threat-model path can be run side by side on the same +repository and compared. + +--- + +## Why a threat model instead of an application type + +The built-in path compresses your entire adversary model into one enum value plus one +boolean (`requires_remote_trigger`), which feeds two hardcoded personas: *"an attacker +on the internet with a browser and nothing else"* and *"a local user with shell +access"*. If neither describes your real adversary, every verdict inherits that error. + +A threat model instead states, explicitly and per-repository: + +- a **free-form classification** (no enum), +- **components** with **free-form component types** and an exposure level, +- **named attacker profiles** with explicit CAN and CANNOT capability lists, +- **input sources** with trust levels and which components handle them, +- what **IS** a vulnerability here and what is **NOT**, +- the concrete **impact** of a successful compromise. + +--- + +## Required heading skeleton + +A well-formed document has these headings, in this order: + +1. `## Purpose` +2. `## Architecture & Components` +3. `## Attacker Profiles` +4. `## Input Sources & Trust Levels` +5. `## What IS a Vulnerability` +6. `## What is NOT a Vulnerability` +7. `## Impact` +8. `## Machine-Readable Threat Model` + +The headings are for humans and for PR review. **The fenced JSON block (a `json` code fence) under +"Machine-Readable Threat Model" is the machine truth** and is what gets strictly +validated. Missing headings produce a warning; a malformed or invalid json block is a +hard error. + +The parser scans **every** JSON code fence in the file and picks the one whose object +declares `"schema": "openant-threat-model"`. Illustrative json elsewhere in your prose +(including everything in this template above the worked example) is ignored. + +--- + +## Schema v1 reference + +### Required top-level fields + +| Field | Type | Notes | +|-------|------|-------| +| `schema` | string | Must be exactly `"openant-threat-model"` | +| `schema_version` | integer | Must be `1`. Any other value is an error naming the supported versions | +| `classification` | string | **Free-form.** e.g. `"Kubernetes deployment orchestrator"` | +| `purpose` | string | 1–3 sentences on what the system does | +| `components` | object[] | See below. Must be non-empty | +| `attacker_profiles` | object[] | See below. Must be non-empty | +| `input_sources` | object | Map of source name → spec. Must be non-empty | +| `vulnerability_criteria` | string[] | What counts as a vulnerability here. Must be non-empty | +| `not_a_vulnerability` | string[] | May be empty, **but the key must be present** | +| `impact_statement` | string | What a successful compromise actually costs | + +### Optional top-level fields + +`architecture`, `intended_behaviors` (string[]), `security_model` (string), +`confidence` (0.0–1.0), `evidence` (string[]), `generated_by` (string). + +### `components[]` + +| Field | Type | Notes | +|-------|------|-------| +| `name` | string | Referenced by `input_sources[*].handled_by` | +| `paths` | string[] | Repo-relative paths/globs. Non-empty | +| `component_type` | string | **Free-form** — `"manifest watcher"`, `"reconciliation loop"`, `"admission webhook"` | +| `exposure` | enum | `remote` \| `local` \| `internal` | +| `description` | string | Optional | + +### `attacker_profiles[]` + +| Field | Type | Notes | +|-------|------|-------| +| `id` | string | Short stable id, referenced in reports | +| `description` | string | Who this actually is, in one sentence | +| `position` | enum | `remote` \| `adjacent` \| `local_user` \| `supply_chain` \| `insider` | +| `capabilities` | string[] | What they **CAN** do. Non-empty | +| `cannot` | string[] | What they **CANNOT** do. Non-empty — this is what kills false positives | +| `entry_via` | string[] | **Must name keys of `input_sources`.** Non-empty | +| `impact` | string | What they achieve if they win | + +### `input_sources` + +Map of source name → `{ "trust": ..., "description": ..., "handled_by": [...] }`. + +- `trust` — `untrusted` \| `semi_trusted` \| `trusted`, accepted case-insensitively. +- `description` — required. +- `handled_by` — optional; **each entry must name a declared component**. + +### Cross-reference rules (dangling references are errors) + +- every `attacker_profiles[*].entry_via` entry must name a key of `input_sources`; +- every `input_sources[*].handled_by` entry must name a `components[*].name`. + +### Derived legacy fields + +You do not write these; OpenAnt derives them so existing consumers keep working: + +- `application_type` = `"custom:" + slug(classification)` +- `trust_boundaries` = `{input source name: trust level}` +- `requires_remote_trigger` = any profile at `position: "remote"`, **or** any input + source marked `untrusted` + +### Validation behaviour + +Validation collects **every** violation and reports them together — you fix a +hand-written document in one pass, not one error per scan. + +**If `OPENANT.THREATMODEL.md` is absent, OpenAnt falls back to the built-in path. If it +is present but malformed, the scan FAILS LOUDLY.** This inverts the behaviour of +`OPENANT.md`, which degrades to a warning. The reason is blast radius: a broken +`OPENANT.md` costs you a better-than-default context, whereas a broken threat model +would silently analyse your repository under the default `web_app` attacker model +while producing a report that looks completely successful. + +--- + +## KNOWN GAP — this file is attacker-influenceable and is NOT prompt-injection-fenced + +**Read this before enabling threat models on repositories you do not control.** + +This file originates in the scanned repository. It is therefore +attacker-influenceable: whoever can land a commit in the target repository can write +its contents. Unlike scanned **source code**, which is wrapped in delimiters before +being placed in a prompt (`prompts/_fence.py`), the threat model's contents are +**NOT prompt-injection-fenced**. Its text — classifications, attacker descriptions, +`not_a_vulnerability` entries, and any prose the author chooses to put in a string +field — reaches the analysis model unfenced. + +A hostile repository can therefore ship a threat model that declares nothing to be a +vulnerability: an empty `vulnerability_criteria`, a `not_a_vulnerability` list that +covers the whole codebase, every input source marked `trusted`, or instructions +embedded in a description field. The scan will then report clean, and it will look +like a normal clean scan. + +**This is an accepted, documented risk, per an explicit user decision. It is not +fixed.** The mitigations are visibility, not prevention: + +**None of the following are implemented yet.** They are the mitigations this +gap REQUIRES before threat models should be trusted on repositories you do not +control. Listing them as if they existed would be worse than the gap itself — +a reviewer would approve the risky configuration on the strength of controls +that are not there: + +- [ ] record the file's SHA-256 in the scan report; +- [ ] render "context supplied by repo-controlled file" in the report header; +- [ ] warn loudly when a threat model marks *every* input source `trusted`; +- [ ] add a test named for this gap so it appears in test output. + +Until those exist, treat a repo-supplied threat model as advisory only. + rather than only in documentation. + +**Operational guidance:** treat `OPENANT.THREATMODEL.md` as you would treat a CI +configuration file committed by a third party. Review it in the diff. For repositories +you do not control, prefer the built-in application-type path, or supply your own +threat model out-of-band rather than trusting the one in the tree. + +--- + +## Complete worked example + +The example below is a case **the built-in four-type enum cannot express**: a +deployment orchestrator that watches a git repository of manifests and reconciles them +into a cluster. Its real adversary is *a developer with commit access to the watched +manifest repo who has no shell on the orchestrator*. That attacker is neither "an +attacker on the internet with a browser" nor "a local user with shell access", so +`web_app` over-flags and `cli_tool` under-flags. Note the `component_type` values +(`"manifest watcher"`, `"reconciliation loop"`, `"admission webhook"`) — none of which +any enum would contain — and the `cannot` lists, which are what suppress false +positives without suppressing the real bug class. + +# Threat Model: GitOps deployment orchestrator + +## Purpose + +Watches a git repository of Kubernetes manifests and continuously reconciles the +declared state into one or more target clusters. Renders templates, resolves secret +references, and applies the result via the Kubernetes API. + +## Architecture & Components + +Long-running controller. No inbound HTTP surface except a cluster-internal admission +webhook and a localhost health endpoint. + +- **manifest-watcher** (manifest watcher, exposure: internal) — `internal/gitwatch/` +- **template-renderer** (template engine, exposure: internal) — `internal/render/` +- **reconciler** (reconciliation loop, exposure: internal) — `internal/reconcile/` +- **admission-webhook** (admission webhook, exposure: local) — `cmd/webhook/` +- **secret-resolver** (secret backend client, exposure: internal) — `internal/secrets/` + +## Attacker Profiles + +### `manifest-committer` — Developer with commit access to the watched manifest repo, no shell on the orchestrator + +**Position:** supply_chain + +**CAN:** +- Commit arbitrary YAML to the watched manifest repository +- Choose template values, file paths and manifest field values freely +- Trigger a reconcile at will by pushing a commit +- Observe reconcile outcomes through the orchestrator's status conditions + +**CANNOT:** +- Execute shell commands on the orchestrator host +- Read the orchestrator's filesystem or environment directly +- Reach the secret backend directly (only via a manifest secret reference) +- Modify the orchestrator's own configuration or its RBAC binding + +**Enters via:** git_manifest_repo, template_values + +**Impact if successful:** Escalates from "may declare workloads" to arbitrary code execution inside the orchestrator's pod, which holds a cluster-admin-equivalent service account. + +### `cluster-tenant` — Namespaced tenant able to submit resources to the admission webhook + +**Position:** adjacent + +**CAN:** +- Submit arbitrary AdmissionReview payloads to the webhook +- Create resources in their own namespace + +**CANNOT:** +- Commit to the manifest repository +- Reach the reconciler or secret resolver directly + +**Enters via:** admission_review_payload + +**Impact if successful:** Denial of service on admissions cluster-wide, or bypass of a policy the webhook is meant to enforce. + +## Input Sources & Trust Levels + +- **git_manifest_repo** — `untrusted` — YAML manifests read from the watched repository (handled by: manifest-watcher, template-renderer) +- **template_values** — `untrusted` — Values files and inline template parameters supplied alongside manifests (handled by: template-renderer) +- **admission_review_payload** — `untrusted` — AdmissionReview objects POSTed by the API server on behalf of any cluster user (handled by: admission-webhook) +- **secret_backend_response** — `semi_trusted` — Secret material returned by the external secret backend (handled by: secret-resolver) +- **orchestrator_config** — `trusted` — Operator-supplied config file and flags, set at deploy time (handled by: reconciler) + +## What IS a Vulnerability + +- Template rendering that allows a manifest author to reach outside the template sandbox (function injection, arbitrary file read via template include, SSTI) +- Path traversal in manifest or values file resolution that reads files outside the checkout +- Any path by which a manifest field reaches a shell, exec, or plugin loader +- Secret material from the secret resolver being written into status, logs, or a rendered manifest visible to the manifest author +- Reconciler applying a manifest that escalates the orchestrator's own RBAC +- Unauthenticated or spoofable admission webhook requests, or a webhook panic that fails open +- Deserialization of manifest YAML into arbitrary Go types + +## What is NOT a Vulnerability + +- The orchestrator applying manifests to the cluster — that is the entire product +- The orchestrator holding a high-privilege service account — required by design, documented, and scoped by the operator at install time +- A manifest author declaring a workload with a privileged securityContext — the cluster's own admission policy governs that, not the orchestrator +- File writes inside the ephemeral checkout directory +- The operator-supplied config file controlling which repos are watched — trusted input, set by whoever deployed the orchestrator +- Resource exhaustion from a very large manifest repository — rate limited and bounded, and the manifest author already controls their own reconcile budget + +## Impact + +Compromise of the orchestrator yields the orchestrator's service account, which is +cluster-admin-equivalent on every target cluster it reconciles into. The realistic +worst case is a developer with commit access to one manifest repository pivoting to +full control of every cluster the orchestrator manages — a large privilege jump from +their intended authority, and the reason template-sandbox escapes are treated as +critical here even though they are "only" reachable from a trusted-ish developer. + +## Machine-Readable Threat Model + +```json +{ + "schema": "openant-threat-model", + "schema_version": 1, + "classification": "GitOps deployment orchestrator", + "purpose": "Watches a git repository of Kubernetes manifests and continuously reconciles the declared state into one or more target clusters.", + "architecture": "Long-running controller. No inbound HTTP surface except a cluster-internal admission webhook and a localhost health endpoint.", + "components": [ + { + "name": "manifest-watcher", + "paths": ["internal/gitwatch/"], + "component_type": "manifest watcher", + "exposure": "internal", + "description": "Clones and polls the watched manifest repository." + }, + { + "name": "template-renderer", + "paths": ["internal/render/"], + "component_type": "template engine", + "exposure": "internal", + "description": "Renders manifest templates against values files." + }, + { + "name": "reconciler", + "paths": ["internal/reconcile/"], + "component_type": "reconciliation loop", + "exposure": "internal", + "description": "Diffs rendered manifests against live cluster state and applies changes." + }, + { + "name": "admission-webhook", + "paths": ["cmd/webhook/"], + "component_type": "admission webhook", + "exposure": "local", + "description": "Cluster-internal HTTPS endpoint invoked by the API server." + }, + { + "name": "secret-resolver", + "paths": ["internal/secrets/"], + "component_type": "secret backend client", + "exposure": "internal", + "description": "Resolves secret references in manifests against an external backend." + } + ], + "attacker_profiles": [ + { + "id": "manifest-committer", + "description": "Developer with commit access to the watched manifest repo, no shell on the orchestrator", + "position": "supply_chain", + "capabilities": [ + "Commit arbitrary YAML to the watched manifest repository", + "Choose template values, file paths and manifest field values freely", + "Trigger a reconcile at will by pushing a commit", + "Observe reconcile outcomes through the orchestrator's status conditions" + ], + "cannot": [ + "Execute shell commands on the orchestrator host", + "Read the orchestrator's filesystem or environment directly", + "Reach the secret backend directly (only via a manifest secret reference)", + "Modify the orchestrator's own configuration or its RBAC binding" + ], + "entry_via": ["git_manifest_repo", "template_values"], + "impact": "Escalates from 'may declare workloads' to arbitrary code execution inside the orchestrator's pod, which holds a cluster-admin-equivalent service account." + }, + { + "id": "cluster-tenant", + "description": "Namespaced tenant able to submit resources to the admission webhook", + "position": "adjacent", + "capabilities": [ + "Submit arbitrary AdmissionReview payloads to the webhook", + "Create resources in their own namespace" + ], + "cannot": [ + "Commit to the manifest repository", + "Reach the reconciler or secret resolver directly" + ], + "entry_via": ["admission_review_payload"], + "impact": "Denial of service on admissions cluster-wide, or bypass of a policy the webhook is meant to enforce." + } + ], + "input_sources": { + "git_manifest_repo": { + "trust": "untrusted", + "description": "YAML manifests read from the watched repository.", + "handled_by": ["manifest-watcher", "template-renderer"] + }, + "template_values": { + "trust": "untrusted", + "description": "Values files and inline template parameters supplied alongside manifests.", + "handled_by": ["template-renderer"] + }, + "admission_review_payload": { + "trust": "untrusted", + "description": "AdmissionReview objects POSTed by the API server on behalf of any cluster user.", + "handled_by": ["admission-webhook"] + }, + "secret_backend_response": { + "trust": "semi_trusted", + "description": "Secret material returned by the external secret backend.", + "handled_by": ["secret-resolver"] + }, + "orchestrator_config": { + "trust": "trusted", + "description": "Operator-supplied config file and flags, set at deploy time.", + "handled_by": ["reconciler"] + } + }, + "vulnerability_criteria": [ + "Template rendering that allows a manifest author to reach outside the template sandbox (function injection, arbitrary file read via template include, SSTI)", + "Path traversal in manifest or values file resolution that reads files outside the checkout", + "Any path by which a manifest field reaches a shell, exec, or plugin loader", + "Secret material from the secret resolver being written into status, logs, or a rendered manifest visible to the manifest author", + "Reconciler applying a manifest that escalates the orchestrator's own RBAC", + "Unauthenticated or spoofable admission webhook requests, or a webhook panic that fails open", + "Deserialization of manifest YAML into arbitrary Go types" + ], + "not_a_vulnerability": [ + "The orchestrator applying manifests to the cluster - that is the entire product", + "The orchestrator holding a high-privilege service account - required by design, documented, and scoped by the operator at install time", + "A manifest author declaring a workload with a privileged securityContext - the cluster's own admission policy governs that, not the orchestrator", + "File writes inside the ephemeral checkout directory", + "The operator-supplied config file controlling which repos are watched - trusted input, set by whoever deployed the orchestrator", + "Resource exhaustion from a very large manifest repository - rate limited and bounded, and the manifest author already controls their own reconcile budget" + ], + "intended_behaviors": [ + "Applies arbitrary Kubernetes resources declared in the watched repository", + "Renders user-authored templates with user-authored values", + "Reads secret material from an external backend and injects it into applied manifests" + ], + "security_model": "Template rendering runs in a restricted function set; manifest paths are resolved against the checkout root; the webhook requires mTLS from the API server; the orchestrator's own RBAC is immutable at runtime.", + "impact_statement": "Compromise of the orchestrator yields a cluster-admin-equivalent service account on every target cluster it reconciles into. The realistic worst case is a developer with commit access to one manifest repository pivoting to full control of every managed cluster.", + "confidence": 0.9, + "evidence": [ + "README.md describes the GitOps reconcile loop", + "internal/render/ uses text/template with a custom function map", + "deploy/rbac.yaml grants cluster-admin to the orchestrator service account" + ], + "generated_by": "manual" +} +``` + +--- + +## Authoring checklist + +- [ ] Every `entry_via` names a key that exists in `input_sources` +- [ ] Every `handled_by` names a component that exists in `components` +- [ ] Every attacker profile has a **non-empty `cannot` list** — this is what prevents false positives +- [ ] `not_a_vulnerability` is present (empty list is allowed, omission is not) +- [ ] `component_type` values describe *your* architecture, not a generic category +- [ ] `classification` is specific enough that the derived `custom:` reads sensibly +- [ ] You have read the KNOWN GAP section above diff --git a/libs/openant-core/context/THREAT_MODEL_AUTHORITY_DESIGN.md b/libs/openant-core/context/THREAT_MODEL_AUTHORITY_DESIGN.md new file mode 100644 index 00000000..221c41d6 --- /dev/null +++ b/libs/openant-core/context/THREAT_MODEL_AUTHORITY_DESIGN.md @@ -0,0 +1,176 @@ +# Threat-model authority: the unsolved risk, and the proposed mitigation + +**Status:** design proposal, NOT implemented. Written 2026-07-19 after an +adversarial review of the threat-model wiring (`context/threat_model.py`, +`prompts/threat_model_render.py`, the Step-2 integration in `core/scanner.py`). + +**Audience:** whoever decides whether OpenAnt may trust a repo-supplied threat +model when scanning code the operator does not control. + +--- + +## 1. The problem, stated precisely + +`OPENANT.THREATMODEL.md` lives in the **scanned repository**. When OpenAnt +audits third-party code, that file is authored by the same party whose code is +under audit — potentially the attacker. Today, when present, it **replaces** +the built-in application context entirely: its attacker profiles replace the +hardcoded persona, and its `not_a_vulnerability` list is rendered into the +Stage-1 prompt under the heading *"These are NOT vulnerabilities here — do not +flag them"*. + +That is the whole feature working as designed. It is also the risk. + +### 1.1 The dangerous primitive is NOT prompt injection + +There is a known, separately-tracked gap: threat-model content reaches prompts +**without** the `prompts/_fence.py` fencing that scanned source code receives. +That gap is real and worth closing. + +**But it is not the main problem, and closing it does not make the design +safe.** A hostile repository does not need to inject anything. It can write a +schema-valid, well-formed threat model containing: + +```json +"not_a_vulnerability": [ + "All input handling in this repository is trusted by design" +], +"attacker_profiles": [ + {"id": "operator", "position": "local_user", + "cannot": ["send network requests", "supply untrusted input"], ...} +] +``` + +Every field is legal. Validation passes. The document simply *declares* the +attack surface out of existence, and OpenAnt faithfully relays that declaration +to the model as authoritative context. + +Fencing prevents a document from *escaping its container* and issuing +instructions. It does nothing about a document whose **legitimate content**, +used exactly as intended, suppresses findings. + +> The authority granted to the untrusted document is the vulnerability — +> not the channel it travels through. + +### 1.2 Why this is worse than a false negative + +A missed finding is a gap. A *suppressed* finding is a gap that looks like a +clean bill of health: the scan succeeds, reports zero vulnerabilities, and the +reason it found nothing is invisible in the output. + +--- + +## 2. Proposed mitigation: an operator-side immutable baseline + +**Principle:** a scanned repository may *narrow* scope; it may never *reduce* +scope below what the operator requires. + +The repo-supplied model becomes **advisory input constrained by an operator +policy**, rather than the authority. Concretely: + +### 2.1 An operator baseline that the repo cannot weaken + +The operator supplies a baseline (config file or CLI flag) declaring the +minimum that must always be analysed — e.g. "command execution, SQL injection, +path traversal and deserialization are ALWAYS in scope, whatever any repo +says". Merge rule: + +| Repo-supplied model says | Operator baseline says | Result | +|---|---|---| +| X is not a vulnerability | X is always in scope | **X stays in scope** (repo ignored, and the attempt is REPORTED) | +| X is not a vulnerability | (silent on X) | X excluded, recorded in the report | +| adds attacker profile P | — | P added | +| removes/narrows a baseline attacker | baseline requires it | baseline attacker retained | + +The merge is **monotonic in the safe direction**: repo input can only add +attackers, add criteria, and add components. Anything that *subtracts* from the +baseline is dropped and surfaced. + +### 2.2 Trust tiers, chosen by the operator — not the repo + +| Tier | Meaning | Default for | +|---|---|---| +| `trusted` | model is authoritative (current behaviour) | first-party repos you own | +| `advisory` | model adds context; baseline governs suppression | **default for third-party code** | +| `ignored` | model is read and reported, never applied | untrusted / adversarial scans | + +The repository must have no say in which tier it gets. `--threat-model-trust` +belongs to the operator invoking the scan. + +### 2.3 Suppression accounting + +Every finding suppressed *because* the repo said so must be recorded, not +silently dropped: + +```json +{"suppressed_by_threat_model": [ + {"unit": "pkg/manifest.py:apply", "criterion": "All input handling is trusted by design"} +]} +``` + +A reviewer must be able to answer "what would this scan have reported if I had +not trusted the repo's own threat model?" — and today they cannot. + +### 2.4 Anomaly detection on the model itself + +Cheap, high-signal heuristics that warrant a loud warning: + +- every input source marked `trusted` +- zero attacker profiles, or every profile's `cannot` list covering the + program's actual entry points +- `not_a_vulnerability` entries phrased as blanket categories rather than + specific behaviours +- a model whose git history shows it was added/edited in the same commit range + as the code being audited + +--- + +## 3. Supporting items (also unimplemented) + +From the same review, ordered by value: + +1. **Per-profile structured verification output.** Stage 2 is told to adopt each + profile "in turn", but nothing requires per-profile results. The model can + return one aggregate verdict without evidence every profile was considered. + Require a per-profile trace. +2. **Policy validation beyond schema.** Duplicate profile ids; `entry_via` + naming a nonexistent input source; `handled_by` naming a nonexistent + component; contradictory CAN/CANNOT; component paths outside the repo. + (Cross-reference validation partially exists — verify coverage.) +3. **Provenance in the report.** The model's SHA-256, its path, and the trust + tier applied, recorded in `pipeline_output.json` and rendered in the report + header. Three of the four mitigations still listed as unimplemented TODOs in + `OPENANT_THREATMODEL_TEMPLATE.md` are exactly this. +4. **Prompt fencing** for threat-model content. Worth doing — it closes the + injection channel — but see §1.1: it is not the fix for the authority + problem, and shipping it alone would create false assurance. + +--- + +## 4. What IS already implemented (do not re-do) + +- Symlink / FIFO / device rejection and a 1 MiB cap, checked via `lstat` + **before** opening (`context/threat_model.py`). A FIFO previously hung the + scanner indefinitely — confirmed empirically. +- Malformed model aborts the scan loudly rather than degrading to a default + context (`core/scanner.py`, Step 2). +- `--no-context` announces that it is discarding a committed threat model. +- `context_source` on `ScanResult` records `threat_model` / `generated` / `none`. +- Attacker profiles render into **both** Stage 1 and Stage 2. + +--- + +## 5. Recommendation + +Before OpenAnt is pointed at third-party code with threat models enabled: + +1. Implement §2.1 (baseline) and §2.2 (tiers), defaulting third-party scans to + `advisory`. +2. Implement §2.3 (suppression accounting). +3. Then §3.1–3.3. + +§3.4 (fencing) may land at any time but must not be described as resolving the +risk in §1. + +Until §2 exists, treat a repo-supplied threat model as safe **only** on +repositories you control. diff --git a/libs/openant-core/context/application_context.py b/libs/openant-core/context/application_context.py index 606a76d3..a2d7b8fd 100644 --- a/libs/openant-core/context/application_context.py +++ b/libs/openant-core/context/application_context.py @@ -129,7 +129,26 @@ class ApplicationContext: # Metadata confidence: float = 0.0 evidence: list[str] = field(default_factory=list) - source: str = "llm" # "llm", "manual", or "merged" + source: str = "llm" # "llm", "manual", "merged", or "threat_model" + + # --- Custom threat-model extension (schema v1, see context/threat_model.py) --- + # + # These are ALL optional with defaults, deliberately. The two deserialization + # sites in the codebase (core/analyzer.py, core/verifier.py) both go through + # ``load_context``, which is ``ApplicationContext(**data)``; ``save_context`` is + # a plain ``asdict``. Because every new field is defaulted, a pre-existing + # ``application_context.json`` written before this extension existed still loads + # unchanged, and the richer schema round-trips through save/load with no changes + # to either function. That is what lets the built-in "app type" arm and the + # custom threat-model arm be the *same* dataclass differing only in which JSON + # file is handed to the pipeline — the precondition for comparing them. + threat_model_version: int | None = None + classification: str | None = None + components: list = field(default_factory=list) + attacker_profiles: list = field(default_factory=list) + input_sources: dict = field(default_factory=dict) + vulnerability_criteria: list = field(default_factory=list) + impact_statement: str | None = None def __post_init__(self): """Validate application_type after initialization.""" @@ -137,12 +156,41 @@ def __post_init__(self): if self.source == "manual": return + # Skip validation for threat-model contexts. This is an EXPLICIT second + # branch rather than a widening of the ``source == "manual"`` bypass above, + # and the duplication is intentional. The two bypasses exist for unrelated + # reasons and must be able to change independently: + # + # * the manual bypass exists because an operator hand-writing OPENANT.md + # is trusted to name any type they like; + # * this bypass exists because a threat model's ``application_type`` is + # *derived*, not chosen — ``threat_model_to_context`` synthesizes + # ``"custom:" + slug(classification)`` from a free-form classification, + # which by construction can never be one of the four enum values. + # + # Folding them together would mean a future tightening of one silently + # loosens the other, and would also make a threat-model context + # indistinguishable from an operator override at the ``source`` field. + if self.threat_model_version is not None: + return + if not ApplicationType.is_supported(self.application_type): raise UnsupportedApplicationTypeError( self.application_type, self.evidence ) + def has_threat_model(self) -> bool: + """Whether this context was built from a custom threat model (schema v1+). + + The single branch predicate at every consumption site: prompt renderers, + the scanner's context step, and the A/B arm labelling. ``threat_model_version`` + is the marker because it is the one field that is meaningless to set by hand + on a legacy context and is written by exactly one producer + (``context.threat_model.threat_model_to_context``). + """ + return self.threat_model_version is not None + def get_type_info(self) -> dict: """Get detailed information about this application type.""" return APPLICATION_TYPE_INFO.get(self.application_type, {}) diff --git a/libs/openant-core/context/threat_model.py b/libs/openant-core/context/threat_model.py new file mode 100644 index 00000000..e23db6eb --- /dev/null +++ b/libs/openant-core/context/threat_model.py @@ -0,0 +1,699 @@ +"""Custom threat models: schema v1, loud validation, and legacy-field derivation. + +OpenAnt's built-in security context collapses an entire attacker model into one of +four ``ApplicationType`` values plus a single boolean (``suppress_local_only``). That +cannot express, say, a deployment orchestrator whose real adversary is "a developer +with commit access to a watched manifest repo and no shell on the orchestrator" — +neither the "remote attacker with a browser" nor the "local user with shell access" +persona fits, and picking either one produces systematically wrong verdicts. + +A *threat model* replaces the four-value enum with a structured description the repo +author writes: free-form classification, components with free-form component types, +named attacker profiles with explicit CAN/CANNOT capabilities, per-input-source trust +levels, and explicit statements of what is and is not a vulnerability *for this +repository*. + +Storage format +-------------- +``OPENANT.THREATMODEL.md`` in the scanned repository's root: human-readable markdown +headings for reviewers and PR diffs, plus **one authoritative fenced ```json block** +that is the machine truth. Markdown-with-embedded-JSON is chosen over YAML +frontmatter because ``check_manual_override`` already proves the seam works with the +same regex, LLMs emit fenced JSON far more reliably than nested YAML, and the +frontmatter path depends on an optional PyYAML import that degrades to a *warning* — +precisely the silent failure this module exists to eliminate. + +``parse_threat_model_md`` scans **every** json block and selects the one whose parsed +object carries ``"schema": "openant-threat-model"``, so a document whose prose +contains illustrative json blocks (a template, a diff, a worked example) still parses. + +Loud failure +------------ +``load_threat_model`` returns ``None`` **only** when the file is absent. If the file +exists but is malformed it **raises**. This is a deliberate inversion of +``check_manual_override``'s catch-all ``except Exception: print(warning); continue``. +The rationale is asymmetric blast radius: a broken ``OPENANT.md`` degrades to +LLM-generated context, which is merely worse; a broken ``OPENANT.THREATMODEL.md`` +degrades to the default ``"web_app"`` assumption, which silently inverts the entire +security model of a scan that the operator explicitly asked to be threat-model driven +— and the resulting report looks completely successful. A typo must not be able to +produce a confident, wrong answer. + +For the same reason ``ThreatModelValidationError`` collects **all** violations rather +than failing fast on the first: a human fixing a hand-written threat model should see +the whole list in one pass, not play whack-a-mole across N scan invocations. + +Known gap: this file originates in the scanned repository and is therefore +attacker-influenceable, and it is NOT prompt-injection-fenced (scanned source code is, +via ``prompts/_fence.py``). See the KNOWN GAP section of +``context/OPENANT_THREATMODEL_TEMPLATE.md``. Accepted, documented risk. +""" + +import json +import os +import re +import stat +from pathlib import Path +from typing import Any + +from context.application_context import ApplicationContext +from utilities.file_io import open_utf8 + +# --- Schema v1 constants ------------------------------------------------------ + +#: Discriminator that identifies the authoritative json block inside the markdown. +# Cap the file size. The document is attacker-authored, and an unbounded read +# is both a memory-exhaustion vector and a way to flood the analysis prompt so +# that real source code falls out of the model's context window. +MAX_THREAT_MODEL_BYTES = 1024 * 1024 + +SCHEMA_NAME = "openant-threat-model" + +#: Current schema version emitted by ``render_threat_model_md``. +SCHEMA_VERSION = 1 + +#: Every schema version this module can ingest. A future v2 that is a superset of +#: v1 would be added here; an incompatible v2 would not be. +SUPPORTED_SCHEMA_VERSIONS = (1,) + +#: Filename looked for in the scanned repository's root. +#: +#: Deliberately NOT added to ``application_context.MANUAL_OVERRIDE_FILES``. If it +#: were, ``check_manual_override`` would consume it on the *built-in* arm too, so +#: both A/B arms would receive threat-model-derived context and the comparison +#: between them would measure nothing. +THREAT_MODEL_FILENAME = "OPENANT.THREATMODEL.md" + +#: Where a component sits relative to the outside world. +EXPOSURE_LEVELS = ("remote", "local", "internal") + +#: Where an attacker stands. Superset of the two personas the built-in path can +#: express ("remote" and "local_user"); the other three are the whole point. +ATTACKER_POSITIONS = ("remote", "adjacent", "local_user", "supply_chain", "insider") + +#: Trust levels for input sources. Matches the vocabulary already used by +#: ``ApplicationContext.trust_boundaries`` so derivation is a straight map. +TRUST_LEVELS = ("untrusted", "semi_trusted", "trusted") + +#: Heading skeleton of the markdown document. Presence of these is advisory — the +#: json block is the machine truth and is what gets strictly validated — but +#: ``render_threat_model_md`` always emits them and reviewers rely on them. +REQUIRED_HEADINGS = ( + "Purpose", + "Architecture & Components", + "Attacker Profiles", + "Input Sources & Trust Levels", + "What IS a Vulnerability", + "What is NOT a Vulnerability", + "Impact", + "Machine-Readable Threat Model", +) + +#: Top-level keys that must be present. ``not_a_vulnerability`` may be an empty +#: list, but the key itself must exist — an author who has genuinely decided that +#: nothing is out of scope should have to say so, not omit it by accident. +REQUIRED_TOP_LEVEL = ( + "schema", + "schema_version", + "classification", + "purpose", + "components", + "attacker_profiles", + "input_sources", + "vulnerability_criteria", + "not_a_vulnerability", + "impact_statement", +) + +#: Recognised-but-optional keys. Listed for documentation and for +#: ``render_threat_model_md``'s ordering; unknown keys are not an error. +OPTIONAL_TOP_LEVEL = ( + "architecture", + "intended_behaviors", + "security_model", + "confidence", + "evidence", + "generated_by", +) + +_JSON_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL) + + +class ThreatModelValidationError(Exception): + """Raised when a threat model is present but unusable. + + Carries the **full** list of violations rather than the first one, plus the + path it came from when known. Collecting everything is not a nicety: the + expected authoring loop is a human editing markdown by hand, and fail-fast + validation turns a five-mistake document into five edit/scan round trips. + """ + + def __init__(self, violations: list[str], path: Path | str | None = None): + self.violations = list(violations) + self.path = Path(path) if path is not None else None + where = f" in {self.path}" if self.path is not None else "" + body = "\n".join(f" - {v}" for v in self.violations) + super().__init__( + f"Invalid threat model{where} ({len(self.violations)} violation(s)):\n{body}" + ) + + +# --- Parsing ------------------------------------------------------------------ + + +def parse_threat_model_md(text: str) -> dict: + """Extract the authoritative threat-model object from markdown text. + + Scans **all** ```json fenced blocks and returns the first whose parsed value is + an object carrying ``"schema": "openant-threat-model"``. Blocks that fail to + parse, or that parse to something else, are skipped rather than fatal — the + document is expected to contain prose examples, and a template's own decoy + blocks must not shadow the real one. + + Args: + text: Full markdown source of an ``OPENANT.THREATMODEL.md``. + + Returns: + The parsed threat-model object. Not validated — call + ``validate_threat_model`` on the result. + + Raises: + ThreatModelValidationError: If no block carries the schema discriminator. + Any json decode errors seen along the way are reported too, since a + typo inside the *real* block is by far the likeliest cause. + """ + blocks = _JSON_BLOCK_RE.findall(text or "") + if not blocks: + raise ThreatModelValidationError( + ["no ```json block found; the machine-readable threat model is required"] + ) + + decode_errors: list[str] = [] + for index, block in enumerate(blocks): + try: + parsed = json.loads(block) + except json.JSONDecodeError as exc: + decode_errors.append(f"json block #{index + 1} is not valid JSON: {exc}") + continue + if isinstance(parsed, dict) and parsed.get("schema") == SCHEMA_NAME: + return parsed + + violations = [ + f"no ```json block declares \"schema\": \"{SCHEMA_NAME}\" " + f"({len(blocks)} json block(s) examined)" + ] + violations.extend(decode_errors) + raise ThreatModelValidationError(violations) + + +def missing_headings(text: str) -> list[str]: + """Headings from ``REQUIRED_HEADINGS`` that do not appear in the document. + + Advisory only — callers surface these as warnings. The json block is the + machine truth, so a document with perfect json and no prose still scans; it is + just useless to the humans who have to review it. + """ + return [h for h in REQUIRED_HEADINGS if h.lower() not in (text or "").lower()] + + +# --- Validation --------------------------------------------------------------- + + +def _require_nonempty_str(value: Any, label: str, violations: list[str]) -> None: + if not isinstance(value, str) or not value.strip(): + violations.append(f"{label} must be a non-empty string (got {_describe(value)})") + + +def _describe(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, str) and not value.strip(): + return "empty string" + return type(value).__name__ + + +def _require_str_list(value: Any, label: str, violations: list[str], *, allow_empty: bool = True) -> None: + if not isinstance(value, list): + violations.append(f"{label} must be a list (got {_describe(value)})") + return + if not allow_empty and not value: + violations.append(f"{label} must not be empty") + for i, item in enumerate(value): + if not isinstance(item, str) or not item.strip(): + violations.append(f"{label}[{i}] must be a non-empty string (got {_describe(item)})") + + +def _require_enum(value: Any, allowed: tuple[str, ...], label: str, violations: list[str], + *, case_insensitive: bool = False) -> None: + candidate = value.lower() if (case_insensitive and isinstance(value, str)) else value + if candidate not in allowed: + violations.append( + f"{label} must be one of {', '.join(allowed)} (got {value!r})" + ) + + +def validate_threat_model(data: Any) -> None: + """Validate a parsed threat model against schema v1, collecting every violation. + + Args: + data: Object returned by ``parse_threat_model_md`` (or hand-built). + + Raises: + ThreatModelValidationError: With ``.violations`` listing **all** problems + found. Every missing required field is named individually, so a + document missing three fields reports three violations, not one. + """ + violations: list[str] = [] + + if not isinstance(data, dict): + raise ThreatModelValidationError( + [f"threat model must be a JSON object (got {_describe(data)})"] + ) + + # Missing-key pass first, so the per-field checks below can assume presence + # and every absent field is named in its own violation. + for key in REQUIRED_TOP_LEVEL: + if key not in data: + violations.append(f"missing required field: {key}") + + if "schema" in data and data["schema"] != SCHEMA_NAME: + violations.append( + f'schema must be "{SCHEMA_NAME}" (got {data["schema"]!r})' + ) + + if "schema_version" in data: + version = data["schema_version"] + if version not in SUPPORTED_SCHEMA_VERSIONS: + supported = ", ".join(str(v) for v in SUPPORTED_SCHEMA_VERSIONS) + violations.append( + f"unsupported schema_version {version!r}; supported versions: {supported}" + ) + + if "classification" in data: + # Free-form on purpose: the whole point is that the four-value enum was + # too small. Only non-emptiness is enforced. + _require_nonempty_str(data["classification"], "classification", violations) + if "purpose" in data: + _require_nonempty_str(data["purpose"], "purpose", violations) + if "impact_statement" in data: + _require_nonempty_str(data["impact_statement"], "impact_statement", violations) + + if "vulnerability_criteria" in data: + _require_str_list( + data["vulnerability_criteria"], "vulnerability_criteria", violations, + allow_empty=False, + ) + if "not_a_vulnerability" in data: + # May legitimately be empty; the key's presence is what is required. + _require_str_list(data["not_a_vulnerability"], "not_a_vulnerability", violations) + + component_names = _validate_components(data.get("components"), violations) \ + if "components" in data else set() + input_source_names = _validate_input_sources(data.get("input_sources"), violations, component_names) \ + if "input_sources" in data else set() + if "attacker_profiles" in data: + _validate_attacker_profiles(data["attacker_profiles"], violations, input_source_names) + + # Optional fields, validated only for type when present. + if data.get("architecture") is not None: + _require_nonempty_str(data["architecture"], "architecture", violations) + if data.get("security_model") is not None: + _require_nonempty_str(data["security_model"], "security_model", violations) + if "intended_behaviors" in data: + _require_str_list(data["intended_behaviors"], "intended_behaviors", violations) + if "evidence" in data: + _require_str_list(data["evidence"], "evidence", violations) + if "confidence" in data: + confidence = data["confidence"] + if not isinstance(confidence, (int, float)) or isinstance(confidence, bool) \ + or not 0.0 <= float(confidence) <= 1.0: + violations.append(f"confidence must be a number in [0.0, 1.0] (got {confidence!r})") + + if violations: + raise ThreatModelValidationError(violations) + + +def _validate_components(components: Any, violations: list[str]) -> set[str]: + """Validate ``components[]``; return the set of declared component names.""" + names: set[str] = set() + if not isinstance(components, list): + violations.append(f"components must be a list (got {_describe(components)})") + return names + if not components: + violations.append("components must not be empty") + for i, comp in enumerate(components): + label = f"components[{i}]" + if not isinstance(comp, dict): + violations.append(f"{label} must be an object (got {_describe(comp)})") + continue + for key in ("name", "component_type"): + if key not in comp: + violations.append(f"{label} missing required field: {key}") + else: + # component_type is FREE-FORM by design ("manifest watcher", + # "reconciliation loop", ...). Constraining it here would + # recreate the four-value enum this schema exists to escape. + _require_nonempty_str(comp[key], f"{label}.{key}", violations) + if "paths" not in comp: + violations.append(f"{label} missing required field: paths") + else: + _require_str_list(comp["paths"], f"{label}.paths", violations, allow_empty=False) + if "exposure" not in comp: + violations.append(f"{label} missing required field: exposure") + else: + _require_enum(comp["exposure"], EXPOSURE_LEVELS, f"{label}.exposure", violations) + if isinstance(comp.get("name"), str) and comp["name"].strip(): + names.add(comp["name"]) + return names + + +def _validate_input_sources(input_sources: Any, violations: list[str], + component_names: set[str]) -> set[str]: + """Validate ``input_sources{}``; return the set of declared source names. + + Also cross-validates ``handled_by[]`` against the declared component names: a + handler that names a component which does not exist is a dangling reference, + and dangling references in a security document are exactly the kind of drift + that makes it quietly stop describing the code. + """ + names: set[str] = set() + if not isinstance(input_sources, dict): + violations.append(f"input_sources must be an object (got {_describe(input_sources)})") + return names + if not input_sources: + violations.append("input_sources must not be empty") + for name, spec in input_sources.items(): + label = f"input_sources[{name!r}]" + names.add(name) + if not isinstance(spec, dict): + violations.append(f"{label} must be an object (got {_describe(spec)})") + continue + if "trust" not in spec: + violations.append(f"{label} missing required field: trust") + else: + # Accepted case-insensitively: these documents are hand-written and + # LLM-written, and "Untrusted" meaning something different from + # "untrusted" would be a hostile piece of API design. + _require_enum(spec["trust"], TRUST_LEVELS, f"{label}.trust", violations, + case_insensitive=True) + if "description" not in spec: + violations.append(f"{label} missing required field: description") + else: + _require_nonempty_str(spec["description"], f"{label}.description", violations) + if "handled_by" in spec: + _require_str_list(spec["handled_by"], f"{label}.handled_by", violations) + if isinstance(spec["handled_by"], list): + for handler in spec["handled_by"]: + if isinstance(handler, str) and handler not in component_names: + violations.append( + f"{label}.handled_by references unknown component {handler!r}; " + f"declared components: {', '.join(sorted(component_names)) or '(none)'}" + ) + return names + + +def _validate_attacker_profiles(profiles: Any, violations: list[str], + input_source_names: set[str]) -> None: + """Validate ``attacker_profiles[]`` and cross-check ``entry_via[]``. + + ``entry_via`` must name a key of ``input_sources``. This is the single most + load-bearing cross-reference in the schema: it is what connects "who the + attacker is" to "what bytes they control", and a dangling entry means a + persona is claiming reach into a channel the document never described. + """ + if not isinstance(profiles, list): + violations.append(f"attacker_profiles must be a list (got {_describe(profiles)})") + return + if not profiles: + violations.append("attacker_profiles must not be empty") + for i, profile in enumerate(profiles): + label = f"attacker_profiles[{i}]" + if not isinstance(profile, dict): + violations.append(f"{label} must be an object (got {_describe(profile)})") + continue + for key in ("id", "description", "impact"): + if key not in profile: + violations.append(f"{label} missing required field: {key}") + else: + _require_nonempty_str(profile[key], f"{label}.{key}", violations) + if "position" not in profile: + violations.append(f"{label} missing required field: position") + else: + _require_enum(profile["position"], ATTACKER_POSITIONS, f"{label}.position", violations) + for key in ("capabilities", "cannot"): + if key not in profile: + violations.append(f"{label} missing required field: {key}") + else: + _require_str_list(profile[key], f"{label}.{key}", violations, allow_empty=False) + if "entry_via" not in profile: + violations.append(f"{label} missing required field: entry_via") + else: + _require_str_list(profile["entry_via"], f"{label}.entry_via", violations, + allow_empty=False) + if isinstance(profile["entry_via"], list): + for entry in profile["entry_via"]: + if isinstance(entry, str) and entry not in input_source_names: + violations.append( + f"{label}.entry_via references unknown input source {entry!r}; " + f"declared input sources: " + f"{', '.join(sorted(input_source_names)) or '(none)'}" + ) + + +# --- Derivation --------------------------------------------------------------- + + +def slug(text: str) -> str: + """Lowercase, hyphenated slug of a free-form classification. + + Used only to build ``application_type = "custom:" + slug(classification)``. + """ + return re.sub(r"-+", "-", re.sub(r"[^a-z0-9]+", "-", (text or "").lower())).strip("-") + + +def threat_model_to_context(data: dict) -> ApplicationContext: + """Build an ``ApplicationContext`` from a validated threat model. + + The new schema fields are carried **verbatim** onto the dataclass, and the + legacy fields are **derived** from them so that every pre-existing consumer + (``format_context_for_prompt``, ``suppress_local_only``, the analyzer and + verifier context threading, ``core/llm_reachability``'s raw-JSON dump) keeps + working without a branch. Threat-model-aware renderers then override the + legacy rendering where it matters; anything not yet converted degrades to a + reasonable approximation rather than to nothing. + + Derivations: + + * ``application_type`` — ``"custom:" + slug(classification)``. Namespaced so it + can never collide with an ``ApplicationType`` value, and so any code that + compares against the enum simply sees "not one of mine" instead of a + plausible-looking wrong match. + * ``trust_boundaries`` — ``{source name: trust level}``, i.e. exactly the legacy + shape, lowercased. This is what keeps ``suppress_local_only`` semantically + sane for residual callers. + * ``requires_remote_trigger`` — true if any attacker profile stands at + ``position == "remote"`` **or** any input source is ``untrusted``. The second + disjunct matters: a supply-chain attacker who controls an untrusted manifest + is not "remote", but suppressing everything they can reach would be wrong. + + Args: + data: A threat model that has already passed ``validate_threat_model``. + + Returns: + ApplicationContext with ``has_threat_model()`` true. + """ + input_sources = data.get("input_sources") or {} + trust_boundaries = { + name: str((spec or {}).get("trust", "")).lower() + for name, spec in input_sources.items() + if isinstance(spec, dict) + } + attacker_profiles = data.get("attacker_profiles") or [] + requires_remote_trigger = any( + isinstance(p, dict) and p.get("position") == "remote" for p in attacker_profiles + ) or any(level == "untrusted" for level in trust_boundaries.values()) + + return ApplicationContext( + application_type="custom:" + slug(data.get("classification", "")), + purpose=data.get("purpose", ""), + intended_behaviors=list(data.get("intended_behaviors") or []), + trust_boundaries=trust_boundaries, + security_model=data.get("security_model"), + not_a_vulnerability=list(data.get("not_a_vulnerability") or []), + requires_remote_trigger=requires_remote_trigger, + confidence=float(data.get("confidence", 0.0) or 0.0), + evidence=list(data.get("evidence") or []), + source="threat_model", + # Carried verbatim. + threat_model_version=data.get("schema_version", SCHEMA_VERSION), + classification=data.get("classification"), + components=list(data.get("components") or []), + attacker_profiles=list(attacker_profiles), + input_sources=dict(input_sources), + vulnerability_criteria=list(data.get("vulnerability_criteria") or []), + impact_statement=data.get("impact_statement"), + ) + + +# --- Rendering ---------------------------------------------------------------- + + +def _bullets(items: Any, empty: str = "_(none)_") -> str: + items = [str(i) for i in (items or [])] + return "\n".join(f"- {i}" for i in items) if items else empty + + +def render_threat_model_md(data: dict) -> str: + """Render a threat model back to ``OPENANT.THREATMODEL.md`` markdown. + + The inverse of ``parse_threat_model_md``: emits the full heading skeleton for + human reviewers followed by the authoritative json block. Round-tripping is + exact for the json (the prose is a projection of it, not an additional source + of truth), which is what lets a generator and a human edit the same file. + """ + lines: list[str] = [ + f"# Threat Model: {data.get('classification', 'unclassified')}", + "", + # NB: no literal triple-backtick-json sequence in this comment. It would open + # a fence that swallows the whole document up to the real block's opener. + "", + "", + "## Purpose", + "", + str(data.get("purpose", "")), + "", + "## Architecture & Components", + "", + ] + if data.get("architecture"): + lines += [str(data["architecture"]), ""] + for comp in data.get("components") or []: + if not isinstance(comp, dict): + continue + lines.append( + f"- **{comp.get('name', '?')}** ({comp.get('component_type', '?')}, " + f"exposure: {comp.get('exposure', '?')}) — " + f"`{'`, `'.join(str(p) for p in comp.get('paths') or [])}`" + ) + if comp.get("description"): + lines.append(f" - {comp['description']}") + lines += ["", "## Attacker Profiles", ""] + for profile in data.get("attacker_profiles") or []: + if not isinstance(profile, dict): + continue + lines += [ + f"### `{profile.get('id', '?')}` — {profile.get('description', '')}", + "", + f"**Position:** {profile.get('position', '?')}", + "", + "**CAN:**", + _bullets(profile.get("capabilities")), + "", + "**CANNOT:**", + _bullets(profile.get("cannot")), + "", + f"**Enters via:** {', '.join(str(e) for e in profile.get('entry_via') or []) or '_(none)_'}", + "", + f"**Impact if successful:** {profile.get('impact', '')}", + "", + ] + lines += ["## Input Sources & Trust Levels", ""] + for name, spec in (data.get("input_sources") or {}).items(): + if not isinstance(spec, dict): + continue + handled = ", ".join(str(h) for h in spec.get("handled_by") or []) + lines.append( + f"- **{name}** — `{spec.get('trust', '?')}` — {spec.get('description', '')}" + + (f" (handled by: {handled})" if handled else "") + ) + lines += [ + "", + "## What IS a Vulnerability", + "", + _bullets(data.get("vulnerability_criteria")), + "", + "## What is NOT a Vulnerability", + "", + _bullets(data.get("not_a_vulnerability")), + "", + "## Impact", + "", + str(data.get("impact_statement", "")), + "", + "## Machine-Readable Threat Model", + "", + "```json", + json.dumps(data, indent=2, ensure_ascii=False), + "```", + "", + ] + return "\n".join(lines) + + +# --- Loading ------------------------------------------------------------------ + + +def threat_model_path(repo_path: Path | str) -> Path: + """Path at which ``load_threat_model`` looks for the threat model.""" + return Path(repo_path) / THREAT_MODEL_FILENAME + + +def load_threat_model(repo_path: Path | str) -> ApplicationContext | None: + """Load ``OPENANT.THREATMODEL.md`` from a repository root, if present. + + Returns: + ``None`` **only** when the file does not exist — the repository simply has + no threat model, and the caller should fall back to the built-in path. + + Raises: + ThreatModelValidationError: When the file exists but cannot be parsed or + fails schema validation. This is the deliberate inversion of + ``check_manual_override``'s catch-all: absence is a choice, but a + *present and broken* threat model is an error the operator must see. + Silently continuing would produce a scan that looks entirely + successful while analysing the repository under the default + ``"web_app"`` attacker model — the exact opposite of what was asked + for, with no signal anywhere in the output that it happened. + OSError: Propagated if the file exists but cannot be read (permissions, + unreadable encoding). Same reasoning: not silently swallowed. + + Note: + The returned context is derived from a file that lives in the *scanned* + repository and is therefore attacker-influenceable, and its contents are + not prompt-injection-fenced. See the KNOWN GAP section of + ``context/OPENANT_THREATMODEL_TEMPLATE.md``. Accepted, documented risk. + """ + path = threat_model_path(repo_path) + if not path.exists(): + return None + + # Guard BEFORE opening. The scanned repository authors this path, so it can + # ship a symlink to a host file, or a FIFO/device that blocks the scanner + # forever. `exists()` and `open()` both follow symlinks, so the check must + # use lstat and must precede the open, not follow it. + link_stat = os.lstat(path) + if stat.S_ISLNK(link_stat.st_mode): + raise ThreatModelValidationError( + [f"{path.name} is a symlink; refusing to follow it out of the " + "scanned repository"], path) + if not stat.S_ISREG(link_stat.st_mode): + raise ThreatModelValidationError( + [f"{path.name} is not a regular file (mode {link_stat.st_mode:o}); " + "a FIFO or device would block the scan indefinitely"], path) + if link_stat.st_size > MAX_THREAT_MODEL_BYTES: + raise ThreatModelValidationError( + [f"{path.name} is too large ({link_stat.st_size} bytes > " + f"{MAX_THREAT_MODEL_BYTES}); refusing to load"], path) + + with open_utf8(path) as handle: + text = handle.read() + + try: + data = parse_threat_model_md(text) + validate_threat_model(data) + except ThreatModelValidationError as exc: + # Re-raise with the path attached so the operator is told *which* file. + raise ThreatModelValidationError(exc.violations, path) from None + + return threat_model_to_context(data) diff --git a/libs/openant-core/context/threat_model_agent.py b/libs/openant-core/context/threat_model_agent.py new file mode 100644 index 00000000..411d1866 --- /dev/null +++ b/libs/openant-core/context/threat_model_agent.py @@ -0,0 +1,202 @@ +"""Generate an OPENANT.THREATMODEL.md for a repository. + +Until this module existed, a custom threat model had to be hand-authored. The +generator surveys the repository — its README and manifests, its directory +shape, and its detected entry points — and produces the document, which is then +committed to the repo root and consumed on subsequent scans. + +Two design choices worth stating: + +**It reuses the ``app_context`` phase rather than adding one.** The phase set in +``utilities/llm/config.py`` is closed, and user configs must list every phase +explicitly — adding a phase would be a breaking config change for every existing +user. Generating a threat model is the same semantic job as generating an +application context, so it rides the same phase. + +**The agent's own output is validated exactly like a human's.** It goes through +``validate_threat_model`` before anything is written. A model that fails +validation is an error, not a file — writing an invalid document would poison +every later scan, and the loader is deliberately strict about malformed input. +""" + +import json +from pathlib import Path + +from context.threat_model import ( + THREAT_MODEL_FILENAME, + ThreatModelValidationError, + render_threat_model_md, + validate_threat_model, +) + +# Kept well above the built-in app-context budget (2000): a full threat model +# carries a component inventory, several attacker profiles and two criteria +# lists, and truncation mid-JSON produces an unparseable document. +MAX_TOKENS = 6000 + + +class ThreatModelGenerationError(Exception): + """Raised when a threat model could not be generated or is unusable.""" + + +GENERATION_PROMPT = """You are a security architect. Study this repository and \ +produce a threat model for it. + +Do NOT assume a generic web application. Classify what this program actually is, +in your own words — the classification is free-form, not drawn from a fixed list. + +Return ONLY a JSON object with exactly these keys: + + schema "openant-threat-model" + schema_version 1 + classification free-form description of what this program is + purpose what it does, for whom + components [{{name, paths[], component_type (FREE-FORM), \ +exposure: remote|local|internal, description?}}] + architecture prose data-flow summary (optional) + attacker_profiles [{{id, description, position: \ +remote|adjacent|local_user|supply_chain|insider, capabilities[], cannot[], \ +entry_via[], impact}}] + input_sources {{name: {{trust: untrusted|semi_trusted|trusted, \ +description, handled_by?[]}}}} + vulnerability_criteria [] what IS a vulnerability in THIS threat model + not_a_vulnerability [] what is NOT — intended behaviour that looks alarming + impact_statement overall worst-case impact + +Rules that matter: +- `entry_via` entries MUST name keys of `input_sources`. +- `cannot` is load-bearing: state what each attacker genuinely cannot do. It is + what makes a later verdict falsifiable. +- Prefer several specific attacker profiles over one generic one. A supply-chain + or adjacent attacker is often the realistic threat, not an anonymous remote user. +- `not_a_vulnerability` should name behaviour that IS intentional here. Do not + use it to wave away whole classes of real risk. + +REPOSITORY: {name} + +--- Context files --- +{sources} + +--- Entry points detected --- +{entry_points} +""" + + +def threat_model_exists(repo_path: Path) -> bool: + """Whether the repository already ships a threat model.""" + return (Path(repo_path) / THREAT_MODEL_FILENAME).exists() + + +def _build_prompt(repo_path: Path) -> str: + """Assemble the survey prompt from the repo's own signals.""" + from context.application_context import detect_entry_points, gather_context_sources + + sources = gather_context_sources(repo_path) + rendered = "\n\n".join( + f"### {name}\n{content[:4000]}" for name, content in sources.items() + ) or "(no context files found)" + + try: + entry_points = detect_entry_points(repo_path) or "(none detected)" + except Exception: # noqa: BLE001 - survey signal only; never fail generation + entry_points = "(entry-point detection unavailable)" + + return GENERATION_PROMPT.format( + name=Path(repo_path).name, sources=rendered, entry_points=entry_points + ) + + +def _extract_json(text: str) -> dict: + """Pull the JSON object out of a model response.""" + stripped = text.strip() + if stripped.startswith("```"): + # Strip a fenced block, tolerating a ```json info-string. + lines = [ln for ln in stripped.splitlines() if not ln.startswith("```")] + stripped = "\n".join(lines) + start, end = stripped.find("{"), stripped.rfind("}") + if start == -1 or end == -1: + raise ThreatModelGenerationError( + f"model response contained no JSON object: {text[:200]!r}" + ) + try: + return json.loads(stripped[start:end + 1]) + except json.JSONDecodeError as exc: + raise ThreatModelGenerationError( + f"model response was not valid JSON: {exc}" + ) from exc + + +def generate_threat_model( + repo_path: Path, + binding, + *, + force: bool = False, + output_path: Path | None = None, +) -> Path: + """Generate a threat model and write it to the repository root. + + Args: + repo_path: Repository to survey. + binding: Phase binding supplying the adapter and model. The + ``app_context`` phase is reused deliberately (see module docstring). + force: Overwrite an existing threat model. The previous file is backed + up to ``.bak`` first — it may be hand-curated, and losing a + human's threat model to a regeneration would be a poor trade. + output_path: Write here instead of the repo root. Used when the repo is + read-only. + + Returns: + Path to the written file. + + Raises: + ThreatModelGenerationError: If a model already exists without ``force``, + the LLM call fails, or the produced model fails validation. + """ + repo_path = Path(repo_path) + target = Path(output_path) if output_path else repo_path / THREAT_MODEL_FILENAME + + if target.exists() and not force: + raise ThreatModelGenerationError( + f"{target} already exists. It may be hand-curated; pass force=True " + "to regenerate (the existing file is backed up first)." + ) + + prompt = _build_prompt(repo_path) + + try: + # Single-shot rather than a tool loop. The survey inputs (README, + # manifests, entry points) are already assembled above, so the agentic + # exploration the enhancer needs per-unit buys little here — and it + # keeps the generator usable on adapters without tool support. + response = binding.adapter.complete(prompt=prompt, max_tokens=MAX_TOKENS) + except ThreatModelGenerationError: + raise + except Exception as exc: # noqa: BLE001 - adapter errors vary by provider + raise ThreatModelGenerationError( + f"threat-model generation call failed: {exc}" + ) from exc + + data = _extract_json(response if isinstance(response, str) else str(response)) + + data.setdefault("generated_by", {}).update({ + "model": getattr(binding, "model", "unknown"), + "provider": getattr(binding, "provider_name", "unknown"), + }) + + # Validate BEFORE writing. An invalid document on disk would fail every + # subsequent scan at load time, which is a worse failure than not writing. + try: + validate_threat_model(data) + except ThreatModelValidationError as exc: + raise ThreatModelGenerationError( + "generated threat model failed validation: " + + "; ".join(getattr(exc, "violations", [str(exc)])) + ) from exc + + if target.exists() and force: + backup = target.with_suffix(target.suffix + ".bak") + backup.write_text(target.read_text(encoding="utf-8"), encoding="utf-8") + + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(render_threat_model_md(data), encoding="utf-8") + return target diff --git a/libs/openant-core/core/dataset_merge.py b/libs/openant-core/core/dataset_merge.py new file mode 100644 index 00000000..64c28453 --- /dev/null +++ b/libs/openant-core/core/dataset_merge.py @@ -0,0 +1,228 @@ +"""Merge per-language parse output into a single dataset. + +The parse fan-out writes ``//`` directories, one per language, each +containing the same flat filenames. This module turns those into the single +``dataset.json`` / ``analyzer_output.json`` that the rest of the pipeline +consumes, so enhance/analyze/verify/report run ONCE over everything rather than +once per language. That works because the LLM stages are deliberately +language-agnostic (see DOCUMENTATION.md) — language is never used for rule or +query selection. + +What is deliberately NOT merged: call graphs. There are no cross-language edges +to resolve (a Python call into a Go binary is not an edge any parser emits), so +unioning the graphs would imply a connectivity that does not exist. Instead +``write_call_graph_index`` records where each language's graph lives, which is +the single seam a future cross-language graph would attach to. +""" + +import os +import sys +from dataclasses import dataclass, field + +from utilities.file_io import read_json, write_json + + +@dataclass +class MergeStats: + """Outcome of merging per-language datasets. + + Attributes: + languages: Languages that contributed units, in merge order. + units_per_language: Unit count contributed by each language. + total_units: Units in the merged dataset. + id_collisions: Unit ids that appeared in more than one language and + were namespaced. Empty in normal operation — see + ``merge_datasets`` for why a collision is possible at all. + """ + + languages: list[str] = field(default_factory=list) + units_per_language: dict[str, int] = field(default_factory=dict) + total_units: int = 0 + id_collisions: list[str] = field(default_factory=list) + + +def _successful(outcomes) -> list: + """Outcomes that produced a dataset we can actually read.""" + return [o for o in outcomes if o.ok and o.dataset_path] + + +def merge_datasets(outcomes, output_path: str) -> MergeStats: + """Union per-language datasets into one, stamping each unit's language. + + Args: + outcomes: ``LanguageParseOutcome`` list from ``parse_repository_multi``. + Failed languages are skipped — a degraded run still produces a + usable dataset from the survivors. + output_path: Where to write the merged ``dataset.json``. + + Returns: + :class:`MergeStats` describing what was merged. + + Raises: + ValueError: If no outcome succeeded. Callers must not silently treat a + fully-failed parse as an empty-but-valid dataset. + """ + usable = _successful(outcomes) + if not usable: + raise ValueError( + "Cannot merge: no successful language parses among " + f"{[o.language for o in outcomes]}" + ) + + merged_units: list[dict] = [] + seen_ids: set[str] = set() + stats = MergeStats() + + # Carry the first language's top-level scalars (name, repository). They + # describe the repo, not the language, so they are identical across + # per-language datasets by construction of the fan-out. + first = read_json(usable[0].dataset_path) + merged: dict = { + key: value + for key, value in first.items() + if key not in ("units", "statistics", "metadata") + } + + per_language: dict[str, dict] = {} + + for outcome in usable: + data = read_json(outcome.dataset_path) + units = data.get("units", []) + + for unit in units: + # setdefault, not assignment: if a parser ever starts emitting a + # more specific language tag, it knows better than we do. + unit.setdefault("language", outcome.language) + + unit_id = unit.get("id") + if unit_id is not None and unit_id in seen_ids: + # Unit ids are `relative/path.ext:name`, so a collision needs + # the same path AND extension in two languages — impossible + # while extension→language is a function, but possible the + # moment a new language claims `.h` alongside C. Namespace the + # LATER one only: rewriting ids unconditionally would break + # core/diff_filter.py and the reporter's caller/callee dedup, + # both of which match on id. + stats.id_collisions.append(unit_id) + unit["id"] = f"{outcome.language}::{unit_id}" + print( + f" [Merge] WARNING: unit id collision {unit_id!r} — " + f"namespaced as {unit['id']!r}", + file=sys.stderr, + ) + if unit.get("id") is not None: + seen_ids.add(unit["id"]) + + merged_units.append(unit) + + stats.languages.append(outcome.language) + stats.units_per_language[outcome.language] = len(units) + + per_language[outcome.language] = { + "units": len(units), + "dataset_path": outcome.dataset_path, + "output_dir": outcome.output_dir, + } + # Raw per-parser statistics are preserved per-language, NOT summed into + # the top-level aggregate. The parsers disagree on naming (Python emits + # `total_units`, JavaScript `totalUnits`), so summing by raw key name + # yields a dict where each convention's key holds only its own + # language's count while reading as a whole-dataset figure. Rather than + # inventing a canonical schema neither parser agreed to, the aggregate + # below carries only figures the merge can compute unambiguously. + per_language[outcome.language]["statistics"] = data.get("statistics") or {} + + merged["units"] = merged_units + merged["statistics"] = { + "total_units": len(merged_units), + "units_per_language": dict(stats.units_per_language), + "languages": list(stats.languages), + } + merged["metadata"] = { + **(first.get("metadata") or {}), + "languages": stats.languages, + "per_language": per_language, + } + + stats.total_units = len(merged_units) + + os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) + write_json(output_path, merged, indent=2) + print( + f"[Merge] {stats.total_units} units from " + f"{len(stats.languages)} language(s): " + + ", ".join(f"{k}={v}" for k, v in stats.units_per_language.items()), + file=sys.stderr, + ) + return stats + + +def merge_analyzer_outputs(outcomes, output_path: str) -> None: + """Union per-language ``analyzer_output.json`` files. + + The parsers do NOT agree on this file's key set — Python emits + ``functions``/``callGraph``/``reverseCallGraph`` while JavaScript adds + ``repository``/``classes``/``call_graph``/``reverse_call_graph``/ + ``indirect_calls``. So the merge unions over whatever keys are present + rather than assuming a schema, and merges each key by type: dicts are + keyed by unit id and get updated; anything else (notably the ``repository`` + string) is taken from the first language that supplied it. + + Only ``functions`` is load-bearing in-repo — ``RepositoryIndex`` at + ``utilities/agentic_enhancer/repository_index.py`` is the sole consumer of + this file's contents; every other reference passes the path through. The + remaining keys are preserved best-effort so nothing is silently dropped. + """ + usable = [o for o in outcomes if o.ok and o.analyzer_output_path] + if not usable: + return + + merged: dict = {} + for outcome in usable: + if not os.path.exists(outcome.analyzer_output_path): + continue + data = read_json(outcome.analyzer_output_path) + for key, value in data.items(): + if isinstance(value, dict): + merged.setdefault(key, {}).update(value) + else: + # Scalars/lists describe the repo, not the language; first + # writer wins rather than concatenating incomparable values. + merged.setdefault(key, value) + + os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) + write_json(output_path, merged, indent=2) + + +def write_call_graph_index(outcomes, output_path: str) -> dict[str, str]: + """Record where each language's ``call_graph.json`` lives. + + Built by PROBING THE FILESYSTEM, never from a hardcoded language list. A + comment in core/scanner.py long claimed only Python and Zig persist a call + graph; JavaScript does too, and which parsers do so changes over time. A + stale list would silently skip post-LLM reachability re-filtering for a + language that actually supports it — the exact cost regression this index + exists to prevent. + + Args: + outcomes: Per-language parse outcomes. + output_path: Where to write ``call_graphs.json``. + + Returns: + Mapping of language → path to its call graph, relative to the run dir. + """ + run_dir = os.path.dirname(os.path.abspath(output_path)) + index: dict[str, str] = {} + + for outcome in outcomes: + if not outcome.ok: + # A stale call_graph.json can outlive a failed re-parse; indexing + # it would feed the previous run's graph into this run's filter. + continue + candidate = os.path.join(outcome.output_dir, "call_graph.json") + if os.path.isfile(candidate): + index[outcome.language] = os.path.relpath(candidate, run_dir) + + os.makedirs(run_dir, exist_ok=True) + write_json(output_path, index, indent=2) + return index diff --git a/libs/openant-core/core/file_boundary.py b/libs/openant-core/core/file_boundary.py new file mode 100644 index 00000000..de7db7b4 --- /dev/null +++ b/libs/openant-core/core/file_boundary.py @@ -0,0 +1,108 @@ +"""Single source of truth for the multi-file unit boundary marker. + +When a unit inlines its dependencies, the parser concatenates several files' +source into one ``primary_code`` blob and separates them with a marker. That +marker must be a COMMENT in the language being parsed — a ``//`` line inside +Python source is a syntax error — so producers emit it with their own comment +prefix: + + python, ruby -> # ========== File Boundary ========== + javascript, go, c, php, -> // ========== File Boundary ========== + zig + +Consumers, however, historically matched the ``//`` form literally +(``prompts/vulnerability_analysis.py``, ``prompts/verification_prompts.py``, +``validate_dataset_schema.py``, ``utilities/agentic_enhancer/agent.py``). For +Python and Ruby the split therefore never fired, and the fallback branch +handed the model the ENTIRE concatenation as the target function while +silently dropping the "Context (do NOT analyze these)" section — so dependency +code was analysed as if it were the unit under test. + +The fix is to agree on the part that does not vary: the text between the +comment prefix and the end of the line. Match on that, emit with the right +prefix. +""" + +import re + +# The invariant substring every producer emits, whatever its comment syntax. +BOUNDARY_TEXT = "========== File Boundary ==========" + +# Comment prefix per language. Anything not listed uses the C-style default, +# matching the pre-existing behaviour of the agentic enhancer. +_COMMENT_PREFIX = { + "python": "#", + "ruby": "#", + "javascript": "//", + "typescript": "//", + "go": "//", + "c": "//", + "cpp": "//", + "php": "//", + "zig": "//", +} + +_DEFAULT_PREFIX = "//" + +# A whole boundary line: optional leading comment prefix, then the invariant +# text. Anchored to line starts so the marker cannot be matched inside a +# string literal that merely contains the words. +_BOUNDARY_LINE = re.compile( + rf"^[ \t]*(?:#|//)?[ \t]*{re.escape(BOUNDARY_TEXT)}[ \t]*$", + re.MULTILINE, +) + + +def has_boundary(code: str) -> bool: + """Whether *code* contains at least one file-boundary marker.""" + if not code: + return False + return _BOUNDARY_LINE.search(code) is not None + + +def split_on_boundary(code: str) -> list[str]: + """Split concatenated multi-file code into its constituent parts. + + Comment-syntax-agnostic: a Python unit separated by ``#`` markers and a + JavaScript unit separated by ``//`` markers both split correctly, as does a + blob carrying both (possible once units from several languages share a + merged dataset). + + Args: + code: The unit's ``primary_code``. + + Returns: + The parts, in order. Index 0 is the target function; the rest are + inlined dependencies. A single-file unit yields a one-element list, so + callers can branch on ``len(parts) > 1`` exactly as before. + """ + if not code: + return [code] + return _BOUNDARY_LINE.split(code) + + +def boundary_in_code(code: str, default_language: str | None = None) -> str: + """The boundary marker as it actually appears in *code*. + + Used when re-joining split parts: echoing back the producer's own marker + keeps the output byte-faithful to the input, and means callers that have no + language parameter (``get_verification_prompt``) need not grow one just to + pick a comment prefix. + + Falls back to ``default_language``'s marker, then to the C-style default, + when *code* carries no boundary. + """ + if code: + match = _BOUNDARY_LINE.search(code) + if match is not None: + return f"\n\n{match.group(0).strip()}\n\n" + return boundary_for_language(default_language) + + +def boundary_for_language(language: str | None) -> str: + """The boundary marker to EMIT for *language*, with surrounding blank lines. + + Matches the producers' formatting so round-tripping is exact. + """ + prefix = _COMMENT_PREFIX.get((language or "").lower(), _DEFAULT_PREFIX) + return f"\n\n{prefix} {BOUNDARY_TEXT}\n\n" diff --git a/libs/openant-core/core/language_registry.py b/libs/openant-core/core/language_registry.py new file mode 100644 index 00000000..bbf56978 --- /dev/null +++ b/libs/openant-core/core/language_registry.py @@ -0,0 +1,259 @@ +"""Single source of truth for the supported-language set. + +Before this module, four places independently described which languages +OpenAnt supports, and they drifted: + + 1. ``config/languages.json`` — the extension→language map used for detection. + 2. The ``if/elif`` dispatch chain in ``core/parser_adapter.py``. + 3. ``argparse`` ``choices=[...]``, duplicated in two places in ``openant/cli.py``. + 4. Go flag help strings in ``cmd/init.go``, ``cmd/scan.go``, ``cmd/parse.go``. + +The drift was not hypothetical: ``scan.go`` and ``parse.go`` omitted Zig from +their help text, and so did ``README.md``. Adding a language meant remembering +seven files. + +``config/languages.json`` is now authoritative and everything else derives from +it. The legacy top-level ``extensions`` and ``skip_dirs`` maps are preserved +byte-for-byte, because the Go detector (``cmd/init.go``) reads the same file and +must keep working without a coordinated cross-language change; a consistency +test asserts the legacy flat map stays exactly the union of the per-language +lists, so the two representations cannot silently diverge. + +Adding a language is now a config edit plus a ``parsers//`` directory. +""" + +import os +import sys +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +from utilities.file_io import read_json + +# Where config/languages.json may live, in priority order: +# 1. $OPENANT_LANGUAGES_CONFIG (explicit operator override) +# 2. upward from this module (monorepo checkout, and an installed layout +# that ships the config as package data) +# 3. upward from the CWD (running against a checkout from elsewhere) +# +# A bare `parent.parent.parent.parent` resolves correctly ONLY in the monorepo +# checkout; under an installed layout it points outside the distribution. That +# was not a graceful degradation: `supported_languages()` runs during argparse +# construction, so a missing config raised FileNotFoundError before the CLI +# parsed a single flag — `openant --help` died. The Go side already searched +# upward from both the executable and the CWD and degraded rather than failing +# at flag-registration time; this brings Python to the same contract. +_CONFIG_REL = Path("config") / "languages.json" +_SEARCH_LEVELS = 6 + + +def _search_upward(start: Path) -> Path | None: + current = start.resolve() + for _ in range(_SEARCH_LEVELS): + candidate = current / _CONFIG_REL + if candidate.is_file(): + return candidate + if current.parent == current: + break + current = current.parent + return None + + +def find_languages_config() -> Path | None: + """Locate config/languages.json, or None if it cannot be found. + + Returning None rather than raising is deliberate: callers on the CLI's + startup path must degrade, not die. + """ + override = os.environ.get("OPENANT_LANGUAGES_CONFIG") + if override: + candidate = Path(override) + if candidate.is_file(): + return candidate + # A stale override must not be more destructive than no override. + print( + f"[languages] OPENANT_LANGUAGES_CONFIG={override!r} not found; " + "falling back to search", + file=sys.stderr, + ) + + found = _search_upward(Path(__file__).parent) + if found is not None: + return found + return _search_upward(Path.cwd()) + +# Root of openant-core, used to resolve parser script paths. +_CORE_ROOT = Path(__file__).parent.parent + +# Key used inside a per-extension ``fence`` mapping for "everything else". +_FENCE_DEFAULT_KEY = "*" + + +@dataclass(frozen=True) +class LanguageSpec: + """Everything OpenAnt knows about one supported language. + + Attributes: + name: Canonical language name (the value used everywhere as the key). + extensions: File extensions claimed by this language, lowercase, with + the leading dot. + parser_mode: ``"inprocess"`` or ``"subprocess"``. Python is parsed + in-process; every other parser is a subprocess with a shared argv + contract. This asymmetry is data rather than control flow so the + dispatch chain can be a lookup. + parser_script: Repo-relative path to the subprocess entry point, or + ``None`` for in-process parsers. + bootstrap: Optional pre-parse hook name (``"npm"`` for JavaScript, + whose parser carries its own ``package.json``). + fence: Markdown code-fence tag. Either a plain string, or a mapping of + extension → tag with a ``"*"`` fallback for languages where one + parser covers several fence tags (``.ts`` must fence as + ``typescript``, ``.cpp`` as ``cpp``). + docker_template: Template name for dynamic exploit testing, or ``None`` + when no template exists. ``None`` means "skip", never "guess". + enabled: Whether this language participates in detection and dispatch. + """ + + name: str + extensions: tuple[str, ...] + parser_mode: str + parser_script: str | None + bootstrap: str | None + fence: str | dict[str, str] + docker_template: str | None + enabled: bool + + +@lru_cache(maxsize=1) +def _load_config() -> dict: + """Read and cache ``config/languages.json``. + + Cached because detection walks large trees and would otherwise re-read the + file per call. Tests that mutate the config must call + ``load_registry.cache_clear()`` / ``_load_config.cache_clear()``. + """ + path = find_languages_config() + if path is None: + # Degrade to an empty registry. Consumers that merely DESCRIBE the + # language set (help text, choices) then show nothing rather than + # crashing; consumers that need to actually parse still fail loudly + # because detection finds no extensions and raises. + return {} + return read_json(path) + + +@lru_cache(maxsize=1) +def load_registry() -> dict[str, LanguageSpec]: + """Build the language registry from config. + + Returns: + Mapping of language name → :class:`LanguageSpec`, insertion-ordered by + language name so downstream iteration is deterministic. + """ + config = _load_config() + raw = config.get("languages", {}) + + registry: dict[str, LanguageSpec] = {} + for name in sorted(raw): + entry = raw[name] + parser = entry.get("parser", {}) + registry[name] = LanguageSpec( + name=name, + extensions=tuple(ext.lower() for ext in entry.get("extensions", [])), + parser_mode=parser.get("mode", "subprocess"), + parser_script=parser.get("script"), + bootstrap=parser.get("bootstrap"), + fence=entry.get("fence", name), + docker_template=entry.get("docker_template"), + enabled=entry.get("enabled", True), + ) + return registry + + +def supported_languages() -> list[str]: + """Enabled language names, sorted — the canonical list for CLI choices.""" + return [name for name, spec in load_registry().items() if spec.enabled] + + +def extension_map() -> dict[str, str]: + """Extension → language name, derived from the per-language lists. + + This is the same content as the legacy top-level ``extensions`` map; a + consistency test asserts they match so the Go reader and the Python reader + cannot drift. + """ + return { + ext: spec.name + for spec in load_registry().values() + if spec.enabled + for ext in spec.extensions + } + + +def skip_dirs() -> frozenset[str]: + """Directory names pruned during detection and scanning.""" + return frozenset(_load_config().get("skip_dirs", [])) + + +def language_for_path(path: str | os.PathLike) -> str | None: + """Language owning this file, by extension, or ``None`` if unsupported.""" + suffix = Path(path).suffix.lower() + return extension_map().get(suffix) + + +def fence_for_path(path: str | os.PathLike, fallback: str | None = None) -> str: + """Markdown code-fence tag for a file, resolved by EXTENSION. + + Resolving per file rather than per scan is a correctness fix, not just + multi-language plumbing: a ``.ts`` file in a JavaScript scan is currently + fenced as ``javascript`` because the caller passes the scan-wide language. + + Args: + path: File path whose extension decides the fence. + fallback: Language name to fall back on when the path has no + recognized extension (e.g. the literal ``"unknown"`` the reporter + synthesizes for a route key with no colon). + + Returns: + The fence tag, or ``""`` when nothing matches — an empty tag is a valid + unhighlighted Markdown fence, so this degrades rather than breaking. + """ + suffix = Path(path).suffix.lower() + registry = load_registry() + + language = extension_map().get(suffix) + if language is not None: + fence = registry[language].fence + if isinstance(fence, dict): + return fence.get(suffix, fence.get(_FENCE_DEFAULT_KEY, "")) + return fence + + # No usable extension — fall back to the caller's scan-wide language. + if fallback: + spec = registry.get(fallback.lower()) + if spec is not None: + fence = spec.fence + if isinstance(fence, dict): + return fence.get(_FENCE_DEFAULT_KEY, "") + return fence + + return "" + + +def docker_template_for(language: str) -> str | None: + """Dynamic-test Docker template for a language, or ``None`` if none exists. + + ``None`` is meaningful and must be honoured by callers: generating a Python + Dockerfile for a C finding burns tokens on a guaranteed failure. Callers + should skip with an explicit reason instead. + """ + spec = load_registry().get(language) + return spec.docker_template if spec else None + + +def parser_script_path(language: str) -> Path | None: + """Absolute path to a language's subprocess parser entry point.""" + spec = load_registry().get(language) + if spec is None or not spec.parser_script: + return None + return _CORE_ROOT / spec.parser_script diff --git a/libs/openant-core/core/language_selection.py b/libs/openant-core/core/language_selection.py new file mode 100644 index 00000000..5e80ca67 --- /dev/null +++ b/libs/openant-core/core/language_selection.py @@ -0,0 +1,199 @@ +"""Deciding WHICH detected languages a scan should actually parse. + +Detection (``core.parser_adapter.detect_languages``) answers "what is in this +repo". This module answers "what is worth parsing", which is a separate +judgement: spawning a full Go parse for one stray ``tools/gen.go`` in a +5,000-file Python repo costs real time and, downstream, real tokens. + +The policy is deliberately conservative in one specific way — the dominant +language is ALWAYS selected, whatever the thresholds say. That guarantees the +selection is never empty for a repo with any supported source, which in turn +guarantees multi-language scanning can never be *less* capable than the +single-language behaviour it replaces. +""" + +import math +import sys +from dataclasses import dataclass, field + +from core.language_registry import supported_languages + +# A language must clear BOTH an absolute floor and a share of the repo. The +# absolute floor stops a handful of files pulling in a whole toolchain; the +# share stops a large-but-proportionally-tiny slice of a monorepo doing the +# same. Tuned to be permissive: the cost of a missed language is a silent +# coverage gap, which is worse than a slightly slow scan. +DEFAULT_MIN_FILES = 5 +DEFAULT_MIN_SHARE = 0.02 # 2% of counted source files + + +class UnknownLanguageError(ValueError): + """Raised when an explicitly requested language is not supported.""" + + def __init__(self, unknown: list[str]): + self.unknown = unknown + supported = ", ".join(supported_languages()) + super().__init__( + f"Unknown language(s): {', '.join(unknown)}. Supported: {supported}" + ) + + +@dataclass +class LanguageSelection: + """The outcome of applying selection policy to a detection result. + + Attributes: + selected: Languages to parse, ordered by descending file count. + counts: The full detection result, including excluded languages. + excluded: Language → human-readable reason it was dropped. + NOT currently surfaced anywhere: callers flatten this object to + ``selected`` before it reaches the scanner, so exclusion reasons are + computed and discarded. The coverage gap this field exists to make + visible is therefore still silent. + primary: The dominant language. Populates every scalar ``language`` + field downstream, preserving back-compat. + """ + + selected: list[str] + counts: dict[str, int] = field(default_factory=dict) + excluded: dict[str, str] = field(default_factory=dict) + primary: str = "" + + @property + def is_multi(self) -> bool: + return len(self.selected) > 1 + + +def select_languages( + counts: dict[str, int], + *, + include: list[str] | None = None, + all_languages: bool = False, + min_files: int = DEFAULT_MIN_FILES, + min_share: float = DEFAULT_MIN_SHARE, +) -> LanguageSelection: + """Choose which detected languages to parse. + + Rules are applied in this order: + + 1. An explicit ``include`` list wins outright — no thresholds. The user + asked for these languages by name; second-guessing them would be + surprising. Unknown names raise rather than being silently dropped. + 2. ``all_languages`` disables thresholds but still requires detection. + 3. Otherwise a language is selected iff its count clears + ``max(min_files, ceil(min_share * total))``. + 4. The dominant language is always selected regardless of the above. + + Args: + counts: Detection result from ``detect_languages``. + include: Explicit language list (from ``--languages``). + all_languages: Select everything detected (from ``--all-languages``). + min_files: Absolute file-count floor. + min_share: Fractional share floor, 0.0-1.0. + + Returns: + A :class:`LanguageSelection`. + + Raises: + ValueError: If ``counts`` is empty, or ``include`` names an unsupported + language. + """ + if not counts: + raise ValueError("No languages detected; nothing to select from.") + + # counts arrives ordered by (-count, name) from detect_languages, but do + # not depend on the caller having preserved that. + ordered = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + primary = ordered[0][0] + + if include: + requested = [lang.strip().lower() for lang in include if lang.strip()] + unknown = [lang for lang in requested if lang not in supported_languages()] + if unknown: + raise UnknownLanguageError(unknown) + + # Preserve detection order, and keep requested-but-absent languages out + # of `selected` — parsing a language with zero files is pure overhead. + selected = [lang for lang, _ in ordered if lang in requested] + excluded = { + lang: "not requested via --languages" + for lang, _ in ordered + if lang not in requested + } + for lang in requested: + if lang not in counts: + excluded[lang] = "explicitly requested but no source files found" + + if not selected: + # Falling through with an empty selection let callers treat it as + # "no multi-language request" and re-detect the dominant language — + # inverting an explicit user instruction and, under `scan`, billing + # LLM analysis of a language the user had scoped out. + raise ValueError( + f"None of the requested language(s) {requested} have source " + f"files in this repository (detected: {sorted(counts)}). " + "Nothing to parse." + ) + + return LanguageSelection( + selected=selected, + counts=dict(ordered), + excluded=excluded, + # `primary` must stay a language we actually parse, otherwise the + # scalar `language` field downstream would name an unscanned one. + primary=selected[0] if selected else primary, + ) + + if all_languages: + return LanguageSelection( + selected=[lang for lang, _ in ordered], + counts=dict(ordered), + excluded={}, + primary=primary, + ) + + total = sum(counts.values()) + threshold = max(min_files, math.ceil(min_share * total)) + + selected: list[str] = [] + excluded: dict[str, str] = {} + for lang, count in ordered: + if lang == primary: + # Rule 4: never let thresholds empty the selection. + selected.append(lang) + continue + if count >= threshold: + selected.append(lang) + else: + share = (count / total) * 100 if total else 0.0 + excluded[lang] = ( + f"{count} file(s) ({share:.2f}%) below threshold of {threshold}" + ) + + return LanguageSelection( + selected=selected, + counts=dict(ordered), + excluded=excluded, + primary=primary, + ) + + +def report_exclusions(excluded: dict[str, str]) -> None: + """Print excluded languages to stderr as an explicit coverage gap. + + Thresholds are allowed to skip work; they are not allowed to do it + quietly. For a security scanner a silently-skipped language is a silently + missed vulnerability class — a 4-file PHP upload handler in a JS monorepo + is precisely what the tool exists to find, and the parse it saves costs + ~0.1s. So the exclusion is reported wherever a human or a CI log will see + it, with the reason verbatim. + """ + if not excluded: + return + print("\n COVERAGE GAP — languages detected but NOT scanned:", file=sys.stderr) + for language, reason in sorted(excluded.items()): + print(f" {language}: {reason}", file=sys.stderr) + print( + " Use --all-languages to scan everything, or --languages to name a set.", + file=sys.stderr, + ) diff --git a/libs/openant-core/core/parser_adapter.py b/libs/openant-core/core/parser_adapter.py index 5df17124..cd38e3b4 100644 --- a/libs/openant-core/core/parser_adapter.py +++ b/libs/openant-core/core/parser_adapter.py @@ -10,13 +10,24 @@ """ import contextlib +import functools import json import os import shutil import subprocess import sys +import time +from collections.abc import Callable +from dataclasses import asdict, dataclass from pathlib import Path +from core.language_registry import ( + extension_map, + load_registry, + parser_script_path, + skip_dirs, + supported_languages, +) from core.schemas import ParseResult from utilities.file_io import open_utf8, read_json, write_json @@ -26,41 +37,48 @@ # JS parser directory (holds its own package.json / node_modules) _JS_PARSER_DIR = _CORE_ROOT / "parsers" / "javascript" -# Shared language detection config (single source of truth: config/languages.json) -_LANGUAGES_CONFIG = Path(__file__).parent.parent.parent.parent / "config" / "languages.json" +def detect_languages(repo_path: str) -> dict[str, int]: + """Count source files per language. + This is the multi-language primitive. ``detect_language`` wraps it for the + single-language callers, which previously threw the count map away — a repo + that is 60% Go and 40% TypeScript was scanned as a Go repo, and the absence + of the TypeScript was never reported anywhere. -def _load_language_config() -> dict: - return read_json(_LANGUAGES_CONFIG) + Directories named in ``skip_dirs`` are PRUNED rather than filtered + per-file. This matches the Go detector's ``filepath.SkipDir`` semantics + exactly (the two implementations previously disagreed on what "skip" + meant), and it stops the walk descending into ``node_modules`` at all, + which is a substantial speedup on JS monorepos. - -def detect_language(repo_path: str) -> str: - """Auto-detect the primary language of a repository. - - Counts source files by extension and returns the dominant language. - Extension mappings and skip directories are loaded from config/languages.json. + Args: + repo_path: Repository root to walk. Returns: - One of: "python", "javascript", "go", "c", "ruby", "php", "zig" + Mapping of language name → source-file count, ordered by descending + count with ties broken alphabetically. The ordering is deterministic: + the previous ``max(counts, key=counts.get)`` returned whichever key + happened to be first in dict order, and the Go side's randomized map + iteration meant the two could disagree on a tie for the same repo. + + Raises: + ValueError: If no supported source files were found. The message is + preserved verbatim so ``detect_language``'s contract is unchanged. """ - config = _load_language_config() - skip_dirs = set(config["skip_dirs"]) - extensions = config["extensions"] + extensions = extension_map() + skipped = skip_dirs() - repo = Path(repo_path) counts: dict[str, int] = {} - for f in repo.rglob("*"): - if not f.is_file(): - continue - # Skip configured non-source dirs - if any(p in skip_dirs for p in f.parts): - continue + for dirpath, dirnames, filenames in os.walk(repo_path): + # Prune in place so os.walk does not descend into skipped trees. + dirnames[:] = [d for d in dirnames if d not in skipped] - suffix = f.suffix.lower() - if suffix in extensions: - lang = extensions[suffix] - counts[lang] = counts.get(lang, 0) + 1 + for filename in filenames: + suffix = os.path.splitext(filename)[1].lower() + lang = extensions.get(suffix) + if lang is not None: + counts[lang] = counts.get(lang, 0) + 1 if not counts: raise ValueError( @@ -68,7 +86,19 @@ def detect_language(repo_path: str) -> str: "Supported languages: Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, Zig." ) - return max(counts, key=counts.get) + return dict(sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))) + + +def detect_language(repo_path: str) -> str: + """Auto-detect the primary (dominant) language of a repository. + + Preserved verbatim in signature and in the ``ValueError`` contract so every + existing caller and test is unaffected by the multi-language work. + + Returns: + One of: "python", "javascript", "go", "c", "ruby", "php", "zig" + """ + return next(iter(detect_languages(repo_path))) def parse_repository( @@ -128,28 +158,185 @@ def parse_repository( language = detect_language(repo_path) print(f" Auto-detected language: {language}", file=sys.stderr) - # Dispatch to the right parser - if language == "python": - result = _parse_python(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - elif language == "javascript": - result = _parse_javascript(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - elif language == "go": - result = _parse_go(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - elif language == "c": - result = _parse_c(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - elif language == "ruby": - result = _parse_ruby(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - elif language == "php": - result = _parse_php(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - elif language == "zig": - result = _parse_zig(repo_path, output_dir, processing_level, skip_tests, name, library_mode) - else: - raise ValueError(f"Unsupported language: {language}") + # Dispatch to the right parser via the registry. + try: + parser = _parser_for(language) + except KeyError: + raise ValueError( + f"Unsupported language: {language}. " + f"Supported: {', '.join(supported_languages())}" + ) from None + + result = parser(repo_path, output_dir, processing_level, skip_tests, name, library_mode) _maybe_apply_diff_filter(result, output_dir, diff_manifest) return result +@dataclass +class LanguageParseOutcome: + """Result of parsing ONE language during a multi-language fan-out. + + Failures are data, not exceptions, because one broken toolchain must not + cost the user every other language in the repo. + + Attributes: + language: Registry language name. + ok: Whether the parse succeeded. + output_dir: The per-language directory written to. + dataset_path: Path to this language's dataset.json, if produced. + analyzer_output_path: Path to analyzer_output.json, if produced. + units_count: Units parsed. + duration_seconds: Wall-clock time for this language. + error: Failure message, when ``ok`` is False. + error_type: Coarse failure class, for reporting and triage. + """ + + language: str + ok: bool + output_dir: str + dataset_path: str | None = None + analyzer_output_path: str | None = None + units_count: int = 0 + duration_seconds: float = 0.0 + error: str | None = None + error_type: str | None = None + + def to_dict(self) -> dict: + return asdict(self) + + +def _classify_parse_error(exc: BaseException) -> str: + """Coarse failure class for a per-language parse error.""" + if isinstance(exc, subprocess.TimeoutExpired): + return "timeout" + if isinstance(exc, FileNotFoundError): + return "missing_dependency" + if isinstance(exc, OSError): + return "os_error" + if isinstance(exc, ValueError): + return "unsupported_language" + return "parser_failed" + + +def parse_repository_multi( + repo_path: str, + run_dir: str, + languages: list[str], + processing_level: str = "reachable", + skip_tests: bool = True, + name: str = None, + fresh: bool = False, + library_mode: bool = False, + strict: bool = False, +) -> list[LanguageParseOutcome]: + """Parse a repository once per language into per-language directories. + + Every parser writes the SAME flat filenames — ``dataset.json``, + ``analyzer_output.json``, ``call_graph.json``, ``scan_result(s).json``, + ``functions.json``, ``pipeline_results.json`` — into whatever output + directory it is handed. Running two languages into one directory therefore + means the second silently overwrites the first. Giving each language its own + ``//`` is the whole reason this function exists, and it + matches the layout the Go CLI already assumes via + ``config.ScanDir(project, sha, language)``. + + **Sequential by design.** This loop must not be parallelised without first + moving cost tracking off the process-global tracker: ``step_context`` + computes usage deltas against it, and concurrent languages would interleave + those deltas and silently corrupt every per-step ``cost_usd``. Two further + reasons: the Python parser runs in-process and mutates ``sys.path``, and + running six tree-sitter/Node/Go parsers at once on a monorepo is a + realistic OOM — which would lose every language, the exact outcome the + partial-success handling below exists to prevent. + + Args: + repo_path: Repository to parse. + run_dir: Run root. Per-language output goes in ``//``. + languages: Languages to parse, in order. + processing_level: "all", "reachable", "codeql" or "exploitable". + skip_tests: Exclude test files. + name: Dataset name override. + fresh: Delete each language's existing dataset.json first. + library_mode: Seed the public API surface as entry points. + strict: Re-raise the first per-language failure instead of continuing. + + Returns: + One :class:`LanguageParseOutcome` per requested language, in order. + + Raises: + ValueError: If ``languages`` is empty. + RuntimeError: If EVERY language failed, aggregating each error. + """ + if not languages: + raise ValueError("parse_repository_multi requires at least one language") + + repo_path = os.path.abspath(repo_path) + run_dir = os.path.abspath(run_dir) + + outcomes: list[LanguageParseOutcome] = [] + + for language in languages: + output_dir = os.path.join(run_dir, language) + started = time.monotonic() + + try: + result = parse_repository( + repo_path=repo_path, + output_dir=output_dir, + language=language, + processing_level=processing_level, + skip_tests=skip_tests, + name=name, + fresh=fresh, + library_mode=library_mode, + ) + except (RuntimeError, subprocess.TimeoutExpired, OSError, ValueError) as exc: + # Deliberately NOT a bare `except Exception`: a KeyboardInterrupt or + # MemoryError mid-fan-out must abort the run, not be logged as + # "this language failed" and then repeated for five more languages. + outcomes.append(LanguageParseOutcome( + language=language, + ok=False, + output_dir=output_dir, + duration_seconds=time.monotonic() - started, + error=str(exc), + error_type=_classify_parse_error(exc), + )) + print( + f" [ERROR] {language} parser failed: {exc} — " + "continuing with remaining languages", + file=sys.stderr, + ) + if strict: + raise + continue + + outcomes.append(LanguageParseOutcome( + language=language, + ok=True, + output_dir=output_dir, + dataset_path=result.dataset_path, + analyzer_output_path=result.analyzer_output_path, + units_count=result.units_count, + duration_seconds=time.monotonic() - started, + )) + + if not any(o.ok for o in outcomes): + detail = "; ".join(f"{o.language}: {o.error}" for o in outcomes) + raise RuntimeError(f"All {len(outcomes)} language parser(s) failed. {detail}") + + failed = [o for o in outcomes if not o.ok] + if failed: + print( + f"[Parser] DEGRADED: {len(failed)} of {len(outcomes)} language(s) failed " + f"({', '.join(o.language for o in failed)}). Results are incomplete.", + file=sys.stderr, + ) + + return outcomes + + def _maybe_apply_diff_filter( result: ParseResult, output_dir: str, @@ -547,199 +734,60 @@ def _file_lock(lock_path: Path): f.close() -def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: - """Invoke the JavaScript/TypeScript parser. - - The JS parser is a PipelineTest class that runs Node.js subprocesses. - We invoke it via subprocess to avoid the sys.path hacks. - """ - _ensure_js_parser_dependencies() - - print("[Parser] Running JavaScript parser...", file=sys.stderr) - - parser_script = _CORE_ROOT / "parsers" / "javascript" / "test_pipeline.py" - - # Build command — analyzer-path now defaults to co-located file in the parser - cmd = [ - sys.executable, str(parser_script), - repo_path, - "--output", output_dir, - "--processing-level", processing_level, - ] - - if name: - cmd.extend(["--name", name]) - if skip_tests: - cmd.append("--skip-tests") - if library_mode: - cmd.append("--library-mode") - - result = subprocess.run( - cmd, - stdout=sys.stderr, - stderr=sys.stderr, - cwd=str(_CORE_ROOT), - timeout=1800, # 30 min — parity with the C/Ruby/PHP/Zig parse subprocesses - ) - - if result.returncode != 0: - raise RuntimeError(f"JavaScript parser failed with exit code {result.returncode}") - - dataset_path = os.path.join(output_dir, "dataset.json") - analyzer_output_path = os.path.join(output_dir, "analyzer_output.json") - - # Count units - units_count = 0 - if os.path.exists(dataset_path): - data = read_json(dataset_path) - units_count = len(data.get("units", [])) - - print(f" JavaScript parser complete: {units_count} units", file=sys.stderr) - - return ParseResult( - dataset_path=dataset_path, - analyzer_output_path=analyzer_output_path if os.path.exists(analyzer_output_path) else None, - units_count=units_count, - language="javascript", - processing_level=processing_level, - ) - - -# --------------------------------------------------------------------------- -# Go parser -# --------------------------------------------------------------------------- - -def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: - """Invoke the Go parser. - - The Go parser is a PipelineTest class that calls a compiled Go binary. - We invoke it via subprocess. - """ - print("[Parser] Running Go parser...", file=sys.stderr) - - parser_script = _CORE_ROOT / "parsers" / "go" / "test_pipeline.py" - - cmd = [ - sys.executable, str(parser_script), - repo_path, - "--output", output_dir, - "--processing-level", processing_level, - ] - - if name: - cmd.extend(["--name", name]) - if skip_tests: - cmd.append("--skip-tests") - if library_mode: - cmd.append("--library-mode") - - result = subprocess.run( - cmd, - stdout=sys.stderr, - stderr=sys.stderr, - cwd=str(_CORE_ROOT), - timeout=1800, # 30 min — parity with the C/Ruby/PHP/Zig parse subprocesses - ) - - if result.returncode != 0: - raise RuntimeError(f"Go parser failed with exit code {result.returncode}") - - dataset_path = os.path.join(output_dir, "dataset.json") - analyzer_output_path = os.path.join(output_dir, "analyzer_output.json") - - # Count units - units_count = 0 - if os.path.exists(dataset_path): - data = read_json(dataset_path) - units_count = len(data.get("units", [])) +def _parse_via_subprocess( + language: str, + repo_path: str, + output_dir: str, + processing_level: str, + skip_tests: bool = True, + name: str = None, + library_mode: bool = False, +) -> ParseResult: + """Invoke a language's parser as a subprocess. - print(f" Go parser complete: {units_count} units", file=sys.stderr) + Every non-Python parser shares one argv contract:: - return ParseResult( - dataset_path=dataset_path, - analyzer_output_path=analyzer_output_path if os.path.exists(analyzer_output_path) else None, - units_count=units_count, - language="go", - processing_level=processing_level, - ) +