From ed211df8dbb4144406189c334c3d07329d6107fb Mon Sep 17 00:00:00 2001 From: Joe Esquibel Date: Sun, 30 Aug 2026 21:37:12 -0400 Subject: [PATCH 1/2] refactor(standards): split language_standards.py into a per-language package The single-file LANGUAGE_DEFINITIONS registry had grown to 14,403 lines / 942KB across 59 languages, the largest file in the repo and the one place that hadn't adopted the per-language-file convention already used by tests/extraction/languages/ and docs/language_status/. Every consumer (detector.py, language_lens.py, prism.py, galaxyscope.py, the supply-chain tools, ~150+ tests) only ever accesses LANGUAGE_DEFINITIONS as an already-merged, in-memory dict via constructor injection or whole-dict import -- nothing depends on the file being physically monolithic. gitgalaxy/standards/language_standards.py is now a package: gitgalaxy/standards/language_standards/__init__.py assembles the exact same merged dict (same 59 keys, same order) from one file per language under languages/, plus sibling modules for the genuinely cross-language pieces (_lens_config.py, _prism_config.py, _shared_patterns.py's GLOBAL_* debt/AI-SDK detectors, _overrides.py's PROJECT_OVERRIDES). Every existing import path (`from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS`, etc.) is unchanged. The split was done as a source-text slice (ast.get_source_segment per top-level dict entry against the git-committed source), not a load-then-repr() round-trip, to preserve every in-block comment and avoid any risk of altering regex literals. Verified via: - a deep-compare script confirming every language's rules dict (patterns + flags) and LENS_CONFIG/PRISM_CONFIG/PROJECT_OVERRIDES/ HTML_NONEXECUTABLE_SCRIPT_TAG are byte-for-byte equivalent to the pre-split file, plus key order preserved - a comment-fidelity check confirming every in-block comment survived - full pytest suite (7138 passed, 0 test files touched) - ruff format/check clean, audit_check.py baselines unchanged - a fresh `pip install -e` sanity check (catches a missing package __init__.py that a stale editable install would mask) - a self-scan confirming no new sec_hardcoded_secrets/secrets_risk findings - tree_sitter_accuracy_audit.py --summary-table and tri_comparison_chart.py --all --ci both clean against the repointed path Companion changes: XRAY_BYPASS_PATHS' exact-file entry -> a directory prefix (binary_anomaly_detector.py's bypass check is substring-based, so this still covers every new file); tree_sitter_accuracy_audit.py's summary-table path repointed to the new __init__.py; four GitHub workflow files' path triggers/exact-file checks updated (tree-sitter-accuracy-audit/ -history, tri-comparison-audit/-history) since they gate on or auto-commit to this exact path; how_to_add_a_language.md's registration steps, the standards README's module breakdown, and the harden-language-extraction skill + two scout subagents updated to describe the new per-language file layout. crucible_check.py and tree_sitter_accuracy_audit.py --all --ci both show pre-existing drift in this environment (confirmed via control runs against unmodified origin/main producing the identical drift) -- local golden-master graph-layout non-determinism and a stale local language-crucible corpus checkout, respectively. Neither is caused by or related to this change; both are orthogonal environment issues. Co-Authored-By: Claude Sonnet 5 --- .claude/agents/class-start-scout.md | 4 +- .claude/agents/strict-signature-scout.md | 2 +- .../harden-language-extraction/SKILL.md | 7 +- .../workflows/tree-sitter-accuracy-audit.yml | 6 +- .../tree-sitter-accuracy-history.yml | 12 +- .github/workflows/tri-comparison-audit.yml | 2 +- .github/workflows/tri-comparison-history.yml | 4 +- gitgalaxy/standards/README.md | 2 +- gitgalaxy/standards/gitgalaxy_config.py | 2 +- gitgalaxy/standards/how_to_add_a_language.md | 26 +- gitgalaxy/standards/language_standards.py | 14587 ---------------- .../standards/language_standards/__init__.py | 204 + .../language_standards/_lens_config.py | 131 + .../language_standards/_overrides.py | 107 + .../language_standards/_prism_config.py | 31 + .../language_standards/_shared_patterns.py | 120 + .../language_standards/languages/__init__.py | 9 + .../language_standards/languages/abap.py | 260 + .../language_standards/languages/ada.py | 365 + .../languages/agc_assembly.py | 250 + .../language_standards/languages/apex.py | 290 + .../language_standards/languages/assembly.py | 291 + .../language_standards/languages/batch.py | 21 + .../language_standards/languages/blp.py | 21 + .../language_standards/languages/c.py | 330 + .../language_standards/languages/cobol.py | 385 + .../language_standards/languages/cpp.py | 480 + .../language_standards/languages/csharp.py | 483 + .../language_standards/languages/css.py | 258 + .../language_standards/languages/csv.py | 28 + .../language_standards/languages/dart.py | 438 + .../languages/dockerfile.py | 309 + .../languages/embedded_python.py | 270 + .../language_standards/languages/fortran.py | 426 + .../language_standards/languages/glsl.py | 21 + .../language_standards/languages/go.py | 343 + .../language_standards/languages/groovy.py | 335 + .../language_standards/languages/haskell.py | 363 + .../language_standards/languages/hlo.py | 32 + .../language_standards/languages/html.py | 339 + .../language_standards/languages/java.py | 375 + .../languages/javascript.py | 376 + .../language_standards/languages/jcl.py | 124 + .../language_standards/languages/json.py | 46 + .../language_standards/languages/kotlin.py | 309 + .../language_standards/languages/livecode.py | 388 + .../language_standards/languages/lua.py | 236 + .../language_standards/languages/m4.py | 205 + .../language_standards/languages/makefile.py | 273 + .../language_standards/languages/markdown.py | 51 + .../language_standards/languages/matlab.py | 266 + .../language_standards/languages/mlir.py | 33 + .../language_standards/languages/nix.py | 21 + .../languages/objectivec.py | 347 + .../language_standards/languages/pbtxt.py | 33 + .../language_standards/languages/perl.py | 385 + .../language_standards/languages/php.py | 334 + .../language_standards/languages/plaintext.py | 62 + .../languages/powershell.py | 369 + .../language_standards/languages/proto.py | 39 + .../language_standards/languages/python.py | 354 + .../language_standards/languages/ruby.py | 307 + .../language_standards/languages/rust.py | 318 + .../language_standards/languages/scala.py | 347 + .../language_standards/languages/scheme.py | 274 + .../language_standards/languages/shell.py | 339 + .../language_standards/languages/solidity.py | 219 + .../language_standards/languages/sqlite.py | 366 + .../language_standards/languages/swift.py | 304 + .../language_standards/languages/tcl.py | 200 + .../language_standards/languages/td.py | 39 + .../languages/typescript.py | 694 + .../language_standards/languages/xml.py | 43 + .../language_standards/languages/yacc.py | 129 + .../language_standards/languages/yaml.py | 173 + .../language_standards/languages/zig.py | 221 + tests/tools/tree_sitter_accuracy_audit.py | 4 +- 77 files changed, 15583 insertions(+), 14614 deletions(-) delete mode 100644 gitgalaxy/standards/language_standards.py create mode 100644 gitgalaxy/standards/language_standards/__init__.py create mode 100644 gitgalaxy/standards/language_standards/_lens_config.py create mode 100644 gitgalaxy/standards/language_standards/_overrides.py create mode 100644 gitgalaxy/standards/language_standards/_prism_config.py create mode 100644 gitgalaxy/standards/language_standards/_shared_patterns.py create mode 100644 gitgalaxy/standards/language_standards/languages/__init__.py create mode 100644 gitgalaxy/standards/language_standards/languages/abap.py create mode 100644 gitgalaxy/standards/language_standards/languages/ada.py create mode 100644 gitgalaxy/standards/language_standards/languages/agc_assembly.py create mode 100644 gitgalaxy/standards/language_standards/languages/apex.py create mode 100644 gitgalaxy/standards/language_standards/languages/assembly.py create mode 100644 gitgalaxy/standards/language_standards/languages/batch.py create mode 100644 gitgalaxy/standards/language_standards/languages/blp.py create mode 100644 gitgalaxy/standards/language_standards/languages/c.py create mode 100644 gitgalaxy/standards/language_standards/languages/cobol.py create mode 100644 gitgalaxy/standards/language_standards/languages/cpp.py create mode 100644 gitgalaxy/standards/language_standards/languages/csharp.py create mode 100644 gitgalaxy/standards/language_standards/languages/css.py create mode 100644 gitgalaxy/standards/language_standards/languages/csv.py create mode 100644 gitgalaxy/standards/language_standards/languages/dart.py create mode 100644 gitgalaxy/standards/language_standards/languages/dockerfile.py create mode 100644 gitgalaxy/standards/language_standards/languages/embedded_python.py create mode 100644 gitgalaxy/standards/language_standards/languages/fortran.py create mode 100644 gitgalaxy/standards/language_standards/languages/glsl.py create mode 100644 gitgalaxy/standards/language_standards/languages/go.py create mode 100644 gitgalaxy/standards/language_standards/languages/groovy.py create mode 100644 gitgalaxy/standards/language_standards/languages/haskell.py create mode 100644 gitgalaxy/standards/language_standards/languages/hlo.py create mode 100644 gitgalaxy/standards/language_standards/languages/html.py create mode 100644 gitgalaxy/standards/language_standards/languages/java.py create mode 100644 gitgalaxy/standards/language_standards/languages/javascript.py create mode 100644 gitgalaxy/standards/language_standards/languages/jcl.py create mode 100644 gitgalaxy/standards/language_standards/languages/json.py create mode 100644 gitgalaxy/standards/language_standards/languages/kotlin.py create mode 100644 gitgalaxy/standards/language_standards/languages/livecode.py create mode 100644 gitgalaxy/standards/language_standards/languages/lua.py create mode 100644 gitgalaxy/standards/language_standards/languages/m4.py create mode 100644 gitgalaxy/standards/language_standards/languages/makefile.py create mode 100644 gitgalaxy/standards/language_standards/languages/markdown.py create mode 100644 gitgalaxy/standards/language_standards/languages/matlab.py create mode 100644 gitgalaxy/standards/language_standards/languages/mlir.py create mode 100644 gitgalaxy/standards/language_standards/languages/nix.py create mode 100644 gitgalaxy/standards/language_standards/languages/objectivec.py create mode 100644 gitgalaxy/standards/language_standards/languages/pbtxt.py create mode 100644 gitgalaxy/standards/language_standards/languages/perl.py create mode 100644 gitgalaxy/standards/language_standards/languages/php.py create mode 100644 gitgalaxy/standards/language_standards/languages/plaintext.py create mode 100644 gitgalaxy/standards/language_standards/languages/powershell.py create mode 100644 gitgalaxy/standards/language_standards/languages/proto.py create mode 100644 gitgalaxy/standards/language_standards/languages/python.py create mode 100644 gitgalaxy/standards/language_standards/languages/ruby.py create mode 100644 gitgalaxy/standards/language_standards/languages/rust.py create mode 100644 gitgalaxy/standards/language_standards/languages/scala.py create mode 100644 gitgalaxy/standards/language_standards/languages/scheme.py create mode 100644 gitgalaxy/standards/language_standards/languages/shell.py create mode 100644 gitgalaxy/standards/language_standards/languages/solidity.py create mode 100644 gitgalaxy/standards/language_standards/languages/sqlite.py create mode 100644 gitgalaxy/standards/language_standards/languages/swift.py create mode 100644 gitgalaxy/standards/language_standards/languages/tcl.py create mode 100644 gitgalaxy/standards/language_standards/languages/td.py create mode 100644 gitgalaxy/standards/language_standards/languages/typescript.py create mode 100644 gitgalaxy/standards/language_standards/languages/xml.py create mode 100644 gitgalaxy/standards/language_standards/languages/yacc.py create mode 100644 gitgalaxy/standards/language_standards/languages/yaml.py create mode 100644 gitgalaxy/standards/language_standards/languages/zig.py diff --git a/.claude/agents/class-start-scout.md b/.claude/agents/class-start-scout.md index b186f6dba..2482c49a7 100644 --- a/.claude/agents/class-start-scout.md +++ b/.claude/agents/class-start-scout.md @@ -1,13 +1,13 @@ --- name: class-start-scout -description: Mechanical diff-reporting and verification-sequence execution for epic #1295 (extending gitgalaxy/core/detector.py's _CLASS_START_NAMED_EXTRACTION_LANGS allowlist). Runs tests/tools/class_start_diff.py and the tree_sitter_accuracy_audit.py/crucible_check.py/audit_check.py verification sequence, reports structured results -- it does not judge whether an extra/missing name is a real bug, decide how to fix a regex, or edit the allowlist itself. Use before a per-language triage pass (to get one language's exact extra/missing name diff without reading language_standards.py cold) and after a regex fix + allowlist flip (to run the full verification sequence and report pass/fail). +description: Mechanical diff-reporting and verification-sequence execution for epic #1295 (extending gitgalaxy/core/detector.py's _CLASS_START_NAMED_EXTRACTION_LANGS allowlist). Runs tests/tools/class_start_diff.py and the tree_sitter_accuracy_audit.py/crucible_check.py/audit_check.py verification sequence, reports structured results -- it does not judge whether an extra/missing name is a real bug, decide how to fix a regex, or edit the allowlist itself. Use before a per-language triage pass (to get one language's exact extra/missing name diff without reading language_standards/languages/.py cold) and after a regex fix + allowlist flip (to run the full verification sequence and report pass/fail). tools: Bash, Read model: haiku --- You run scripts and report their output cleanly. You do not judge whether a `class_start` regex is buggy, decide whether an extra/missing name is a real bug versus a ground-truth scope mismatch, -or edit `gitgalaxy/core/detector.py`/`gitgalaxy/standards/language_standards.py` yourself -- that +or edit `gitgalaxy/core/detector.py`/`gitgalaxy/standards/language_standards/` yourself -- that judgment belongs to whoever asked for the scout pass (the main conversation or a case-authoring subagent). If something looks ambiguous, report it plainly rather than guessing what it means. diff --git a/.claude/agents/strict-signature-scout.md b/.claude/agents/strict-signature-scout.md index e288fdad7..113d10672 100644 --- a/.claude/agents/strict-signature-scout.md +++ b/.claude/agents/strict-signature-scout.md @@ -1,6 +1,6 @@ --- name: strict-signature-scout -description: Mechanical gap-reporting and candidate-verification for epic #1069 (hardening tests/extraction/languages/test__strict.py). Runs tests/extraction/tools/audit_strict_coverage.py and tests/extraction/tools/verify_candidates.py and reports structured results -- it does not draft test cases or judge realism, only executes scripts and formats their output. Use before a case-writing pass (to get one language's exact gap list without loading language_standards.py cold) and after drafting candidate cases (to batch-verify them against the real compiled regex before they're written into a test file). +description: Mechanical gap-reporting and candidate-verification for epic #1069 (hardening tests/extraction/languages/test__strict.py). Runs tests/extraction/tools/audit_strict_coverage.py and tests/extraction/tools/verify_candidates.py and reports structured results -- it does not draft test cases or judge realism, only executes scripts and formats their output. Use before a case-writing pass (to get one language's exact gap list without loading language_standards/languages/.py cold) and after drafting candidate cases (to batch-verify them against the real compiled regex before they're written into a test file). tools: Bash, Read model: haiku --- diff --git a/.claude/skills/harden-language-extraction/SKILL.md b/.claude/skills/harden-language-extraction/SKILL.md index 2cc721dcd..a54641333 100644 --- a/.claude/skills/harden-language-extraction/SKILL.md +++ b/.claude/skills/harden-language-extraction/SKILL.md @@ -1,6 +1,6 @@ --- name: harden-language-extraction -description: Deepen or fix a language's structural-extraction accuracy in gitgalaxy/standards/language_standards.py (func_start/args/class_start/_dependency_capture regexes) using the epic #813 methodology. Use when the user asks to "harden extraction for X", "add strict tests for language X", "find bugs in X's parsing", or similar per-language regex-correctness work -- not for adding a brand-new language from scratch (that's how_to_add_a_language.md's LLM generation prompt) and not for the broader ReDoS/boundary-correctness rules covered there either. +description: Deepen or fix a language's structural-extraction accuracy in gitgalaxy/standards/language_standards/languages/.py (func_start/args/class_start/_dependency_capture regexes) using the epic #813 methodology. Use when the user asks to "harden extraction for X", "add strict tests for language X", "find bugs in X's parsing", or similar per-language regex-correctness work -- not for adding a brand-new language from scratch (that's how_to_add_a_language.md's LLM generation prompt) and not for the broader ReDoS/boundary-correctness rules covered there either. --- Source of truth is `tests/extraction/how_to_harden_extraction.md` -- read it directly, don't work @@ -12,7 +12,8 @@ a language-specific quirk. ## Process (translating that doc's 5-stage pipeline to this tool's Agent tool) -1. **Load the target.** Read the language's full `rules` dict from `language_standards.py` and its +1. **Load the target.** Read the language's full `rules` dict from + `gitgalaxy/standards/language_standards/languages/.py` and its existing cases in `tests/extraction/languages/test_.py` (or the old monolithic `test_*_extraction_strict.py` files if it hasn't migrated yet). Work one language, all four gauntlets, in one sitting -- not gauntlet-by-gauntlet across languages. @@ -30,7 +31,7 @@ a language-specific quirk. unrealistic payload), not an automatic fix. 5. **Fix real bugs with the full discipline**: ReDoS scaling check on any quantifier change, `python tests/tools/audit_check.py` (add `--regenerate` for pure line-shifts), then - `pytest tests/extraction/languages/test_.py`. If the fix touches `language_standards.py`, + `pytest tests/extraction/languages/test_.py`. If the fix touches `language_standards/`, `detector.py`, or `prism.py`: `python tests/tools/crucible_check.py` -- confirm any real diff is confined to the language(s) actually changed (check `_dependency_capture` fixes for legitimate cross-language DAG ripple, per the doc's note) before `--update --yes`. diff --git a/.github/workflows/tree-sitter-accuracy-audit.yml b/.github/workflows/tree-sitter-accuracy-audit.yml index aa071304c..f2722c56f 100644 --- a/.github/workflows/tree-sitter-accuracy-audit.yml +++ b/.github/workflows/tree-sitter-accuracy-audit.yml @@ -18,7 +18,7 @@ on: paths: - "gitgalaxy/core/detector.py" - "gitgalaxy/core/prism.py" - - "gitgalaxy/standards/language_standards.py" + - "gitgalaxy/standards/language_standards/**" - "tests/tools/tree_sitter_accuracy_audit.py" - "tests/tree_sitter_accuracy_baseline_*.json" - ".github/workflows/tree-sitter-accuracy-audit.yml" @@ -63,8 +63,8 @@ jobs: run: | cd gitgalaxy python tests/tools/tree_sitter_accuracy_audit.py --summary-table - if ! git diff --exit-code -- gitgalaxy/standards/language_standards.py; then - echo "::error::gitgalaxy/standards/language_standards.py's summary table is stale." + if ! git diff --exit-code -- gitgalaxy/standards/language_standards/__init__.py; then + echo "::error::gitgalaxy/standards/language_standards/__init__.py's summary table is stale." echo "Run 'python tests/tools/tree_sitter_accuracy_audit.py --summary-table' and commit the result." exit 1 fi diff --git a/.github/workflows/tree-sitter-accuracy-history.yml b/.github/workflows/tree-sitter-accuracy-history.yml index 4c52a9e4f..429be57ab 100644 --- a/.github/workflows/tree-sitter-accuracy-history.yml +++ b/.github/workflows/tree-sitter-accuracy-history.yml @@ -10,7 +10,7 @@ name: Tree-sitter Accuracy History # baselined language again and append one row each to # docs/self_scan/tree_sitter_accuracy_history.csv (an accumulating time series, unlike the # gitignored self-scan DB -- this is meant to be graphed later), regenerate the Markdown -# summary table embedded in language_standards.py's docstring so it can't drift from the +# summary table embedded in language_standards/__init__.py's docstring so it can't drift from the # committed baselines between now and the next manual --summary-table run, and re-render # docs/self_scan/tree_sitter_accuracy_chart.svg (a small-multiples bar chart of the most # recent history batch -- always the one this same job just appended). Opens an auto-merged @@ -19,7 +19,7 @@ name: Tree-sitter Accuracy History # protection), and why the loop guard below is load-bearing. # # Adaptive by design, not just by trigger path: the trigger below fires on any push touching -# detector.py/prism.py/language_standards.py, but not every such push actually moves a +# detector.py/prism.py/language_standards/, but not every such push actually moves a # language's measured accuracy. --history itself detects that (see # tree_sitter_accuracy_audit.py's `_batch_matches_measured`) and skips appending a row when the # fresh measurement is identical to the last recorded batch -- which means --chart re-renders @@ -34,7 +34,7 @@ on: paths: - "gitgalaxy/core/detector.py" - "gitgalaxy/core/prism.py" - - "gitgalaxy/standards/language_standards.py" + - "gitgalaxy/standards/language_standards/**" - "tests/tools/tree_sitter_accuracy_audit.py" - "tests/tree_sitter_accuracy_baseline_*.json" @@ -44,7 +44,7 @@ permissions: jobs: tree-sitter-accuracy-history: # Guards against the same infinite-loop shape documented in gitgalaxy.yml's - # full-report job: this workflow's own bot commit touches language_standards.py and + # full-report job: this workflow's own bot commit touches language_standards/__init__.py and # the baseline-adjacent history CSV, which are both in the trigger's `paths` filter, # so without this it would re-trigger itself on every merge of its own PR. if: | @@ -114,7 +114,7 @@ jobs: ${{ steps.blurbs.outputs.blurbs }} - Appends one row per language to `docs/self_scan/tree_sitter_accuracy_history.csv`. - - Regenerates the summary table in `gitgalaxy/standards/language_standards.py`. + - Regenerates the summary table in `gitgalaxy/standards/language_standards/__init__.py`. - Re-renders `docs/self_scan/tree_sitter_accuracy_chart.svg` from that same batch. Never touches `tests/tree_sitter_accuracy_baseline_*.json` -- those stay @@ -124,7 +124,7 @@ jobs: add-paths: | docs/self_scan/tree_sitter_accuracy_history.csv docs/self_scan/tree_sitter_accuracy_chart.svg - gitgalaxy/standards/language_standards.py + gitgalaxy/standards/language_standards/__init__.py # Same rationale as gitgalaxy.yml's "Merge the LLM Brief PR" step -- nothing else # gates this PR, so without an explicit merge it just sits mergeable-but-unmerged. diff --git a/.github/workflows/tri-comparison-audit.yml b/.github/workflows/tri-comparison-audit.yml index 58434bfa3..df025c2c1 100644 --- a/.github/workflows/tri-comparison-audit.yml +++ b/.github/workflows/tri-comparison-audit.yml @@ -24,7 +24,7 @@ on: paths: - "gitgalaxy/core/detector.py" - "gitgalaxy/core/prism.py" - - "gitgalaxy/standards/language_standards.py" + - "gitgalaxy/standards/language_standards/**" - "tests/tools/tri_comparison_chart.py" - "tests/tools/tri_comparison_gatherer.py" - "tests/tools/tri_comparison_reconcile.py" diff --git a/.github/workflows/tri-comparison-history.yml b/.github/workflows/tri-comparison-history.yml index 879463953..c83d8efb2 100644 --- a/.github/workflows/tri-comparison-history.yml +++ b/.github/workflows/tri-comparison-history.yml @@ -16,7 +16,7 @@ name: Tri-Comparison History # whatever actually changed -- same mechanism gitgalaxy.yml uses for the LLM architecture brief. # # Adaptive by design, not just by trigger path: the trigger below fires on any push touching -# detector.py/prism.py/language_standards.py, but not every such push moves a language's measured +# detector.py/prism.py/language_standards/, but not every such push moves a language's measured # numbers or an existing ledger entry's counts. Unlike the tree-sitter history CSV (an # append-only file that needs its own _batch_matches_measured dedup check to stay a no-op), the # three files here are overwritten in place -- "no diff" already IS the adaptivity, so @@ -29,7 +29,7 @@ on: paths: - "gitgalaxy/core/detector.py" - "gitgalaxy/core/prism.py" - - "gitgalaxy/standards/language_standards.py" + - "gitgalaxy/standards/language_standards/**" - "tests/tools/tri_comparison_chart.py" - "tests/tools/tri_comparison_gatherer.py" - "tests/tools/tri_comparison_reconcile.py" diff --git a/gitgalaxy/standards/README.md b/gitgalaxy/standards/README.md index bbf9c0bdd..4acb08fec 100644 --- a/gitgalaxy/standards/README.md +++ b/gitgalaxy/standards/README.md @@ -45,7 +45,7 @@ Each file in this directory serves a distinct calibration purpose for the downst * **`gitgalaxy_config.py` (Global Ingestion Firewall):** Defines Zero-Trust ingestion boundaries. It houses the Supply Chain Firewall configurations (approved vs. blacklisted imports), global file denylists, X-Ray binary scanner bypasses, and physical file-size clamps. * **`language_lens.py` (Identity Classifier):** The Bayesian engine that assigns definitive "Identity Locks" to files. It defines the multi-tiered confidence hierarchy, resolving extension collisions by weighing exact filename matches against Contextual Baselines. -* **`language_standards.py` (Structural Signature Registry):** The massive, highly optimized structural mapping registry for 50+ languages. It defines exactly how to slice a language into Branch Logic, State Flux, High-Risk Execution, and Object Declarations using ReDoS-proof regular expressions. +* **`language_standards/` (Structural Signature Registry):** A package with one file per language under `languages/` (59 today), assembled by `__init__.py` into the `LANGUAGE_DEFINITIONS` registry every downstream engine imports. Each per-language file defines exactly how to slice that language into Branch Logic, State Flux, High-Risk Execution, and Object Declarations using ReDoS-proof regular expressions; cross-language shared pieces (`LENS_CONFIG`, `PRISM_CONFIG`, the `GLOBAL_*` debt/AI-SDK detectors, `PROJECT_OVERRIDES`) live in their own sibling modules. See `how_to_add_a_language.md` for the registration mechanics. * **`analysis_lens.py` (Mathematical Constants & Threat Policies):** The repository of Threat Policies, Sigmoid Curve tuning, and K-Means clustering medians. It dictates how raw structural signals are mathematically converted into normalized 0-100% risk exposure vectors. * **`how_to_add_a_language.md` (Extension Protocol):** Contains the strict prompt engineering protocols required to generate ReDoS-proof language dictionaries using advanced LLMs, bypassing the need for manual parser development. diff --git a/gitgalaxy/standards/gitgalaxy_config.py b/gitgalaxy/standards/gitgalaxy_config.py index c178b3e73..6c153c015 100644 --- a/gitgalaxy/standards/gitgalaxy_config.py +++ b/gitgalaxy/standards/gitgalaxy_config.py @@ -99,7 +99,7 @@ "yarn.lock", "composer.lock", "gitgalaxy/core/aperture.py", - "gitgalaxy/standards/language_standards.py", + "gitgalaxy/standards/language_standards/", "gitgalaxy/security/security_lens.py", "gitgalaxy/tools/network_auditing/full_api_network_map.py", "gitgalaxy/tools/cobol_to_cobol/cobol_schema_forge.py", diff --git a/gitgalaxy/standards/how_to_add_a_language.md b/gitgalaxy/standards/how_to_add_a_language.md index b89bd1b1c..2cce3f365 100644 --- a/gitgalaxy/standards/how_to_add_a_language.md +++ b/gitgalaxy/standards/how_to_add_a_language.md @@ -14,16 +14,32 @@ To add a new language to the Language Classifier, you will use an advanced LLM ( --- ### Step 1: Initialize the LLM Context -Before asking the LLM to generate the new language signatures, upload the `gitgalaxy/standards/language_standards.py` file to the chat window. Issue this exact command: -> *"Read this file to understand how the GitGalaxy Structural Signature Analysis Engine uses bounded regex to guarantee ReDoS immunity. Pay close attention to how C++ and Python are mapped to prevent Catastrophic Backtracking."* +`gitgalaxy/standards/language_standards/` is a package with one file per language (each a +`DEFINITION: dict[str, Any] = {...}` literal, assembled into the `LANGUAGE_DEFINITIONS` registry +by `__init__.py`), not a single monolithic file. Before asking the LLM to generate new language +signatures, upload `gitgalaxy/standards/language_standards/__init__.py` plus one or two +representative per-language files (e.g. `languages/python.py` and `languages/cpp.py`) to the chat +window. Issue this exact command: +> *"Read these files to understand how the GitGalaxy Structural Signature Analysis Engine uses bounded regex to guarantee ReDoS immunity, and how each language's rules dict is registered in `__init__.py`. Pay close attention to how C++ and Python are mapped to prevent Catastrophic Backtracking."* ### Step 2: Inject the Structural Signature Prompt Copy the **Generation Prompt** below and paste it into the LLM. Replace `[TARGET LANGUAGE]` with the exact language you want to map. ### Step 3: Register the Signatures -1. Open `gitgalaxy/standards/language_standards.py`. -2. Locate the `LANGUAGE_DEFINITIONS` registry. -3. Paste the generated Python dictionary directly into the registry to instantly grant the engine native support for the new language architecture. +1. Create `gitgalaxy/standards/language_standards/languages/.py` containing + `DEFINITION: dict[str, Any] = {...}` with the generated dictionary as its value (see any + existing file in that directory for the exact shape — license header, `import re`, and a + `from .._shared_patterns import ...` line only if the rules reference one of the shared + `GLOBAL_*` cross-language constants). +2. In `gitgalaxy/standards/language_standards/__init__.py`, add an + `from .languages import as _` import (isort-sorted alphabetically among the + others) and a `"": _.DEFINITION,` line inside the `LANGUAGE_DEFINITIONS` dict, in + the same position other entries follow (insertion order is preserved end-to-end, so add new + languages at the end unless you have a specific reason not to). +3. If the language's key isn't a valid Python identifier (spaces, hyphens, etc. — the only + existing case is `"objective-c"`, whose module is `languages/objectivec.py`), the file's own + module name still needs to be identifier-safe even though the dict key itself doesn't; pick a + concatenated or underscored slug and keep the dict key exactly as the LLM/consumers expect it.

diff --git a/gitgalaxy/standards/language_standards.py b/gitgalaxy/standards/language_standards.py deleted file mode 100644 index 751d21f85..000000000 --- a/gitgalaxy/standards/language_standards.py +++ /dev/null @@ -1,14587 +0,0 @@ -# ============================================================================== -# GitGalaxy -# Copyright (c) 2026 Joe Esquibel -# -# This source code is licensed under the PolyForm Noncommercial License 1.0.0. -# You may not use this file except in compliance with the License. -# A copy of the license can be found in the LICENSE file in the root directory -# of this project, or at https://polyformproject.org/licenses/noncommercial/1.0.0/ -# ============================================================================== - -# galaxyscope:ignore sec_hardcoded_secrets, secrets_risk - -import re -from typing import Any, TypedDict - -""" -language_standards.py -Phase 2 & 3: The Lexical Registry & Syntax Dictionaries. - -This file contains the compiled regular expressions, mechanical delimiters, -and language-specific rules used to physically slice, parse, and identify -source code across the repository. - -## Tree-sitter Accuracy Audit Summary -This data is automatically generated by `tests/tools/tree_sitter_accuracy_audit.py --summary-table` -from the committed `tests/tree_sitter_accuracy_baseline_.json` files -- do not hand-edit the -table below, regenerate it instead (a stale table fails the tree-sitter-accuracy-audit CI check). -The script analyzes the GitGalaxy regex engine's performance against the empirical AST -ground truth provided by `tree_sitter_language_pack`, using the `language-crucible` corpus. -Recall is calculated as (found / real). Precision is calculated as (found / (found + extra)). -A language only appears once it has a committed baseline -- see docs/self_scan/tree_sitter_accuracy_history.csv -for the same metrics tracked over time across pushes to main. - - -| Language | Func Recall | Func Precision | Class Recall | Class Precision | -| -------- | ----------- | -------------- | ------------ | --------------- | -| Apex | 100.0% | 100.0% | 100.0% | 100.0% | -| C | 100.0% | 99.5% | 100.0% | 100.0% | -| Cpp | 100.0% | 100.0% | 100.0% | 100.0% | -| Csharp | 100.0% | 100.0% | 100.0% | 100.0% | -| Css | 100.0% | 100.0% | N/A | N/A | -| Dart | 100.0% | 100.0% | 100.0% | 100.0% | -| Fortran | 100.0% | 100.0% | 100.0% | 100.0% | -| Go | 100.0% | 100.0% | 100.0% | 100.0% | -| Groovy | N/A | 0.0% | N/A | N/A | -| Haskell | 100.0% | 99.3% | 100.0% | 100.0% | -| Html | 100.0% | 100.0% | N/A | N/A | -| Java | 100.0% | 100.0% | 100.0% | 100.0% | -| Javascript | 100.0% | 98.0% | 100.0% | 100.0% | -| Kotlin | 100.0% | 100.0% | 100.0% | 100.0% | -| Lua | 100.0% | 100.0% | N/A | 0.0% | -| Makefile | 100.0% | 100.0% | N/A | N/A | -| Matlab | 100.0% | 100.0% | N/A | N/A | -| Objective-C | 100.0% | 99.4% | 100.0% | 100.0% | -| Perl | 100.0% | 100.0% | 100.0% | 100.0% | -| Php | 100.0% | 99.9% | 100.0% | 100.0% | -| Powershell | 100.0% | 100.0% | 100.0% | 100.0% | -| Python | 100.0% | 99.6% | 100.0% | 100.0% | -| Ruby | 100.0% | 100.0% | 100.0% | 100.0% | -| Rust | 100.0% | 100.0% | 100.0% | 100.0% | -| Scala | 100.0% | 100.0% | 100.0% | 100.0% | -| Shell | 99.8% | 97.0% | N/A | N/A | -| Solidity | 100.0% | 94.3% | 100.0% | 100.0% | -| Swift | 100.0% | 99.2% | 100.0% | 100.0% | -| Tcl | 100.0% | 99.1% | N/A | N/A | -| Typescript | 100.0% | 100.0% | 100.0% | 100.0% | -| Zig | 100.0% | 100.0% | 100.0% | 100.0% | - -""" - - -class LensConfig(TypedDict): - # LENS_CONFIG's mixed set/dict/list values were widening to - # Collection[object] under mypy, so every .get()/.items() call on it - # throughout language_lens.py errored (#431). HANDSHAKE_REGISTRY's - # inner dicts stay Dict[str, Any] rather than their own TypedDict -- - # "pair" is None for two of the three current entries and a tuple for - # the third, and nothing here needs to type-check their contents, - # only LENS_CONFIG's own top-level shape. - COLLISION_FREQUENCIES: set[str] - PROSE_ANCHORS: set[str] - DISQUALIFIERS: dict[str, str] - HANDSHAKE_REGISTRY: list[dict[str, Any]] - THRESHOLDS: dict[str, float] - - -# ------------------------------------------------------------------------------ -# 1. STRUCTURAL SIGNATURE CONFIGURATION (Language Identification & Disambiguation) -# Consumed by: language_lens.py -# ------------------------------------------------------------------------------ -LENS_CONFIG: LensConfig = { - "COLLISION_FREQUENCIES": {".inc", ".h", ".py", ".cshtml", ".c", ".y", ".m"}, - "PROSE_ANCHORS": { - "README", - "LICENSE", - "LICENCE", - "CONTRIBUTING", - "CHANGELOG", - "AUTHORS", - "INSTALL", - "NOTICE", - "COPYING", - "TODO", - "FAQ", - "NOTES", - "CREDITS", - "HISTORY", - "MANIFEST", - "FILES", - "FILES2", - "ACKNOWLEDGEMENTS", - "AGREEMENT", - "CONTRIBUTORS", - "HACKING", - "HACKERS", - "AUTHOR", - "CHANGES", - "NEWS", - "RELEASE_NOTES", - "RELEASENOTES", - "UPGRADE", - "UPGRADING", - "VERSION", - "BUGS", - "FEATURES", - "ARCHITECTURE", - "DESIGN", - "GUIDE", - "USAGE", - "TUTORIAL", - "DOCS", - "CODE_OF_CONDUCT", - "SECURITY", - "SUPPORT", - "COPYRIGHT", - "PATENTS", - "LEGAL", - "THANKS", - "OWNERS", - "CODEOWNERS", - "MAINTAINERS", - "POSTAMBLE", - "README_BUFRLIB", - }, - "DISQUALIFIERS": { - "single_line_only": r"(?:^\s*using\s+namespace\b|^\s*public\s+(?:class|interface)\b|<\?php)", - "column_sensitive": r"(?:^\s*(?:import|export)\s+\{|", - "target": "javascript", - "pair": None, - }, - { - "trigger": r"^[ \t]*", - "target": "css", - "pair": None, - }, - { - # #1198: same drift #1183 fixed for ", + "target": "javascript", + "pair": None, + }, + { + "trigger": r"^[ \t]*", + "target": "css", + "pair": None, + }, + { + # #1198: same drift #1183 fixed for