From 7402442b870a986f084a7b1879237d5e33de5e33 Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Sun, 30 Aug 2026 15:50:51 -0400 Subject: [PATCH] core: shield string literals in positional comment stripper (#259) `Prism._strip_positional_comments` (the `positional_anchored` / `positional_abap` families -- cobol, fortran, abap) split each line directly on `*>` / `!` / `"` with no string-literal shielding, unlike every other stripping path in prism.py (`_strip_single_line_comments`, `_strip_segment_comments`, `_strip_nested_comments` all mask via `LITERAL_MASK_PATTERN` first). A delimiter-shaped character inside a literal -- `DISPLAY "Rate *> 5%"`, `PRINT *, "Warning!"`, ABAP `x = 'he said "hi"'` -- was misread as a real inline-comment marker: the tail of the literal plus the rest of the statement moved to the comment stream and `code_stream` was left with a dangling unterminated quote, deflating `coding_loc` and dropping real structural signals. Fix: mask literals per line (the same bounded discipline `_strip_single_line_comments` adopted for #1184) before the inline `*>` / `"` / `!` search, restoring them into both halves after the split. The column-1 / column-7 anchor check stays on the raw line so column positions are exact. ABAP masks single-quote / backtick only -- its `"` is the comment delimiter, not a string quote (new `ABAP_LITERAL_MASK_PATTERN`). `_mask_line_literals` gains an optional `pattern` override. Column-7 `-` continued literals remain out of scope (masking is deliberately line-bounded). Golden masters re-blessed (both modes): 5 diffs, all on `cobol/che-che4z_lsp_project_fixtures/special_schema.cbl`, whose DATA VALUE clauses hold XML-ish string literals containing `!` (`'Not here!'`). With the literal no longer truncated at the `!`, one previously-hidden `safety_bypasses` signal is now correctly retained (0 -> 1) and its downstream risk-exposure percentages shift. No line-shift noise anywhere else across the ~80-repo corpus. Closes #259 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni57RP5ATiqfN3XzJvRDPY --- gitgalaxy/core/prism.py | 55 ++++++++++++++++++------- tests/core_engine/test_prism.py | 48 +++++++++++++++++++++ tests/golden_master_audit.json | 14 +++---- tests/golden_master_zero_dep_audit.json | 14 +++---- 4 files changed, 101 insertions(+), 30 deletions(-) diff --git a/gitgalaxy/core/prism.py b/gitgalaxy/core/prism.py index 079c7e49e..506463bb9 100644 --- a/gitgalaxy/core/prism.py +++ b/gitgalaxy/core/prism.py @@ -133,6 +133,14 @@ def __init__( # "Android/*"`, swallowed ~17 handlers). Single-line, no-escape. self.MULTI_STYLE_LIVE_LITERAL_MASK_PATTERN = r'("[^"\r\n]*")' + # #259: ABAP's `"` is a comment delimiter, never a string quote -- ABAP + # string literals are `'...'` and string templates use backtick. Masking + # with the shared SHIELD_PATTERN's `"..."` branch in _strip_positional_ + # comments would swallow an ABAP comment that happens to contain an inner + # `"..."` pair as if it were a live literal, so ABAP's positional stripper + # masks single-quote / backtick literals only. + self.ABAP_LITERAL_MASK_PATTERN = r"((?), and ABAP (") comments - if "*>" in line: - parts = line.split("*>", 1) - code.append(parts[0]) - lits.append("*>" + parts[1]) - elif abap_mode and '"' in line: - parts = line.split('"', 1) - code.append(parts[0]) - lits.append('"' + parts[1]) - elif not abap_mode and "!" in line: + # 2. Modern Inline Fortran (!), COBOL (*>), and ABAP (") comments. + # #259: mask string/char literals first -- per line, the same bounded + # discipline _strip_single_line_comments adopted for #1184 -- so a + # delimiter-shaped character INSIDE a literal (`DISPLAY "Rate *> 5%"`, + # `PRINT *, "Warning!"`, ABAP `x = 'he said "hi"'`) isn't mistaken for + # a real inline-comment marker, truncating the statement mid-literal + # and leaving code_stream with a dangling unterminated quote. Every + # other stripping path in this file already shields this way; this one + # didn't. ABAP masks single-quote/backtick only (its `"` is the + # comment delimiter, not a quote -- see ABAP_LITERAL_MASK_PATTERN). + # A literal continued onto a later line via a fixed-form column-7 `-` + # indicator is out of scope: masking is deliberately line-bounded, so + # a delimiter in the continuation half stays unshielded (#1184). + mask_pat = self.ABAP_LITERAL_MASK_PATTERN if abap_mode else self.LITERAL_MASK_PATTERN + masked_line, masked_lits = self._mask_line_literals(line, mask_pat) + + if "*>" in masked_line: + head, tail = masked_line.split("*>", 1) + code.append(self._restore_masked_literals(head, masked_lits)) + lits.append(self._restore_masked_literals("*>" + tail, masked_lits)) + elif abap_mode and '"' in masked_line: + head, tail = masked_line.split('"', 1) + code.append(self._restore_masked_literals(head, masked_lits)) + lits.append(self._restore_masked_literals('"' + tail, masked_lits)) + elif not abap_mode and "!" in masked_line: # #1911: `!` has no comment meaning in ABAP at all (its only # real markers are `*` in column 1 and `"` inline) -- it's # the classic ABAP formal-parameter-name escape prefix @@ -1017,9 +1040,9 @@ def _strip_positional_comments( # signature's IMPORTING/EXPORTING/CHANGING clause. Without # this gate every `!param` line was truncated at the `!`, # erasing the parameter name and its TYPE clause. - parts = line.split("!", 1) - code.append(parts[0]) - lits.append("!" + parts[1]) + head, tail = masked_line.split("!", 1) + code.append(self._restore_masked_literals(head, masked_lits)) + lits.append(self._restore_masked_literals("!" + tail, masked_lits)) else: code.append(line) lits.append("") @@ -1242,15 +1265,15 @@ def _mask_perl_line(line: str, masked_literals: list[str]) -> str: return "\n".join(code), "\n".join(comments) - def _mask_line_literals(self, line: str) -> tuple[str, list[str]]: - """Replaces each string/char literal on a single line with a `__MASK_N__` placeholder, returning the masked line and the literals in match order.""" + def _mask_line_literals(self, line: str, pattern: Optional[str] = None) -> tuple[str, list[str]]: + """Replaces each string/char literal on a single line with a `__MASK_N__` placeholder, returning the masked line and the literals in match order. `pattern` overrides the default LITERAL_MASK_PATTERN (e.g. #259's ABAP mask, which must not treat `"` as a quote).""" masked_literals: list[str] = [] def shield_callback(m: re.Match) -> str: masked_literals.append(m.group(0)) return f"__MASK_{len(masked_literals) - 1}__" - return re.sub(self.LITERAL_MASK_PATTERN, shield_callback, line), masked_literals + return re.sub(pattern or self.LITERAL_MASK_PATTERN, shield_callback, line), masked_literals def _restore_masked_literals(self, masked: str, masked_literals: list[str]) -> str: """Reverses _mask_line_literals, substituting each `__MASK_N__` placeholder back for its original literal text.""" diff --git a/tests/core_engine/test_prism.py b/tests/core_engine/test_prism.py index 18a0e8be1..c7bfd88e2 100644 --- a/tests/core_engine/test_prism.py +++ b/tests/core_engine/test_prism.py @@ -232,6 +232,54 @@ def test_prism_positional_anchors(prism_engine): assert "This is an inline comment" in docs +# #259: _strip_positional_comments split directly on `*>` / `!` / `"` with no +# string-literal shielding, unlike every other stripper in prism.py -- a +# delimiter-shaped char inside a literal was misread as a real inline comment, +# truncating the statement and leaving code_stream with a dangling quote. +def test_prism_positional_delimiter_inside_literal_is_shielded(prism_engine): + """#259: `*>` / `!` inside a COBOL/Fortran string literal is not a comment.""" + # COBOL `*>` inside a "..." literal + code, lits = prism_engine._strip_positional_comments( + ' DISPLAY "Rate *> 5%" TO CONSOLE.\n MOVE X TO Y.', cobol_mode=True + ) + assert code == ' DISPLAY "Rate *> 5%" TO CONSOLE.\n MOVE X TO Y.' + assert lits == "\n" + + # COBOL `!` inside a "..." literal + code, lits = prism_engine._strip_positional_comments( + ' DISPLAY "Warning!" TO CONSOLE.\n MOVE X TO Y.', cobol_mode=True + ) + assert code == ' DISPLAY "Warning!" TO CONSOLE.\n MOVE X TO Y.' + assert lits == "\n" + + # Fortran `!` inside a '...' literal + code, lits = prism_engine._strip_positional_comments(" PRINT *, 'Warning!'") + assert code == " PRINT *, 'Warning!'" + assert lits == "" + + +def test_prism_positional_real_inline_comment_still_stripped(prism_engine): + """#259 regression guard: a genuine inline `*>` / `!` comment (outside any + literal) must still be routed to the comment stream after the fix.""" + code, lits = prism_engine._strip_positional_comments( + ' MOVE "x" TO Y. *> real comment\n X = 1 ! also real', cobol_mode=True + ) + assert code.splitlines()[0].strip() == 'MOVE "x" TO Y.' + assert "real comment" in lits + assert code.splitlines()[1].strip() == "X = 1" + assert "also real" in lits + + +def test_prism_positional_abap_quote_inside_string_literal_is_shielded(prism_engine): + """#259: ABAP's `"` comment delimiter must not split a `'...'` literal that + contains a `"`; a real trailing `"` comment is still stripped.""" + code, lits = prism_engine._strip_positional_comments( + " result = 'he said \"hi\"'. \" trailing comment", abap_mode=True + ) + assert code.rstrip() == " result = 'he said \"hi\"'." + assert "trailing comment" in lits + + # ============================================================================== # TEST 5: HARDENED PYTHON DOCSTRINGS # ============================================================================== diff --git a/tests/golden_master_audit.json b/tests/golden_master_audit.json index 2e1999b33..241cc92c4 100644 --- a/tests/golden_master_audit.json +++ b/tests/golden_master_audit.json @@ -12,8 +12,8 @@ }, "Target Root Name": "data", "Absolute Project Path": "/srv/storage_16tb/projects/gitgalaxy/language-crucible/data", - "Analysis ISO Timestamp": "2026-08-30T17:26:41.339898+00:00", - "Total Scan Duration": "19.95 seconds" + "Analysis ISO Timestamp": "2026-08-30T19:44:27.267935+00:00", + "Total Scan Duration": "19.45 seconds" }, "Source Control Footprint (Immutable Anchor)": { "Active Branch": "HEAD", @@ -256,7 +256,7 @@ }, "health": { "avg_cognitive_load": 30.017, - "avg_safety_score": 47.674, + "avg_safety_score": 47.675, "avg_tech_debt": 39.553, "avg_documentation": 29.98 }, @@ -775,7 +775,7 @@ "total_mass": 868.13, "avg_exposures": { "cognitive_load": 18.82, - "safety_score": 50.47, + "safety_score": 50.51, "tech_debt": 51.02, "verification": 1.25, "api_exposure": 0.89, @@ -672145,7 +672145,7 @@ }, "Average Risk Exposures": { "Cognitive Load Exposure": "18.82%", - "Error & Exception Exposure": "50.47%", + "Error & Exception Exposure": "50.51%", "Tech Debt Exposure": "51.02%", "Testing Exposure": "1.25%", "API Exposure": "0.89%", @@ -683170,7 +683170,7 @@ }, "4. Vulnerability & Risk Exposures": { "Cognitive Load Exposure": "10.55%", - "Error & Exception Exposure": "62.98%", + "Error & Exception Exposure": "65.52%", "Tech Debt Exposure": "42.7%", "Testing Exposure": "2.37%", "API Exposure": "0.0%", @@ -683223,7 +683223,7 @@ "Function/Method Declarations": 3, "Class/Entity Declarations": 1, "Defensive Programming Constructs": 1, - "Type/Safety Bypasses": 0, + "Type/Safety Bypasses": 1, "High-Risk Execution Commands": 0, "I/O and Network Boundaries": 0, "Exposed API / Public Exports": 0, diff --git a/tests/golden_master_zero_dep_audit.json b/tests/golden_master_zero_dep_audit.json index da3e2ced1..a4c2d2da0 100644 --- a/tests/golden_master_zero_dep_audit.json +++ b/tests/golden_master_zero_dep_audit.json @@ -12,8 +12,8 @@ }, "Target Root Name": "data", "Absolute Project Path": "/srv/storage_16tb/projects/gitgalaxy/language-crucible/data", - "Analysis ISO Timestamp": "2026-08-30T17:27:08.124169+00:00", - "Total Scan Duration": "18.53 seconds" + "Analysis ISO Timestamp": "2026-08-30T19:45:09.149092+00:00", + "Total Scan Duration": "18.54 seconds" }, "Source Control Footprint (Immutable Anchor)": { "Active Branch": "HEAD", @@ -256,7 +256,7 @@ }, "health": { "avg_cognitive_load": 30.017, - "avg_safety_score": 47.674, + "avg_safety_score": 47.675, "avg_tech_debt": 39.553, "avg_documentation": 29.98 }, @@ -775,7 +775,7 @@ "total_mass": 868.13, "avg_exposures": { "cognitive_load": 18.82, - "safety_score": 50.47, + "safety_score": 50.51, "tech_debt": 51.02, "verification": 1.25, "api_exposure": 0.89, @@ -672145,7 +672145,7 @@ }, "Average Risk Exposures": { "Cognitive Load Exposure": "18.82%", - "Error & Exception Exposure": "50.47%", + "Error & Exception Exposure": "50.51%", "Tech Debt Exposure": "51.02%", "Testing Exposure": "1.25%", "API Exposure": "0.89%", @@ -683170,7 +683170,7 @@ }, "4. Vulnerability & Risk Exposures": { "Cognitive Load Exposure": "10.55%", - "Error & Exception Exposure": "62.98%", + "Error & Exception Exposure": "65.52%", "Tech Debt Exposure": "42.7%", "Testing Exposure": "2.37%", "API Exposure": "0.0%", @@ -683223,7 +683223,7 @@ "Function/Method Declarations": 3, "Class/Entity Declarations": 1, "Defensive Programming Constructs": 1, - "Type/Safety Bypasses": 0, + "Type/Safety Bypasses": 1, "High-Risk Execution Commands": 0, "I/O and Network Boundaries": 0, "Exposed API / Public Exports": 0,