From eb7236bc71aa607e1fd5ca54a06aa6d90b470ac8 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 1 Aug 2026 18:19:56 -0500 Subject: [PATCH 1/2] fix: separate schema constructor drift contexts --- .../python/detectors/dict_keys/schema.py | 46 ++++++- .../python/tests/test_py_dict_keys.py | 118 ++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) diff --git a/desloppify/languages/python/detectors/dict_keys/schema.py b/desloppify/languages/python/detectors/dict_keys/schema.py index b60c918b7..014c4f06c 100644 --- a/desloppify/languages/python/detectors/dict_keys/schema.py +++ b/desloppify/languages/python/detectors/dict_keys/schema.py @@ -63,6 +63,34 @@ def _extract_literal_keyset(node: ast.Dict) -> frozenset[str] | None: return frozenset(literal_keys) +def _call_context_name(node: ast.Call) -> str: + """Return a stable callee label for a call containing a dict literal.""" + + if isinstance(node.func, ast.Name): + return node.func.id + if isinstance(node.func, ast.Attribute): + return node.func.attr + return "" + + +def _literal_comparison_scope(node: ast.Dict, parents: dict[ast.AST, ast.AST]) -> str: + """Return the drift-comparison scope for a dict literal. + + A dict passed to a named ``*Schema`` constructor declares a validation + contract, whereas ordinary dict literals commonly carry runtime payloads. + Keep each schema constructor family out of the generic payload cluster; + preserve the detector's historical global comparison for every other + literal. + """ + + parent = parents.get(node) + if isinstance(parent, ast.Call): + callee_name = _call_context_name(parent) + if callee_name.endswith("Schema"): + return f"schema-constructor:{callee_name}" + return "generic" + + def _collect_schema_literals(files: list[str]) -> list[dict]: literals: list[dict] = [] for filepath in files: @@ -72,6 +100,11 @@ def _collect_schema_literals(files: list[str]) -> list[dict]: tree = _parse_python_ast(source, filepath=filepath) if tree is None: continue + parents = { + child: parent + for parent in ast.walk(tree) + for child in ast.iter_child_nodes(parent) + } for node in ast.walk(tree): if not isinstance(node, ast.Dict): @@ -79,7 +112,14 @@ def _collect_schema_literals(files: list[str]) -> list[dict]: keyset = _extract_literal_keyset(node) if keyset is None: continue - literals.append({"file": filepath, "line": node.lineno, "keys": keyset}) + literals.append( + { + "file": filepath, + "line": node.lineno, + "keys": keyset, + "comparison_scope": _literal_comparison_scope(node, parents), + } + ) return literals @@ -97,6 +137,10 @@ def _cluster_by_jaccard(literals: list[dict], *, threshold: float = 0.8) -> list if assigned[probe_idx]: continue candidate = literals[probe_idx] + if candidate.get("comparison_scope", "generic") != literal.get( + "comparison_scope", "generic" + ): + continue if any( _jaccard(member["keys"], candidate["keys"]) >= threshold for member in cluster diff --git a/desloppify/languages/python/tests/test_py_dict_keys.py b/desloppify/languages/python/tests/test_py_dict_keys.py index c4e8ef131..9ff268f4f 100644 --- a/desloppify/languages/python/tests/test_py_dict_keys.py +++ b/desloppify/languages/python/tests/test_py_dict_keys.py @@ -344,6 +344,124 @@ def test_no_drift_identical_dicts(self, tmp_path): entries, _ = detect_schema_drift(path) assert len(entries) == 0 + def test_does_not_compare_validation_schema_with_returned_payloads(self, tmp_path): + """Constructor schemas and runtime snapshots can intentionally differ.""" + code = textwrap.dedent("""\ + ValidationSchema( + { + "event_id": 1, + "requested_frequencies": [], + "analyzer_id": 9, + "frequency_range": [], + "existing_assignments": [], + "options": {}, + } + ) + + def legacy_snapshot(): + return { + "event_id": 1, + "requested_frequencies": [], + "frequency_range": [], + "existing_assignments": [], + "options": {}, + } + + def canonical_snapshot(): + return { + "event_id": 1, + "requested_frequencies": [], + "frequency_range": [], + "existing_assignments": [], + "options": {}, + } + + def normalized_snapshot(): + return { + "event_id": 1, + "requested_frequencies": [], + "frequency_range": [], + "existing_assignments": [], + "options": {}, + } + """) + path = _write_py(tmp_path, code) + + entries, count = detect_schema_drift(path) + + assert count == 4 + assert entries == [] + + def test_detects_drift_within_validation_schema_constructor(self, tmp_path): + code = textwrap.dedent("""\ + ValidationSchema( + {"event_id": 1, "requested_frequencies": [], "frequency_range": [], "options": {}} + ) + ValidationSchema( + {"event_id": 2, "requested_frequencies": [], "frequency_range": [], "options": {}} + ) + ValidationSchema( + {"event_id": 3, "requested_frequencies": [], "frequency_range": [], "options": {}} + ) + ValidationSchema( + { + "event_id": 4, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + "analyzer_id": 9, + } + ) + """) + path = _write_py(tmp_path, code) + + entries, count = detect_schema_drift(path) + + assert count == 4 + assert any(entry["key"] == "analyzer_id" for entry in entries) + + def test_detects_drift_within_returned_payloads(self, tmp_path): + code = textwrap.dedent("""\ + def first_snapshot(): + return { + "event_id": 1, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + } + + def second_snapshot(): + return { + "event_id": 2, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + } + + def third_snapshot(): + return { + "event_id": 3, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + } + + def fourth_snapshot(): + return { + "event_id": 4, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + "analyzer_id": 9, + } + """) + path = _write_py(tmp_path, code) + + entries, count = detect_schema_drift(path) + + assert count == 4 + assert any(entry["key"] == "analyzer_id" for entry in entries) + def test_too_few_literals_no_issues(self, tmp_path): code = textwrap.dedent("""\ d1 = {"name": "a", "age": 1, "city": "x"} From 6cacc8155e794b76fd2a52bc228f742d661eae04 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 1 Aug 2026 18:43:34 -0500 Subject: [PATCH 2/2] fix: reconcile invalidated schema drift findings --- .../app/commands/scan/plan_reconcile.py | 1 + desloppify/app/commands/scan/workflow.py | 5 + .../engine/_plan/scan_issue_reconcile.py | 30 ++++ desloppify/engine/_state/merge.py | 13 ++ desloppify/engine/_state/merge_issues.py | 78 ++++++++++ desloppify/languages/_framework/base/types.py | 1 + .../_framework/runtime_support/accessors.py | 10 ++ .../_framework/runtime_support/runtime.py | 1 + .../python/detectors/dict_keys/schema.py | 94 +++++++++++- desloppify/languages/python/phases_quality.py | 37 ++++- .../python/tests/test_py_dict_keys.py | 79 +++++++++- .../commands/scan/test_plan_reconcile.py | 25 +++ desloppify/tests/plan/test_reconcile.py | 19 +++ desloppify/tests/scoring/test_scoring.py | 16 +- desloppify/tests/state/test_state.py | 142 ++++++++++++++++++ 15 files changed, 536 insertions(+), 15 deletions(-) diff --git a/desloppify/app/commands/scan/plan_reconcile.py b/desloppify/app/commands/scan/plan_reconcile.py index ec54b7799..a78a7ae05 100644 --- a/desloppify/app/commands/scan/plan_reconcile.py +++ b/desloppify/app/commands/scan/plan_reconcile.py @@ -87,6 +87,7 @@ def _plan_has_user_content(plan: dict[str, object]) -> bool: or plan.get("overrides") or plan.get("clusters") or plan.get("skipped") + or plan.get("uncommitted_issues") ) diff --git a/desloppify/app/commands/scan/workflow.py b/desloppify/app/commands/scan/workflow.py index ad4329429..d7b2fc78b 100644 --- a/desloppify/app/commands/scan/workflow.py +++ b/desloppify/app/commands/scan/workflow.py @@ -470,6 +470,11 @@ def merge_scan_results( subjective_integrity_target=target_score, project_root=str(get_project_root()), zone_map=runtime.lang.zone_map if runtime.lang else None, + semantic_corrections=( + runtime.lang.semantic_corrections + if runtime.lang is not None and scan_path_rel == "." + else None + ), ), ) diff --git a/desloppify/engine/_plan/scan_issue_reconcile.py b/desloppify/engine/_plan/scan_issue_reconcile.py index f791c3e72..1ec733f67 100644 --- a/desloppify/engine/_plan/scan_issue_reconcile.py +++ b/desloppify/engine/_plan/scan_issue_reconcile.py @@ -367,6 +367,35 @@ def _sync_skipped_issue_statuses(plan: PlanModel, state: StateModel) -> None: issue["status"] = target_status +def _purge_semantic_correction_uncommitted( + plan: PlanModel, + state: StateModel, + *, + result: ReconcileResult, +) -> None: + """Drop system-invalidated findings from the pending commit ledger.""" + + issues = state.get("work_items") or state.get("issues", {}) + uncommitted = plan.get("uncommitted_issues", []) + retained: list[str] = [] + removed = 0 + for issue_id in uncommitted: + issue = issues.get(issue_id) + attestation = issue.get("resolution_attestation", {}) if issue else {} + if ( + issue + and issue.get("status") == "false_positive" + and isinstance(attestation, dict) + and attestation.get("kind") == "detector_semantic_correction" + ): + removed += 1 + continue + retained.append(issue_id) + if removed: + plan["uncommitted_issues"] = retained + result.changes += removed + + def reconcile_plan_after_scan( plan: PlanModel, state: StateModel, @@ -397,6 +426,7 @@ def reconcile_plan_after_scan( # Sync state status for issues in plan.skipped that are still "open" in state. # This migrates existing data: temporary skips → deferred, triaged_out skips → triaged_out. _sync_skipped_issue_statuses(plan, state) + _purge_semantic_correction_uncommitted(plan, state, result=result) _supersede_dead_references( plan, diff --git a/desloppify/engine/_state/merge.py b/desloppify/engine/_state/merge.py index 17be311b9..29bc72baf 100644 --- a/desloppify/engine/_state/merge.py +++ b/desloppify/engine/_state/merge.py @@ -20,6 +20,7 @@ _record_scan_metadata, ) from desloppify.engine._state.merge_issues import ( + apply_semantic_corrections, verify_disappeared, find_suspect_detectors, upsert_issues, @@ -149,6 +150,7 @@ class MergeScanOptions: subjective_integrity_target: float | None = None project_root: str | None = None zone_map: Any | None = None + semantic_corrections: dict[str, dict[str, str]] | None = None def merge_scan( @@ -219,6 +221,16 @@ def merge_scan( resolved_options.force_resolve, ran_detectors, ) + semantic_correction_ids = apply_semantic_corrections( + existing, + current_ids, + resolved_options.semantic_corrections, + now, + lang=resolved_options.lang, + scan_path=resolved_options.scan_path, + confirmed_detectors=confirmed_detectors, + suspect_detectors=suspect_detectors, + ) auto_resolved, skipped_other_lang, resolved_out_of_scope, resolve_changed = verify_disappeared( existing, current_ids, @@ -230,6 +242,7 @@ def merge_scan( project_root=resolved_options.project_root, zone_map=resolved_options.zone_map, confirmed_detectors=confirmed_detectors, + semantic_correction_ids=semantic_correction_ids, ) # Mark subjective assessments stale when mechanical issues changed. diff --git a/desloppify/engine/_state/merge_issues.py b/desloppify/engine/_state/merge_issues.py index 946ffc98f..55ed34e36 100644 --- a/desloppify/engine/_state/merge_issues.py +++ b/desloppify/engine/_state/merge_issues.py @@ -79,6 +79,80 @@ def _mark_scan_verified( existing["scan_verification_text"] = attestation_text +def apply_semantic_corrections( + existing: dict, + current_ids: set[str], + corrections: Mapping[str, Mapping[str, str]] | None, + now: str, + *, + lang: str | None, + scan_path: str | None, + confirmed_detectors: set[str], + suspect_detectors: set[str], +) -> set[str]: + """Apply detector-proven false-positive corrections from a full scan. + + Corrections are scanner-owned evidence, not a generic replacement for a + user disposition: the item must still be open, absent from the current + detector output, and produced by a confirmed non-suspect detector. + """ + + if scan_path != "." or not corrections: + return set() + + corrected_issue_ids: set[str] = set() + for issue_id, correction in corrections.items(): + previous = existing.get(issue_id) + if not isinstance(previous, dict) or issue_id in current_ids: + continue + if previous.get("status") != "open" or previous.get("suppressed"): + continue + detector = str(correction.get("detector", "")) + correction_kind = str(correction.get("kind", "")).strip() + correction_file = str(correction.get("file", "")).strip() + correction_key = str(correction.get("key", "")).strip() + correction_line = str(correction.get("line", "")).strip() + detail = previous.get("detail") + if ( + not detector + or not correction_kind + or not correction_file + or not correction_key + or not correction_line + or previous.get("detector") != detector + or previous.get("file") != correction_file + or detector not in confirmed_detectors + or detector in suspect_detectors + or not isinstance(detail, dict) + or detail.get("kind") != correction_kind + or str(detail.get("key", "")) != correction_key + or str(detail.get("line", "")) != correction_line + ): + continue + if lang and previous.get("lang") not in (None, lang): + continue + rule = str(correction.get("rule", "")).strip() + evidence = str(correction.get("evidence", "")).strip() + if not rule or not evidence: + continue + + previous.update( + status="false_positive", + resolved_at=now, + note=f"Detector semantic correction ({rule}): {evidence}", + resolution_attestation={ + "kind": "detector_semantic_correction", + "rule": rule, + "scan_verified": True, + "scan_verified_at": now, + "scan_verification_text": evidence, + }, + ) + corrected_issue_ids.add(issue_id) + + return corrected_issue_ids + + def verify_disappeared( existing: dict, current_ids: set[str], @@ -91,6 +165,7 @@ def verify_disappeared( project_root: str | None = None, zone_map=None, confirmed_detectors: set[str] | None = None, + semantic_correction_ids: set[str] | None = None, ) -> tuple[int, int, int, set[str]]: """Update scan corroboration for issues absent from scan. @@ -103,6 +178,8 @@ def verify_disappeared( resolved_detectors: set[str] = set() for issue_id, previous in existing.items(): + if semantic_correction_ids and issue_id in semantic_correction_ids: + continue previous_status = previous.get("status") if issue_id in current_ids or previous_status not in ( "open", @@ -325,6 +402,7 @@ def _suppression_metadata_from_state( __all__ = [ + "apply_semantic_corrections", "verify_disappeared", "find_suspect_detectors", "upsert_issues", diff --git a/desloppify/languages/_framework/base/types.py b/desloppify/languages/_framework/base/types.py index 41d5fe768..5c84239e3 100644 --- a/desloppify/languages/_framework/base/types.py +++ b/desloppify/languages/_framework/base/types.py @@ -88,6 +88,7 @@ class LangRuntimeContract(Protocol): subjective_assessments: dict[str, Any] detector_coverage: dict[str, DetectorCoverageRecord] coverage_warnings: list[DetectorCoverageRecord] + semantic_corrections: dict[str, dict[str, str]] def runtime_setting(self, key: str, default: Any = None) -> Any: ... diff --git a/desloppify/languages/_framework/runtime_support/accessors.py b/desloppify/languages/_framework/runtime_support/accessors.py index 9a7cf8bfd..58f96fb15 100644 --- a/desloppify/languages/_framework/runtime_support/accessors.py +++ b/desloppify/languages/_framework/runtime_support/accessors.py @@ -118,6 +118,16 @@ def coverage_warnings(self) -> list[DetectorCoverageRecord]: def coverage_warnings(self, value: list[DetectorCoverageRecord]) -> None: self.state.coverage_warnings = value + @property + def semantic_corrections(self) -> dict[str, dict[str, str]]: + """Detector-proven historical findings invalidated during this scan.""" + + return self.state.semantic_corrections + + @semantic_corrections.setter + def semantic_corrections(self, value: dict[str, dict[str, str]]) -> None: + self.state.semantic_corrections = value + @property def large_threshold(self) -> int: override = self.state.large_threshold_override diff --git a/desloppify/languages/_framework/runtime_support/runtime.py b/desloppify/languages/_framework/runtime_support/runtime.py index 83542b81d..c6d9646e3 100644 --- a/desloppify/languages/_framework/runtime_support/runtime.py +++ b/desloppify/languages/_framework/runtime_support/runtime.py @@ -36,6 +36,7 @@ class LangRuntimeState: props_threshold_override: int = 0 detector_coverage: dict[str, DetectorCoverageRecord] = field(default_factory=dict) coverage_warnings: list[DetectorCoverageRecord] = field(default_factory=list) + semantic_corrections: dict[str, dict[str, str]] = field(default_factory=dict) @dataclass diff --git a/desloppify/languages/python/detectors/dict_keys/schema.py b/desloppify/languages/python/detectors/dict_keys/schema.py index 014c4f06c..9fb68f220 100644 --- a/desloppify/languages/python/detectors/dict_keys/schema.py +++ b/desloppify/languages/python/detectors/dict_keys/schema.py @@ -14,6 +14,10 @@ logger = logging.getLogger(__name__) +_SCHEMA_CONSTRUCTOR_SCOPE_CORRECTION_RULE = ( + "python.dict_keys.schema_drift.schema_constructor_scope.v1" +) + def _jaccard(a: frozenset, b: frozenset) -> float: if not a and not b: @@ -123,7 +127,12 @@ def _collect_schema_literals(files: list[str]) -> list[dict]: return literals -def _cluster_by_jaccard(literals: list[dict], *, threshold: float = 0.8) -> list[list[dict]]: +def _cluster_by_jaccard( + literals: list[dict], + *, + threshold: float = 0.8, + respect_comparison_scope: bool = True, +) -> list[list[dict]]: """Greedy single-linkage clustering by Jaccard similarity threshold.""" clusters: list[list[dict]] = [] assigned = [False] * len(literals) @@ -137,9 +146,9 @@ def _cluster_by_jaccard(literals: list[dict], *, threshold: float = 0.8) -> list if assigned[probe_idx]: continue candidate = literals[probe_idx] - if candidate.get("comparison_scope", "generic") != literal.get( + if respect_comparison_scope and candidate.get( "comparison_scope", "generic" - ): + ) != literal.get("comparison_scope", "generic"): continue if any( _jaccard(member["keys"], candidate["keys"]) >= threshold @@ -207,13 +216,84 @@ def _build_schema_drift_issues(clusters: list[list[dict]]) -> list[dict]: return issues -def detect_schema_drift(path: Path) -> tuple[list[dict], int]: - """Cluster dict literals by key similarity and report outlier keys.""" +def _issue_locator(issue: dict) -> tuple[str, int, str]: + """Return the stable locator shared by schema-drift findings and literals.""" + + return str(issue["file"]), int(issue["line"]), str(issue["key"]) + + +def _semantic_correction_entries( + literals: list[dict], + scoped_issues: list[dict], +) -> list[dict]: + """Identify legacy-only findings invalidated by schema-constructor scope. + + Recreate only direct cross-scope clusters rooted at schema constructors. + That proves the legacy global comparison would have joined the literals + without repeating a second quadratic cluster pass over every generic + runtime payload in the repository. + """ + + scoped_locators = {_issue_locator(issue) for issue in scoped_issues} + corrections: list[dict] = [] + + for literal in literals: + comparison_scope = literal.get("comparison_scope", "generic") + if not comparison_scope.startswith("schema-constructor:"): + continue + cross_scope_cluster = [ + literal, + *[ + candidate + for candidate in literals + if candidate is not literal + and candidate.get("comparison_scope", "generic") != comparison_scope + and _jaccard(literal["keys"], candidate["keys"]) >= 0.8 + ], + ] + for issue in _build_schema_drift_issues([cross_scope_cluster]): + locator = _issue_locator(issue) + if locator in scoped_locators: + continue + if ( + locator[0] != literal["file"] + or locator[1] != literal["line"] + or locator[2] not in literal["keys"] + ): + continue + corrections.append( + { + "file": locator[0], + "line": locator[1], + "key": locator[2], + "tier": issue["tier"], + "confidence": issue["confidence"], + "summary": issue["summary"], + "comparison_scope": comparison_scope, + "rule": _SCHEMA_CONSTRUCTOR_SCOPE_CORRECTION_RULE, + } + ) + + return corrections + + +def detect_schema_drift_with_semantic_corrections( + path: Path, +) -> tuple[list[dict], int, list[dict]]: + """Detect drift and return proven legacy-only schema-scope corrections.""" files = find_py_files(path) all_literals = _collect_schema_literals(files) if len(all_literals) < 3: - return [], len(all_literals) + return [], len(all_literals), [] clusters = _cluster_by_jaccard(all_literals, threshold=0.8) issues = _build_schema_drift_issues(clusters) - return issues, len(all_literals) + corrections = _semantic_correction_entries(all_literals, issues) + return issues, len(all_literals), corrections + + +def detect_schema_drift(path: Path) -> tuple[list[dict], int]: + """Cluster dict literals by key similarity and report outlier keys.""" + + issues, checked, _corrections = detect_schema_drift_with_semantic_corrections(path) + return issues, checked diff --git a/desloppify/languages/python/phases_quality.py b/desloppify/languages/python/phases_quality.py index 0826c378f..eea57db90 100644 --- a/desloppify/languages/python/phases_quality.py +++ b/desloppify/languages/python/phases_quality.py @@ -10,6 +10,9 @@ from desloppify.languages._framework.issue_factories import make_smell_issues from desloppify.languages._framework.base.types import LangRuntimeContract from desloppify.languages.python.detectors import dict_keys as dict_keys_detector_mod +from desloppify.languages.python.detectors.dict_keys.schema import ( + detect_schema_drift_with_semantic_corrections, +) from desloppify.languages.python.detectors import ( import_linter_adapter as import_linter_adapter_mod, ) @@ -117,7 +120,9 @@ def phase_dict_keys(path: Path, lang: LangRuntimeContract) -> tuple[list[Issue], ) ) - drift_entries, _ = dict_keys_detector_mod.detect_schema_drift(path) + drift_entries, _, semantic_correction_entries = ( + detect_schema_drift_with_semantic_corrections(path) + ) drift_entries = filter_entries(lang.zone_map, drift_entries, "dict_keys") for entry in drift_entries: results.append( @@ -137,6 +142,36 @@ def phase_dict_keys(path: Path, lang: LangRuntimeContract) -> tuple[list[Issue], ) ) + semantic_correction_entries = filter_entries( + lang.zone_map, + semantic_correction_entries, + "dict_keys", + ) + semantic_corrections = getattr(lang, "semantic_corrections", None) + if isinstance(semantic_corrections, dict): + for entry in semantic_correction_entries: + correction_issue = make_issue( + "dict_keys", + entry["file"], + f"schema_drift::{entry['key']}::{entry['line']}", + tier=entry["tier"], + confidence=entry["confidence"], + summary=entry["summary"], + ) + semantic_corrections[correction_issue["id"]] = { + "detector": "dict_keys", + "kind": "schema_drift", + "file": correction_issue["file"], + "key": entry["key"], + "line": str(entry["line"]), + "rule": entry["rule"], + "evidence": ( + f"{entry['file']}:{entry['line']} is " + f"{entry['comparison_scope']} and is absent from " + "the scoped schema-drift output" + ), + } + log(f" -> {len(results)} dict key issues") return results, { "dict_keys": adjust_potential(lang.zone_map, files_checked), diff --git a/desloppify/languages/python/tests/test_py_dict_keys.py b/desloppify/languages/python/tests/test_py_dict_keys.py index 9ff268f4f..d432449ee 100644 --- a/desloppify/languages/python/tests/test_py_dict_keys.py +++ b/desloppify/languages/python/tests/test_py_dict_keys.py @@ -2,12 +2,17 @@ import textwrap from pathlib import Path +from types import SimpleNamespace +from desloppify.languages.python import phases_quality as phases_quality_mod from desloppify.languages.python.detectors import dict_keys as dict_keys_mod from desloppify.languages.python.detectors.dict_keys import ( detect_dict_key_flow, detect_schema_drift, ) +from desloppify.languages.python.detectors.dict_keys.schema import ( + detect_schema_drift_with_semantic_corrections, +) # ── Helpers ──────────────────────────────────────────────── @@ -387,21 +392,44 @@ def normalized_snapshot(): """) path = _write_py(tmp_path, code) - entries, count = detect_schema_drift(path) + entries, count, corrections = ( + detect_schema_drift_with_semantic_corrections(path) + ) assert count == 4 assert entries == [] + assert len(corrections) == 1 + assert corrections[0]["key"] == "analyzer_id" + assert ( + corrections[0]["comparison_scope"] + == "schema-constructor:ValidationSchema" + ) def test_detects_drift_within_validation_schema_constructor(self, tmp_path): code = textwrap.dedent("""\ ValidationSchema( - {"event_id": 1, "requested_frequencies": [], "frequency_range": [], "options": {}} + { + "event_id": 1, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + } ) ValidationSchema( - {"event_id": 2, "requested_frequencies": [], "frequency_range": [], "options": {}} + { + "event_id": 2, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + } ) ValidationSchema( - {"event_id": 3, "requested_frequencies": [], "frequency_range": [], "options": {}} + { + "event_id": 3, + "requested_frequencies": [], + "frequency_range": [], + "options": {}, + } ) ValidationSchema( { @@ -415,10 +443,13 @@ def test_detects_drift_within_validation_schema_constructor(self, tmp_path): """) path = _write_py(tmp_path, code) - entries, count = detect_schema_drift(path) + entries, count, corrections = ( + detect_schema_drift_with_semantic_corrections(path) + ) assert count == 4 assert any(entry["key"] == "analyzer_id" for entry in entries) + assert corrections == [] def test_detects_drift_within_returned_payloads(self, tmp_path): code = textwrap.dedent("""\ @@ -457,10 +488,13 @@ def fourth_snapshot(): """) path = _write_py(tmp_path, code) - entries, count = detect_schema_drift(path) + entries, count, corrections = ( + detect_schema_drift_with_semantic_corrections(path) + ) assert count == 4 assert any(entry["key"] == "analyzer_id" for entry in entries) + assert corrections == [] def test_too_few_literals_no_issues(self, tmp_path): code = textwrap.dedent("""\ @@ -472,6 +506,39 @@ def test_too_few_literals_no_issues(self, tmp_path): # Fewer than 3 literals -> no issues assert len(entries) == 0 + def test_phase_records_proven_semantic_correction(self, monkeypatch, tmp_path): + correction_entry = { + "file": "schemas.py", + "line": 12, + "key": "analyzer_id", + "tier": 3, + "confidence": "medium", + "summary": "Legacy-only schema drift", + "comparison_scope": "schema-constructor:ValidationSchema", + "rule": "python.dict_keys.schema_drift.schema_constructor_scope.v1", + } + monkeypatch.setattr( + phases_quality_mod.dict_keys_detector_mod, + "detect_dict_key_flow", + lambda _path: ([], 1), + ) + monkeypatch.setattr( + phases_quality_mod, + "detect_schema_drift_with_semantic_corrections", + lambda _path: ([], 1, [correction_entry]), + ) + lang = SimpleNamespace(zone_map=None, semantic_corrections={}) + + entries, potentials = phases_quality_mod.phase_dict_keys(tmp_path, lang) + + assert entries == [] + assert potentials == {"dict_keys": 1} + correction_id, correction = next(iter(lang.semantic_corrections.items())) + assert correction_id.endswith("::schema_drift::analyzer_id::12") + assert correction["kind"] == "schema_drift" + assert correction["rule"] == correction_entry["rule"] + assert "schema-constructor:ValidationSchema" in correction["evidence"] + # ── Output structure ────────────────────────────────────── diff --git a/desloppify/tests/commands/scan/test_plan_reconcile.py b/desloppify/tests/commands/scan/test_plan_reconcile.py index 6aa3afcc7..6a2573176 100644 --- a/desloppify/tests/commands/scan/test_plan_reconcile.py +++ b/desloppify/tests/commands/scan/test_plan_reconcile.py @@ -96,6 +96,11 @@ def test_plan_with_skipped(self): }} assert reconcile_mod._plan_has_user_content(plan) is True + def test_plan_with_uncommitted_issues(self): + plan = empty_plan() + plan["uncommitted_issues"] = ["issue-1"] + assert reconcile_mod._plan_has_user_content(plan) is True + def test_empty_collections_are_falsy(self): """Empty queue_order, overrides, clusters, skipped all return False.""" plan = empty_plan() @@ -211,6 +216,26 @@ def test_skips_when_no_user_content(self): changed = reconcile_mod._apply_plan_reconciliation(plan, state) assert changed is False + def test_purges_semantic_correction_from_uncommitted_only(self): + plan = empty_plan() + plan["uncommitted_issues"] = ["schema-drift"] + state = _make_state(issues={ + "schema-drift": _make_issue( + status="false_positive", + resolution_attestation={ + "kind": "detector_semantic_correction", + }, + ), + }) + + changed = reconcile_mod._apply_plan_reconciliation(plan, state) + + assert changed is True + assert plan["uncommitted_issues"] == [] + assert plan["queue_order"] == [] + assert plan["superseded"] == {} + assert plan["skipped"] == {} + # --------------------------------------------------------------------------- # Tests: _display_reconcile_results diff --git a/desloppify/tests/plan/test_reconcile.py b/desloppify/tests/plan/test_reconcile.py index 742ff50b0..754cc7741 100644 --- a/desloppify/tests/plan/test_reconcile.py +++ b/desloppify/tests/plan/test_reconcile.py @@ -196,6 +196,25 @@ def test_reconcile_supersedes_resolved_action_references(): assert "b" in plan["promoted_ids"] +def test_reconcile_supersedes_system_false_positive_without_skip(): + """Detector corrections leave a supersession record, never a user skip.""" + plan = _plan_with_queue("schema-drift") + ensure_plan_defaults(plan) + plan["uncommitted_issues"] = ["schema-drift"] + state = _state_with_issues("schema-drift", status="false_positive") + state["issues"]["schema-drift"]["resolution_attestation"] = { + "kind": "detector_semantic_correction", + } + + result = reconcile_plan_after_scan(plan, state) + + assert "schema-drift" in result.superseded + assert "schema-drift" not in plan["queue_order"] + assert "schema-drift" in plan["superseded"] + assert "schema-drift" not in plan["skipped"] + assert "schema-drift" not in plan["uncommitted_issues"] + + # --------------------------------------------------------------------------- # Active clusters completed when all items resolved # --------------------------------------------------------------------------- diff --git a/desloppify/tests/scoring/test_scoring.py b/desloppify/tests/scoring/test_scoring.py index 13c0884b4..2d8371ef6 100644 --- a/desloppify/tests/scoring/test_scoring.py +++ b/desloppify/tests/scoring/test_scoring.py @@ -507,6 +507,21 @@ def test_single_dimension_perfect(self): class TestComputeScoreBundle: + def test_auto_resolved_and_false_positive_have_distinct_strict_semantics(self): + auto_resolved = compute_score_bundle( + _issues_dict(_issue("unused", status="auto_resolved")), + {"unused": 10}, + ) + false_positive = compute_score_bundle( + _issues_dict(_issue("unused", status="false_positive")), + {"unused": 10}, + ) + + assert auto_resolved.strict_dimension_scores["Code quality"]["score"] == 90.0 + assert auto_resolved.verified_strict_dimension_scores["Code quality"]["score"] == 100.0 + assert false_positive.strict_dimension_scores["Code quality"]["score"] == 100.0 + assert false_positive.verified_strict_dimension_scores["Code quality"]["score"] == 90.0 + def test_bundle_mode_dimensions(self): issues = _issues_dict( _issue("unused", status="open", confidence="high"), @@ -881,4 +896,3 @@ def test_mixed_review_issues_excluded(self): # Subjective dimension scoring # =================================================================== - diff --git a/desloppify/tests/state/test_state.py b/desloppify/tests/state/test_state.py index 188a1fa68..359301ef1 100644 --- a/desloppify/tests/state/test_state.py +++ b/desloppify/tests/state/test_state.py @@ -672,6 +672,148 @@ def test_missing_fixed_issue_gets_scan_verified(self): assert "scan_verified_at" in st["issues"]["det::a.py::fn"]["resolution_attestation"] +# --------------------------------------------------------------------------- +# Detector semantic corrections +# --------------------------------------------------------------------------- + + +class TestDetectorSemanticCorrections: + _issue_id = "dict_keys::schemas.py::schema_drift::analyzer_id::12" + + @classmethod + def _corrections(cls) -> dict[str, dict[str, str]]: + return { + cls._issue_id: { + "detector": "dict_keys", + "kind": "schema_drift", + "file": "schemas.py", + "key": "analyzer_id", + "line": "12", + "rule": "python.dict_keys.schema_drift.schema_constructor_scope.v1", + "evidence": "schemas.py:12 is schema-constructor:ValidationSchema", + } + } + + @classmethod + def _state_with_open_issue(cls): + state = empty_state() + issue = _make_raw_issue( + cls._issue_id, + detector="dict_keys", + file="schemas.py", + lang="python", + ) + issue["detail"] = { + "kind": "schema_drift", + "key": "analyzer_id", + "line": 12, + } + state["issues"][issue["id"]] = issue + return state + + def test_full_confirmed_scan_marks_proven_correction_false_positive(self): + state = self._state_with_open_issue() + + diff = merge_scan( + state, + [], + MergeScanOptions( + lang="python", + scan_path=".", + potentials={"dict_keys": 20}, + semantic_corrections=self._corrections(), + ), + ) + + issue = state["issues"][self._issue_id] + assert diff["auto_resolved"] == 0 + assert issue["status"] == "false_positive" + assert issue["resolution_attestation"]["kind"] == "detector_semantic_correction" + assert issue["resolution_attestation"]["scan_verified"] is True + assert "schema_constructor_scope.v1" in issue["note"] + + def test_correction_requires_full_confirmed_scan(self): + state = self._state_with_open_issue() + + merge_scan( + state, + [], + MergeScanOptions( + lang="python", + scan_path="src", + potentials={"dict_keys": 20}, + semantic_corrections=self._corrections(), + ), + ) + + assert state["issues"][self._issue_id]["status"] == "open" + + def test_correction_requires_the_detector_to_be_confirmed(self): + state = self._state_with_open_issue() + + merge_scan( + state, + [], + MergeScanOptions( + lang="python", + scan_path=".", + potentials={"smells": 20}, + semantic_corrections=self._corrections(), + ), + ) + + assert state["issues"][self._issue_id]["status"] == "open" + + def test_correction_requires_the_exact_schema_drift_fingerprint(self): + state = self._state_with_open_issue() + state["issues"][self._issue_id]["detail"]["line"] = 13 + + merge_scan( + state, + [], + MergeScanOptions( + lang="python", + scan_path=".", + potentials={"dict_keys": 20}, + semantic_corrections=self._corrections(), + ), + ) + + assert state["issues"][self._issue_id]["status"] == "auto_resolved" + + def test_reemitted_issue_remains_open(self): + state = self._state_with_open_issue() + correction_options = MergeScanOptions( + lang="python", + scan_path=".", + potentials={"dict_keys": 20}, + semantic_corrections=self._corrections(), + ) + merge_scan(state, [], correction_options) + assert state["issues"][self._issue_id]["status"] == "false_positive" + + current = _make_raw_issue( + self._issue_id, + detector="dict_keys", + file="schemas.py", + lang="python", + ) + + merge_scan( + state, + [current], + MergeScanOptions( + lang="python", + scan_path=".", + potentials={"dict_keys": 20}, + ), + ) + + issue = state["issues"][self._issue_id] + assert issue["status"] == "open" + assert "resolution_attestation" not in issue + + # --------------------------------------------------------------------------- # #53: Wontfix auto-resolution via potentials (ran_detectors) # ---------------------------------------------------------------------------