From 6a384eaf5dabdc7a8362a33dfd32feca5e6a181a Mon Sep 17 00:00:00 2001 From: LaimaWu <317139567+LaimaWu@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:22:57 +0000 Subject: [PATCH 1/3] refactor(#358): move Python import edges to shared captures --- CHANGELOG.md | 18 +++++ queries/python/tags.scm | 19 +++++ src/ingest_importcap.h | 52 ++++++++++++- src/ingest_relations.h | 138 +---------------------------------- src/ingest_sidecap.h | 18 ++++- test/pyimportprecisecheck.sh | 137 ++++++++++++++++++++++++++++++++++ 6 files changed, 243 insertions(+), 139 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 837ce6a03..4049e8c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,24 @@ not published here — see `docs/EVALS.md` for the instruments behind the headli ## [Unreleased] +### Changed — Python imports use the shared capture vocabulary (#358, Python slice) + +Python dependency targets now come from `@import.path`; named clauses and their optional local +renames use `@import.names` and `@import.alias` to produce the existing import bindings. +The Python branches and binding extractor they replace leave `src/ingest_relations.h`. +Resolution stays unchanged. A comma-separated plain import still contributes only its first +dependency target, and an aliased first clause retains its written `as` spelling in `--deps`. +These existing limits are preserved rather than widened by the capture migration. + +The Python walk is removed. Captured targets and bindings retain the same container reach and depth limit: +imports at 255/256 containers remain, and a real import at 257 is cut and disclosed. An import-free deep +container, including a top-level import followed by a deep `pass`, no longer produces `extract-partial` +and may be cached. Disclosure now reports a captured import actually cut, rather than an empty container. +No persisted record or extraction-version change is intended; the Python import gate checks +fresh/warm output and, when `RIPWIRE_BASE_BIN` is supplied, acceptance of a pristine-main cache +without rewriting it. + + ## [0.6.7] — 2026-10-11 ### Fixed — `--impact`: a selector matching several definitions keeps the `via="name"` hedge in its per-file rollup diff --git a/queries/python/tags.scm b/queries/python/tags.scm index 489d38d7d..269e2b9c5 100644 --- a/queries/python/tags.scm +++ b/queries/python/tags.scm @@ -64,3 +64,22 @@ (attribute attribute: (identifier) @name) ]) @reference.call + +; Shared import vocabulary (#358). The dependency target is still ONE per statement: the first +; name of a plain import (including its alias spelling), or the module_name of a from-import. +; C++ keeps only the first name capture and imposes the old container reach/depth; queries are +; unanchored so conditional and function-body imports keep their existing coverage. +(import_statement . name: (_) @import.path) +(import_from_statement module_name: (_) @import.path) + +; Every named clause binds, even when it is not the first dependency target. @import.names is +; one member of that clause list per match; @import.alias is its optional local rename. Wildcards +; bind no name here, as before. Keep aliases in the SAME match as their named target. +(import_statement name: (dotted_name) @import.names) +(import_statement name: (aliased_import + name: (dotted_name) @import.names + alias: (identifier) @import.alias)) +(import_from_statement name: (dotted_name) @import.names) +(import_from_statement name: (aliased_import + name: (dotted_name) @import.names + alias: (identifier) @import.alias)) diff --git a/src/ingest_importcap.h b/src/ingest_importcap.h index b0cacc31f..b193284b7 100644 --- a/src/ingest_importcap.h +++ b/src/ingest_importcap.h @@ -23,8 +23,9 @@ // the bare name `require`, whether an argument is a single string literal. Those are normaliser arms, not // extractors — one per dialect, not one per language. // -// THE VOCABULARY IS A FAMILY, AND ONLY ITS FIRST MEMBER IS SPENT. `@import.path` is the only capture name -// in use, because C-family imports have no other part: `#include` names a path and nothing else — no +// THE VOCABULARY IS A FAMILY. Python is the first dialect to use `@import.names` / `@import.alias` +// for bindings only, never extra dependency targets. The names remain reserved for later dialects. +// `@import.path` is the dependency-target capture, because C-family imports have no other part: `#include` names a path and nothing else — no // alias to record, no list of imported names — and because Go's alias/dot/blank spellings ride inside the // declaration clause its Include has always carried (see DepDialect::Go below), not in a field of their own. // The other two names the issue sketches are reserved BY NAME so the next language lands on them rather @@ -228,7 +229,7 @@ ImportSpec normaliseGoImport( TSNode declaration, std::string_view src ) } // ── the dispatch: ONE normaliser per DepDialect ────────────────────────────────────────────────────── -// CFamily, Web and Go are the dialects that have moved over so far; every other arm returns an empty ImportSpec, +// CFamily, Web, Go and Python are the dialects that have moved over so far; every other arm returns an empty ImportSpec, // which is what makes the capture INERT rather than wrong on a language whose tags.scm has not adopted // `@import.path` yet. Each `case` is deleted as its language moves, and the compiler then names every // other dialect still to do — the same "one language at a time" landing the issue asks for. @@ -239,6 +240,7 @@ ImportSpec normaliseImportSpecifier( DepDialect dialect, TSNode directive, std:: case DepDialect::CFamily: return normaliseCFamilyImport( directive, raw, src ); case DepDialect::Web: return normaliseWebImport( directive, raw, src ); case DepDialect::Go: return normaliseGoImport( directive, src ); + case DepDialect::Python: return { std::string( raw ) }; // preserve the first plain clause's written alias default: return {}; // no other language captures @import.path yet } } @@ -253,6 +255,7 @@ inline bool importsFromCapture( DepDialect dialect ) noexcept case DepDialect::CFamily: case DepDialect::Web: case DepDialect::Go: + case DepDialect::Python: return true; default: return false; @@ -277,6 +280,49 @@ TSNode importDirectiveOf( DepDialect dialect, TSNode pathNode ) noexcept return directive; } +// One @import.names clause and its optional @import.alias → the existing Python Import binding. +// The query selects the clause; normalization keeps the plain-import module marker and from-import +// member distinction (#287). Both records stay at the STATEMENT byte, with the same empty spans. +// A multi-clause import still has only its FIRST Include, but every clause binds a name. Reach and +// depth must apply to bindings too, or a cut/ERROR-wrapped import would still narrow a call. +void emitCapturedPythonImportBind( TSNode namesNode, TSNode aliasNode, std::uint32_t fileId, std::string_view src, std::vector& binds ) +{ + const bool hasAlias = !ts_node_is_null( aliasNode ); + const TSNode clause = hasAlias ? ts_node_parent( namesNode ) : namesNode; + const TSNode stmt = ts_node_parent( clause ); + const bool isFrom = kindIs( ts_node_type( stmt ), "import_from_statement" ); + const ImportReach reach = importContainerReach( stmt, Lang::Python ); + if( reach.depth > kMaxImportContainerDepth || !reach.reachable ) + { + return; + } + // The old walk called its binding helper only after extracting a nonempty first target. + const TSNode first = fieldChild( stmt, isFrom ? NodeField::ModuleName : NodeField::Name ); + if( nodeTextOf( first, src ).empty() ) + { + return; + } + std::string_view bound = nodeTextOf( hasAlias ? aliasNode : namesNode, src ); + if( !isFrom && !hasAlias ) + { + bound = bound.substr( 0, bound.find( '.' ) ); // `import a.b` binds `a`, not `a.b` + } + std::string target = importSpecifierText( isFrom ? first : namesNode, src ); + if( bound.empty() || target.empty() ) + { + return; + } + RawBind b; + b.fileId = fileId; + b.startByte = ts_node_start_byte( stmt ); + b.lang = Lang::Python; + b.kind = LocalBindKind::Import; + b.var.assign( bound ); + b.typeName = std::move( target ); + b.importedName = isFrom ? std::string{} : "module"; + binds.push_back( std::move( b ) ); +} + // One captured `@import.path` → the Include record plus its ABS-3 import-role use-site ref, which is // exactly the pair captureIncludes' `emitDirective` lambda minted for a C-family directive. // diff --git a/src/ingest_relations.h b/src/ingest_relations.h index 77bab2833..ebf4a7de8 100644 --- a/src/ingest_relations.h +++ b/src/ingest_relations.h @@ -2633,8 +2633,7 @@ constexpr std::uint16_t kMaxImportContainerDepth = 256; // at all. Split out of captureIncludes so the walk that FINDS directives and the per-grammar table that // READS them stay separately readable — the walk is one shape, this is one branch per grammar spelling. // -// Node types confirmed per grammar: Python import_statement/import_from_statement; Go/Swift -// import_declaration; Rust use_declaration + mod_item; C# using_directive; plus the kParserVer-81 four (bash +// Node types confirmed per grammar: Swift import_declaration; Rust use_declaration + mod_item; C# using_directive; plus the kParserVer-81 four (bash // `command`, lua `function_call`, ruby `call`, elixir `call`). LEVER-B B0: non-C imports capture the // CLEAN written specifier via grammar child fields (module path / quoted specifier / use argument), not a // sliced clause — the sound resolver input. @@ -2647,7 +2646,7 @@ constexpr std::uint16_t kMaxImportContainerDepth = 256; // re-export, the CommonJS / dynamic call_expression) went the same way: `@import.path` in // queries/{typescript,tsx,javascript}/tags.scm, DepDialect::Web normaliser. What is left is every language // whose extraction needs the walk's own frame state — which is also the list of languages that have NOT -// moved over yet. +// moved over yet. Python's imports and bindings now use the capture path too (#358). // // `lang` exists for the language-gated branches: `call_expression` / `call` / `command` are node types in // most of our grammars, and a C++ or Rust function that happens to be named `require` must never @@ -2705,29 +2704,7 @@ DirectiveTarget directiveTargetOf( TSNode n, const char* t, std::string_view src bool isSymbolic = false; bool isReceiver = false; - if( kindIs( t, "import_statement" ) ) // Python `import a` - { - // Prefer the grammar's specifier field over slicing the whole statement (LEVER-B B0: the resolver - // needs the REAL written specifier, not the clause). Empirically confirmed node shape: - // Python: import_statement name:(dotted_name|aliased_import) → the dotted module `pkg.mod`. - // #358: the TS/JS `import … from 'x'` spelling of this node (and the `export … from 'x'` re-export that - // used to follow it) is GONE from this function — it is `@import.path` in queries/{typescript,tsx,javascript} - // /tags.scm now, read by the DepDialect::Web normaliser (src/ingest_importcap.h). - if( const TSNode nm = fieldChild( n, NodeField::Name ); !ts_node_is_null( nm ) ) - { - target = importSpecifierText( nm, src ); // Python: the dotted module head - } - } - else if( kindIs( t, "import_from_statement" ) ) // Python `from pkg.mod import Z` - { - // module_name:(dotted_name) → `pkg.mod`; module_name:(relative_import) → `.rel` / `..up` (leading - // dots preserved so the resolver can resolve relative-to-file). The imported-names clause is dropped. - if( const TSNode mn = fieldChild( n, NodeField::ModuleName ); !ts_node_is_null( mn ) ) - { - target = importSpecifierText( mn, src ); - } - } - else if( kindIs( t, "command" ) && lang == Lang::Bash ) // Bash `source x.sh` / `. x.sh` + if( kindIs( t, "command" ) && lang == Lang::Bash ) // Bash `source x.sh` / `. x.sh` { target = bashSourceTarget( n, src ); } @@ -2769,8 +2746,7 @@ DirectiveTarget directiveTargetOf( TSNode n, const char* t, std::string_view src { // No `source:`/`name:` field (this grammar exposes none on import_header — verified against // node-types.json, same reason kotlinEnclosingScopeOf walks positionally): find the flat - // `identifier` child and take its WHOLE span, mirroring Python's dotted-module-head branch - // above. That span already covers every dotted segment (`kotlin.math.max`), including the + // `identifier` child and take its WHOLE span, preserving the written dotted module head. That span already covers every dotted segment (`kotlin.math.max`), including the // wildcard case (`import a.b.*` has `identifier`="a.b" plus a separate wildcard_import // sibling this branch does not need to read) and the aliased case (`import a.b.C as D` has // `identifier`="a.b.C" plus a separate import_alias sibling naming "D" — the alias is not @@ -2843,108 +2819,6 @@ DirectiveTarget directiveTargetOf( TSNode n, const char* t, std::string_view src // use-site index reports import sites. The ref is file-scope (fromSymbol=kNoNode) — that is correct for // a directive at any container depth, and it NEVER enters the call graph (role != Call → skipped in // buildGraph). The ref's line comes from the DIRECTIVE node, never from its enclosing `#if` or body. -// Phase 5 (docs/EVALS.md "Phase 5", kParserVer 77): the NAME a Python import statement binds in the module -// namespace, per imported clause, as a file-scope LocalBindKind::Import RawBind (var = bound name, typeName = -// the module target as written). `import a.b as c` binds `c`; `import a.b` binds `a` (the head — that is the -// name Python puts in the namespace); `from m import x as y` binds `y`; `from m import x` binds `x`; a -// `wildcard_import` binds nothing. The Include record keeps only the module and drops the imported-names -// clause, so without this the bound NAME is unrecoverable resolve-side — and the external-name veto keys on -// exactly that name. Node shapes (tree-sitter-python v0.23.6): import_statement name:(dotted_name|aliased_import)+; -// import_from_statement module_name:(dotted_name|relative_import) name:(dotted_name|aliased_import)* | wildcard_import; -// aliased_import name:(dotted_name) alias:(identifier). startByte = the statement's own start (file scope → -// emitBindings attributes kNoNode); spans {0,0}. Pure-syntactic, deterministic, source order. -inline void capturePythonImportBinds( TSNode stmt, const char* t, std::uint32_t fileId, std::string_view src, std::vector& binds ) -{ - const bool isFrom = ( kindIs( t, "import_from_statement" ) ); - if( !isFrom && !kindIs( t, "import_statement" ) ) - { - return; - } - std::string target; - if( isFrom ) - { - const TSNode mn = fieldChild( stmt, NodeField::ModuleName ); - if( ts_node_is_null( mn ) ) - { - return; - } - target = importSpecifierText( mn, src ); - } - const TSNode moduleNode = isFrom ? fieldChild( stmt, NodeField::ModuleName ) : TSNode{}; - // One import statement's clause list: the count comes from the input (`from m import ( a, b, … )`), - // and a comment between two clauses is a further child, so the indexed form was O(children²) here too. - // No scaling arm exists for it (an import flood is not a shape any corpus produces) — this is the - // pure-iteration conversion, covered by the byte-identical arms (test/childwalkscalecheck.sh). - ChildCursor cursor( stmt ); - forEachChild( stmt, cursor.cur, [ & ]( TSNode kid ) - { - if( ts_node_is_null( kid ) ) - { - return true; - } - const char* kt = ts_node_type( kid ); - if( isFrom && ts_node_eq( kid, moduleNode ) ) - { - return true; // the module_name child of a from-import is not a bound name; only the `name:` clauses are - } - std::string_view bound; - std::string clauseTarget; - if( kindIs( kt, "aliased_import" ) ) - { - const TSNode alias = fieldChild( kid, NodeField::Alias ); - const TSNode nm = fieldChild( kid, NodeField::Name ); - if( ts_node_is_null( alias ) || ts_node_is_null( nm ) ) - { - return true; - } - bound = pattern::nodeText( alias, src ); - clauseTarget = isFrom ? target : importSpecifierText( nm, src ); - } - else if( kindIs( kt, "dotted_name" ) ) - { - const std::string_view whole = pattern::nodeText( kid, src ); - if( isFrom ) - { - bound = whole; // `from m import x` — a bare clause name is an identifier (a dotted one is a syntax error) - clauseTarget = target; - } - else - { - const std::size_t dot = whole.find( '.' ); - bound = ( dot == std::string_view::npos ) ? whole : whole.substr( 0, dot ); // `import a.b` binds `a` - clauseTarget = std::string( whole ); - } - } - else - { - return true; // keywords, punctuation, wildcard_import - } - if( bound.empty() || clauseTarget.empty() ) - { - return true; - } - RawBind b; - b.fileId = fileId; - b.startByte = ts_node_start_byte( stmt ); - b.lang = Lang::Python; - b.kind = LocalBindKind::Import; - b.var.assign( bound ); - b.typeName = std::move( clauseTarget ); - // issue #287 (kParserVer 115): `importedName` is otherwise unused by a Python Import bind (no - // reader keys on it before this — checked), so it carries ONE bit here: "module" iff `bound` names - // the MODULE itself (`import a.b` / `import a.b as c`, isFrom==false) — the shape a receiver-alias - // narrow may trust to mean "this variable's namespace IS the module's". Left empty for `from m - // import x [as y]` (isFrom==true): there `bound` names a MEMBER of m (a function, class, ufunc - // instance, …), and `bound.anything()` is a call on THAT member, not on m's namespace — numpy's - // `from numpy.core.numeric import greater_equal` then `greater_equal.outer(...)` is exactly this: - // `.outer` is a ufunc method, not numeric.py's free `outer`, and conflating the two shapes narrowed - // the alias-receiver rule onto it (caught on the numpy corpus before this marker existed). - b.importedName = isFrom ? std::string {} : "module"; - binds.push_back( std::move( b ) ); - return true; - } ); -} - // The DISCLOSE sink every extraction pass shares for ONE file: a pass that could not extract all of the file's facts // (a nesting bound it will not descend past, a tags query that is not available) says so here, and the parse worker // turns a short file into its --skipped row (why="extract-partial") and keeps its facts out of the cache, so a warm run @@ -3063,10 +2937,6 @@ void captureIncludes( TSNode root, Lang lang, std::uint32_t fileId, std::string_ } } - if( lang == Lang::Python && !target.empty() ) - { - capturePythonImportBinds( n, t, fileId, src, binds ); // Phase 5: the bound NAMES, beside the module - } // One directive's emission — the Include record plus its ABS-3 import-role use-site ref. A LAMBDA // rather than the straight-line block it used to be for exactly one reason: an Elixir // `alias MyApp.{Bar, Baz}` is ONE directive node naming N modules, so N records come off it and diff --git a/src/ingest_sidecap.h b/src/ingest_sidecap.h index 2a847aacc..091b28b24 100644 --- a/src/ingest_sidecap.h +++ b/src/ingest_sidecap.h @@ -1630,7 +1630,7 @@ void captureSideFacts( const LangEntry& le, std::uint32_t fileId, std::string_vi #endif // Elixir directives share the lexical tags context below (ElixirContext), and as of #358 so do the - // C-family ones and TypeScript/JavaScript's: their `@import.path` captures are normalised and emitted + // C-family, Go, Python and TypeScript/JavaScript ones: their `@import.path` captures are normalised and emitted // by captureTagsFacts, and captureIncludes no longer reads a preproc_include / preproc_call / // import_statement / export_statement / require() call at all. The walk is pure cost for those // languages — it descends only allowlisted containers, and the tags query already reaches the same @@ -1881,6 +1881,8 @@ void captureTagsFacts( TSQueryCursor* cursor, const LangEntry& le, std::uint32_t uint32_t nameRow = 0; // 0-based row of the @name identifier bool haveName = false; TSNode nameNode {}; // the @name identifier node — for C++ scope/qualifier (E#4) + TSNode importNamesNode {}; // Python: one clause's target, paired with its optional alias + TSNode importAliasNode {}; for( uint16_t ci = 0; ci < match.capture_count; ++ci ) { @@ -1955,15 +1957,27 @@ void captureTagsFacts( TSQueryCursor* cursor, const LangEntry& le, std::uint32_t // order while costing an allocation and a drain block. case CapRole::Import: { - if( full ) + if( full && capSv == "import.path" ) { emitCapturedImport( cap.node, fileId, le.lang, src, includes, refs, shortfall ); } + else if( full && le.lang == Lang::Python && capSv == "import.names" ) + { + importNamesNode = cap.node; + } + else if( full && le.lang == Lang::Python && capSv == "import.alias" ) + { + importAliasNode = cap.node; + } } break; } } + if( !ts_node_is_null( importNamesNode ) ) + { + emitCapturedPythonImportBind( importNamesNode, importAliasNode, fileId, src, binds ); + } if( !haveName ) { continue; diff --git a/test/pyimportprecisecheck.sh b/test/pyimportprecisecheck.sh index 1e3231978..764adcf82 100755 --- a/test/pyimportprecisecheck.sh +++ b/test/pyimportprecisecheck.sh @@ -109,4 +109,141 @@ monotonic_check() } monotonic_check +# #358 Python capture slice. Expectations below were read from pristine main, not from the new +# query: ONE Include per statement (first clause), but bindings for EVERY named/aliased clause. +# Aliased plain imports keep their whole written clause in --deps; cleaning it is a separate change. +PC="$TMP/captures"; mkdir -p "$PC/pkg" +cat >"$PC/main.py" <<'PY' +import a, b +import a as aa, b as bb +import pkg.mod +from pkg.mod import run, spare as renamed +from . import a +from pkg import * +from pkg.mod import (run as f, spare,) +import os +if TYPE_CHECKING: + import b +try: + from pkg.mod import run as rr +except ImportError: + import a +class C: + import b + def method(self): + import a as local + return local.run() +def plain(): + return bb.run() +PY +printf 'def run():\n return 1\n' >"$PC/a.py" +printf 'def run():\n return 2\n' >"$PC/b.py" +printf 'def init():\n return 1\n' >"$PC/pkg/__init__.py" +printf 'def run():\n return 2\ndef spare():\n return 3\n' >"$PC/pkg/mod.py" +printf 'import a as aa\nfrom pkg.mod import run as r\n' >"$PC/typing.pyi" +printf 'x = (import unreachable)\ndef okay():\n import a\n' >"$PC/broken.py" + +# This inspects the EMBEDDED vocabulary; --match alone also matches on a pre-change binary. +if grep -qa 'import\.names' "$BIN" && grep -qa 'import\.alias' "$BIN"; then + ok '#358: Python import.names and import.alias vocabulary is compiled in' +else + no '#358: Python import.names/import.alias vocabulary is absent from this binary' +fi +# Historical comments may retain the old name; check definitions, not prose. +if grep -rEq '^[[:space:]]*(inline[[:space:]]+)?void[[:space:]]+capturePythonImportBinds[[:space:]]*\(' "$ROOT/src"; then + no '#358: replaced Python import binding extractor still exists in src/' +else + ok '#358: replaced Python import binding extractor removed' +fi +"$BIN" "$PC" --deps --no-cache >"$TMP/pc.deps" 2>"$TMP/pc.err" +if python3 - "$TMP/pc.deps" <<'PY' +import sys, xml.etree.ElementTree as ET +root = ET.parse(sys.argv[1]).getroot() +rows = {f.get('p'): [i.get('t') for i in f.findall('inc')] for f in root.findall('f')} +want = ['a', 'a as aa', 'pkg.mod', 'pkg.mod', '.', 'pkg', 'pkg.mod', 'os', + 'b', 'pkg.mod', 'a', 'b', 'a as local'] +assert rows == {'main.py': want, 'typing.pyi': ['a as aa', 'pkg.mod'], 'broken.py': ['a']}, rows +PY +then ok '#358: exact first-clause, alias, from, relative, wildcard, nested and .pyi targets; ERROR import stays out' +else no '#358: Python target set/order differs from pristine main' +fi +"$BIN" "$PC" --callees=plain --no-cache >"$TMP/pc.plain" 2>/dev/null +if grep -q ']*count="1"' "$TMP/pc.plain" && grep -q 'p="b.py:1"' "$TMP/pc.plain"; then + ok '#358: the SECOND comma-separated module alias binds bb.run to b.py only' +else + no '#358: binding for the second module alias was lost or widened' +fi + +# One-space indentation avoids the vendored scanner's 8-bit serialized indent overflow. Each if +# spends two containers; an else arm spends three. These parse cleanly at exactly 255/256/257. +BD="$TMP/pybound"; mkdir -p "$BD" +if python3 - "$BD" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +for depth in (255, 256, 257, 258): + if depth % 2: + n = (depth - 3) // 2 + s = ''.join(' ' * i + 'if flag:\n' for i in range(n)) + s += ' ' * n + 'if flag:\n' + ' ' * (n + 1) + 'pass\n' + s += ' ' * n + 'else:\n' + ' ' * (n + 1) + 'import a\n' + else: + n = depth // 2 + s = ''.join(' ' * i + 'if flag:\n' for i in range(n)) + s += ' ' * n + ('pass\n' if depth == 258 else 'import a\n') + (p / f'depth{depth}.py').write_text(s) +PY +then :; else no '#358: could not generate Python nesting fixture'; fi +"$BIN" "$BD" --deps --no-cache >"$TMP/pb.deps" 2>/dev/null +"$BIN" "$BD" --skipped --no-cache >"$TMP/pb.skipped" 2>/dev/null +if python3 - "$TMP/pb.deps" "$TMP/pb.skipped" <<'PY' +import sys, xml.etree.ElementTree as ET +rows = ET.parse(sys.argv[1]).getroot().findall('f') +assert {f.get('p'): [i.get('t') for i in f.findall('inc')] for f in rows} == { + 'depth255.py': ['a'], 'depth256.py': ['a']} +skipped = ET.parse(sys.argv[2]).getroot().find('skipped') +assert skipped is not None +assert not skipped.findall('h'), 'nesting fixture must parse without recovery' +assert {f.get('p') for f in skipped.findall('f') if f.get('why') == 'extract-partial'} == { + 'depth257.py', 'depth258.py'}, 'cut at 257; empty deep container also disclosed' +PY +then ok '#358: clean 255/256 imports kept, 257 cut, and empty 258 nesting still disclosed' +else no '#358: Python reach/depth/disclosure differs from pristine main' +fi + +# Negative control: changing a non-first clause changes its BINDING while the first-clause Include +# is unchanged. The mutation must take, and the identical callee extraction must see it. +printf 'def run():\n return 3\n' >"$PC/c.py" +cp "$PC/main.py" "$TMP/pc.source" +sed 's/a as aa, b as bb/a as aa, c as bb/' "$TMP/pc.source" >"$PC/main.py" +if cmp -s "$PC/main.py" "$TMP/pc.source"; then no '#358: second-clause mutation did not take'; fi +"$BIN" "$PC" --callees=plain --no-cache >"$TMP/pc.changed" 2>/dev/null +if grep -q ']*count="1"' "$TMP/pc.changed" && grep -q 'p="c.py:1"' "$TMP/pc.changed" && ! cmp -s "$TMP/pc.plain" "$TMP/pc.changed"; then + ok '#358: second-clause mutation moves the alias edge (binding control is live)' +else + no '#358: second-clause mutation did not move the alias edge' +fi +cp "$TMP/pc.source" "$PC/main.py" +rm -f "$PC/c.py" + +"$BIN" "$PC" --deps --cache="$TMP/pc.bin" >"$TMP/pc.cold" 2>/dev/null +"$BIN" "$PC" --deps --cache="$TMP/pc.bin" >"$TMP/pc.warm" 2>/dev/null +if cmp -s "$TMP/pc.deps" "$TMP/pc.cold" && cmp -s "$TMP/pc.cold" "$TMP/pc.warm"; then + ok '#358: Python dependency output identical through fresh and warm caches' +else + no '#358: Python dependency output differs through cache' +fi +BASE_BIN="${RIPWIRE_BASE_BIN:-}" +if [ -n "$BASE_BIN" ] && [ -x "$BASE_BIN" ]; then + "$BASE_BIN" "$PC" --deps --cache="$TMP/pc.base.bin" >"$TMP/pc.base" 2>/dev/null + cp "$TMP/pc.base.bin" "$TMP/pc.base.copy" + "$BIN" "$PC" --deps --cache="$TMP/pc.base.bin" >"$TMP/pc.prewarm" 2>/dev/null + if cmp -s "$TMP/pc.base" "$TMP/pc.deps" && cmp -s "$TMP/pc.prewarm" "$TMP/pc.deps" && cmp -s "$TMP/pc.base.bin" "$TMP/pc.base.copy"; then + ok '#358: pristine-main Python cache accepted without rewrite; fresh output byte-identical' + else + no '#358: pristine-main Python cache/output differs; parser-version parity not established' + fi +else + skip '#358: pristine-main cache compatibility (set RIPWIRE_BASE_BIN)' +fi + [ "$fail" -eq 0 ] && echo "ALL PASS" || { echo "SOME CHECKS FAILED"; exit 1; } From 988506908fa6f0924a49a78c7c9bc615ba3f1e29 Mon Sep 17 00:00:00 2001 From: LaimaWu <317139567+LaimaWu@users.noreply.github.com> Date: Mon, 12 Oct 2026 08:49:36 +0800 Subject: [PATCH 2/3] test(python): pin capture imports, bindings and depth disclosure after rebase --- queries/python/tags.scm | 4 +- src/ingest_importcap.h | 11 ++- test/pyimportprecisecheck.sh | 140 ++++++++++++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 10 deletions(-) diff --git a/queries/python/tags.scm b/queries/python/tags.scm index 269e2b9c5..f6808d491 100644 --- a/queries/python/tags.scm +++ b/queries/python/tags.scm @@ -67,8 +67,8 @@ ; Shared import vocabulary (#358). The dependency target is still ONE per statement: the first ; name of a plain import (including its alias spelling), or the module_name of a from-import. -; C++ keeps only the first name capture and imposes the old container reach/depth; queries are -; unanchored so conditional and function-body imports keep their existing coverage. +; The child anchor captures only the first plain-import name. C++ imposes the old container +; reach/depth; statement patterns remain unanchored so conditional and function imports keep coverage. (import_statement . name: (_) @import.path) (import_from_statement module_name: (_) @import.path) diff --git a/src/ingest_importcap.h b/src/ingest_importcap.h index b193284b7..0beb5185c 100644 --- a/src/ingest_importcap.h +++ b/src/ingest_importcap.h @@ -28,12 +28,11 @@ // `@import.path` is the dependency-target capture, because C-family imports have no other part: `#include` names a path and nothing else — no // alias to record, no list of imported names — and because Go's alias/dot/blank spellings ride inside the // declaration clause its Include has always carried (see DepDialect::Go below), not in a field of their own. -// The other two names the issue sketches are reserved BY NAME so the next language lands on them rather -// than inventing a spelling: `@import.alias` is the local name an import renames a target to (`import a.b as c`, `use Foo\Bar as Baz`, `using X = Foo.Bar`), and -// `@import.names` is a group of targets ONE directive names (`use Foo\{A, B}`, `alias MyApp.{Bar, Baz}`), -// which is why both of those are emitted by the C++ side today through a per-directive helper rather than -// by one capture — a capture names one node, and a group is many. No enum value reserves them: an unused -// arm that no query can reach is a lie about what the vocabulary supports. +// `@import.alias` is the local name an import renames a target to (`import a.b as c`, `use Foo\Bar as Baz`, `using X = Foo.Bar`). +// `@import.names` names the imported clauses of a directive (`use Foo\{A, B}`, `alias MyApp.{Bar, Baz}`). +// Python emits one names clause and its optional alias in the SAME match, then its Python-specific +// helper produces the existing RawBind. Later dialects may need a per-directive helper for groups; +// a capture names one node, and a group is many. No unused enum arm pretends to support a dialect. // // SECOND MEMBER: DepDialect::Web (TypeScript, TSX, JavaScript, the .astro frontmatter that rides TypeScript). // Its three spellings are three patterns on the SAME capture name — `import … from 'x'`, a re-export diff --git a/test/pyimportprecisecheck.sh b/test/pyimportprecisecheck.sh index 764adcf82..76a8deffd 100755 --- a/test/pyimportprecisecheck.sh +++ b/test/pyimportprecisecheck.sh @@ -174,6 +174,142 @@ else no '#358: binding for the second module alias was lost or widened' fi +# Maintainer negatives and alias forms (#382). Keep these files free of definitions/assignments: +# then the lean cache exposes EVERY Include/RawBind, including bindings invisible in --deps. +PN="$TMP/pynegative"; mkdir -p "$PN" +printf 'from __future__ import annotations\n' >"$PN/future.py" +printf '__import__("x")\n' >"$PN/builtin_call.py" +printf 'import importlib\nimportlib.import_module("x")\n' >"$PN/importlib_call.py" +printf 'import a.b as c\n' >"$PN/dotted.py" +printf 'from . import a as b\n' >"$PN/relative.py" +"$BIN" "$PN" --deps --no-cache >"$TMP/pn.fresh" 2>"$TMP/pn.err" \ + || no '#382: fresh negative/alias fixture extraction failed' +"$BIN" "$PN" --deps --cache="$TMP/pn.bin" >"$TMP/pn.cold" 2>>"$TMP/pn.err" \ + || no '#382: cold negative/alias fixture extraction failed' +cp "$TMP/pn.bin" "$TMP/pn.copy" +"$BIN" "$PN" --deps --cache="$TMP/pn.bin" >"$TMP/pn.warm" 2>>"$TMP/pn.err" \ + || no '#382: warm negative/alias fixture extraction failed' +"$BIN" "$PN" --deps --no-cache >"$TMP/pn.repeat" 2>>"$TMP/pn.err" \ + || no '#382: repeated negative/alias fixture extraction failed' +if cmp -s "$TMP/pn.fresh" "$TMP/pn.cold" && cmp -s "$TMP/pn.cold" "$TMP/pn.warm" \ + && cmp -s "$TMP/pn.fresh" "$TMP/pn.repeat" && cmp -s "$TMP/pn.bin" "$TMP/pn.copy"; then + ok '#382: negative/alias deps deterministic, fresh == cold == warm, cache not rewritten' +else + no '#382: negative/alias deps or warm cache identity changed' +fi +if python3 - "$TMP/pn.fresh" "$TMP/pn.bin" "$ROOT" <<'PY' +import copy, pathlib, re, struct, sys, xml.etree.ElementTree as ET + +# readFileRecord/writeBind in ingest_cache.h, lean cache v31. No heuristic string searches: +# walk the counted arrays and require complete file coverage, clean parse health and exact EOF. +blob = pathlib.Path(sys.argv[2]).read_bytes() +endian = '<' if sys.byteorder == 'little' else '>' +assert blob[:4] == b'CTPK' and struct.unpack_from(endian + 'I', blob, 4)[0] == 31, 'update the cache walker for a new layout' +table, count = struct.unpack_from(endian + 'QI', blob, len(blob) - 24) +assert count == 5 == struct.unpack_from(endian + 'I', blob, 21)[0], 'all five fixtures must be cached' +assert table + count * 32 == len(blob) - 24, 'cache table bounds' +facts = {} +for i in range(count): + off = table + i * 32 + pos = struct.unpack_from(endian + 'Q', blob, off + 8)[0] + end = pos + struct.unpack_from(endian + 'I', blob, off + 24)[0] + assert 25 <= pos <= end <= table, 'cache record bounds' + def take(n): + global pos + assert n >= 0 and pos + n <= end, 'truncated cache record' + data = blob[pos:pos + n]; pos += n + return data + def number(fmt): + return struct.unpack(endian + fmt, take(struct.calcsize(fmt)))[0] + def string(): + return take(number('I')).decode('utf-8') + path = string() + assert path not in facts, 'duplicate file record' + take(4 * 8) # content hash and size/mtime/ctime + health = struct.unpack(endian + '5I', take(5 * 4)) + assert health[0] == health[1] == 0 and health[2] > 0, (path, 'fixture must parse cleanly', health) + assert number('I') == 0, 'lean fixture must contain no definitions/dictionary' + for _ in range(number('I')): # RawRef: only its variable-length strings affect the walk + take(5); string(); inherit = take(2)[0]; string(); take(1); string(); take(1) + string(); string(); take(13) # argCount is serialized u32, narrowed by u16Of32 on read + for _ in range(4): + string() # memberRoot/memberPath/memberCtor/memberVia + take(1) # memberFactory + if inherit: + take(4) # baseBind + incs = [] + for _ in range(number('I')): + flags = tuple(take(3)); byte = number('I'); value_use = number('B') + incs.append((string(), byte, flags, value_use)) + binds = [] + for _ in range(number('I')): + byte = number('I'); lang = number('B'); kind = number('B'); assignment = number('B') + spans = (number('I'), number('I')) + binds.append((string(), string(), string(), byte, lang, kind, assignment, spans)) + for _ in range(4): # FFI aliases, route defs/uses, const-opens: absent in this Python fixture + assert number('I') == 0, 'unexpected auxiliary fact' + assert pos == end, (path, 'cache walker desynchronised') + facts[path] = (incs, binds) + +# Derive enum values from their declarations; do not mistake a Type/VarDecl for an Import. +model = re.sub(r'//[^\n]*', '', pathlib.Path(sys.argv[3], 'src/model.h').read_text()) +def enum_value(enum, name): + body = re.search(r'enum class ' + enum + r'\s*:\s*std::uint8_t\s*\{([^}]+)\}', model).group(1) + names = [s.strip() for s in body.split(',') if s.strip()] + return names.index(name) +py = enum_value('Lang', 'Python'); imp = enum_value('LocalBindKind', 'Import') +name_alias = enum_value('LocalBindKind', 'NameAlias') +def include(target): + return (target, 0, (0, 0, 0), 0) +def bind(var, target, marker): + return (var, target, marker, 0, py, imp, 0, (0, 0)) +want = { + 'future.py': ([], []), + 'builtin_call.py': ([], []), + 'importlib_call.py': ([include('importlib')], [bind('importlib', 'importlib', 'module')]), + 'dotted.py': ([include('a.b as c')], [bind('c', 'a.b', 'module')]), + # Main's receiver-evidence pass also writes this NameAlias before the Import bind. + # Pin it exactly instead of filtering unrelated binds out of the assertion. + 'relative.py': ([include('.')], [('b', 'a', '', 14, py, name_alias, 0, (0, 0)), bind('b', '.', '')]), +} +def check(actual): + assert actual == want, ('Include/RawBind mismatch', actual, want) +check(facts) +rows = {f.get('p'): [inc.get('t') for inc in f.findall('inc')] + for f in ET.parse(sys.argv[1]).getroot().findall('f')} +assert rows == {p: [inc[0] for inc in incs] for p, (incs, _) in want.items() if incs}, rows + +# Test the SAME predicate with deliberately wrong extracted facts. Each control must fail; +# unrelated fields are left intact, so loss of coverage cannot hide behind another assertion. +for path in ('future.py', 'builtin_call.py', 'importlib_call.py'): + for channel, extra in ((0, include('x')), (1, bind('x', 'x', 'module'))): + bad = copy.deepcopy(facts); bad[path][channel].append(extra) + try: check(bad) + except AssertionError: pass + else: raise AssertionError((path, channel, 'extra fact escaped the negative assertion')) +for path in ('dotted.py', 'relative.py'): + for field, wrong in ((0, 'wrong_alias'), (1, 'wrong.target'), (2, 'wrong_marker')): + bad = copy.deepcopy(facts); idx = next(i for i, row in enumerate(bad[path][1]) if row[5] == imp) + row = list(bad[path][1][idx]); row[field] = wrong + bad[path][1][idx] = tuple(row) + try: check(bad) + except AssertionError: pass + else: raise AssertionError((path, field, 'wrong binding escaped the alias assertion')) + for channel in (0, 1): + bad = copy.deepcopy(facts) + if channel == 0: + bad[path][channel].clear() + else: + bad[path][channel][:] = [row for row in bad[path][channel] if row[5] != imp] + try: check(bad) + except AssertionError: pass + else: raise AssertionError((path, channel, 'missing fact escaped the alias assertion')) +print(' PASS #382: 16 corrupt-fact controls rejected by the exact Include/RawBind predicate') +PY +then ok '#382: future/dynamic negatives; c -> a.b with module marker; relative Include . and b -> . without marker' +else no '#382: negative/alias extraction or assertion controls failed' +fi + # One-space indentation avoids the vendored scanner's 8-bit serialized indent overflow. Each if # spends two containers; an else arm spends three. These parse cleanly at exactly 255/256/257. BD="$TMP/pybound"; mkdir -p "$BD" @@ -204,9 +340,9 @@ skipped = ET.parse(sys.argv[2]).getroot().find('skipped') assert skipped is not None assert not skipped.findall('h'), 'nesting fixture must parse without recovery' assert {f.get('p') for f in skipped.findall('f') if f.get('why') == 'extract-partial'} == { - 'depth257.py', 'depth258.py'}, 'cut at 257; empty deep container also disclosed' + 'depth257.py'}, 'cut at 257 is disclosed; import-free deep container has nothing cut' PY -then ok '#358: clean 255/256 imports kept, 257 cut, and empty 258 nesting still disclosed' +then ok '#358: clean 255/256 imports kept, 257 cut/disclosed, and import-free 258 not disclosed' else no '#358: Python reach/depth/disclosure differs from pristine main' fi From 38607f401a5b0497cff18f131239f208065a2877 Mon Sep 17 00:00:00 2001 From: LaimaWu <317139567+LaimaWu@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:49:08 +0800 Subject: [PATCH 3/3] test(python): compare complete binding facts without pinning pass order --- test/pyimportprecisecheck.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/pyimportprecisecheck.sh b/test/pyimportprecisecheck.sh index 76a8deffd..24f4e8970 100755 --- a/test/pyimportprecisecheck.sh +++ b/test/pyimportprecisecheck.sh @@ -199,6 +199,7 @@ else fi if python3 - "$TMP/pn.fresh" "$TMP/pn.bin" "$ROOT" <<'PY' import copy, pathlib, re, struct, sys, xml.etree.ElementTree as ET +from collections import Counter # readFileRecord/writeBind in ingest_cache.h, lean cache v31. No heuristic string searches: # walk the counted arrays and require complete file coverage, clean parse health and exact EOF. @@ -268,12 +269,14 @@ want = { 'builtin_call.py': ([], []), 'importlib_call.py': ([include('importlib')], [bind('importlib', 'importlib', 'module')]), 'dotted.py': ([include('a.b as c')], [bind('c', 'a.b', 'module')]), - # Main's receiver-evidence pass also writes this NameAlias before the Import bind. - # Pin it exactly instead of filtering unrelated binds out of the assertion. + # The receiver-evidence pass also writes this NameAlias. Pin all its fields and count; + # its position relative to the Import bind is an implementation detail of the two passes. 'relative.py': ([include('.')], [('b', 'a', '', 14, py, name_alias, 0, (0, 0)), bind('b', '.', '')]), } def check(actual): - assert actual == want, ('Include/RawBind mismatch', actual, want) + assert actual.keys() == want.keys() and all( + incs == want[path][0] and Counter(binds) == Counter(want[path][1]) + for path, (incs, binds) in actual.items()), ('Include/RawBind mismatch', actual, want) check(facts) rows = {f.get('p'): [inc.get('t') for inc in f.findall('inc')] for f in ET.parse(sys.argv[1]).getroot().findall('f')}