ISS-10889: index every tracked file in a supported language; skip-list directories judge only untracked files

docs/reference/cli.md

@@ -699,11 +699,14 @@ lc tools call grep --args '{"path":".","content_regex":"TODO"}'
 
 ### Excluding files from indexing
 
-`lc code index` walks git-visible files (`git ls-files`, so `.gitignore` is
-already honored) plus a built-in skip list for caches/build output. To
-exclude specific files -- data dumps, fixtures, generated JSON -- that are
-tracked in git and therefore not covered by `.gitignore`, add gitignore-syntax
-patterns to `.lemoncrow/.ignore`:
+`lc code index` indexes every git-tracked file in a supported language,
+whatever its directory is called. Files git does not track -- new files not
+yet added, and every file when the directory is not a git repository -- are
+indexed too, unless `.gitignore` excludes them or they sit under a built-in
+skip list of cache, dependency, and build-output directories (`node_modules`,
+`.venv`, `dist`, `build`, `data`, `results`, and similar). To exclude specific
+tracked files -- data dumps, fixtures, generated JSON -- that `.gitignore`
+cannot cover, add gitignore-syntax patterns to `.lemoncrow/.ignore`:
 
 ```gitignore
 # .lemoncrow/.ignore

src/lemoncrow/infra/code_intel/coverage.py

@@ -55,6 +55,7 @@
     exclusion_rule,
     free_tier_selection,
     git_ignored,
+    git_tracked,
     load_lemoncrow_ignore_spec,
     run_git,
     source_file_patterns,
@@ -214,6 +215,7 @@ def _exclusion(
     rel: str,
     selection: _IndexSelection,
     ignored: frozenset[str],
+    tracked: frozenset[str],
     patterns: Sequence[str],
 ) -> tuple[str, str]:
     """``(rule, reason)`` for an on-disk path the indexer's scan passed over.
@@ -223,7 +225,9 @@ def _exclusion(
     when the scan passed a path over for a reason no rule sees -- an untracked
     file inside a submodule, which ``git ls-files --others`` does not recurse.
     """
-    return exclusion_rule(rel, ignore_spec=selection.ignore_spec, ignored=ignored, patterns=patterns) or (
+    return exclusion_rule(
+        rel, ignore_spec=selection.ignore_spec, ignored=ignored, tracked=tracked, patterns=patterns
+    ) or (
         RULE_SOURCE_FILE_SCAN,
         REASON_SOURCE_FILE_SCAN,
     )
@@ -290,7 +294,9 @@ def check_coverage(paths: list[str] | None = None, repo_root: Path | str = ".")
 
     unindexed_on_disk = [rel for rel in candidates if rel not in indexed and (root / rel).exists()]
     selection = _index_selection(root) if unindexed_on_disk else _NO_SELECTION
-    ignored = git_ignored(root, [rel for rel in unindexed_on_disk if rel not in selection.selected])
+    passed_over = [rel for rel in unindexed_on_disk if rel not in selection.selected]
+    ignored = git_ignored(root, passed_over)
+    tracked = git_tracked(root, passed_over)
     patterns = source_file_patterns() if unindexed_on_disk else []
 
     entries: list[PathCoverage] = []
@@ -321,7 +327,7 @@ def check_coverage(paths: list[str] | None = None, repo_root: Path | str = ".")
         elif not exists:
             entry = PathCoverage(rel, "missing", "not on disk and not indexed", language_name, 0)
         elif rel not in selection.selected:
-            rule, reason = _exclusion(rel, selection, ignored, patterns)
+            rule, reason = _exclusion(rel, selection, ignored, tracked, patterns)
             entry = PathCoverage(rel, "excluded", reason, language_name, 0, rule)
         elif rel not in selection.kept:
             entry = PathCoverage(rel, "excluded", "free-tier file cap", language_name, 0, RULE_FREE_TIER_FILE_CAP)

src/lemoncrow/infra/code_intel/inclusion.py

@@ -15,6 +15,13 @@
 *down* into this module, which is why nothing here imports from
 :mod:`lemoncrow.pro` at any scope -- the dependency runs one way.
 
+A file git tracks is the owner's word that it belongs to the project, so it is
+indexed whatever its directory is called: the skipped-directory list judges only
+files git does not track -- untracked, not-ignored files and every file of the
+non-git fallback walk -- where nothing else tells build output or a dependency
+tree apart from source. ``.lemoncrow/.ignore`` is an explicit opt-out and binds
+tracked files too; it is the way to keep a tracked data dump out.
+
 Fail-open by design: a rule that cannot be evaluated -- git missing, the
 directory not a repository, ``pathspec`` not installed -- admits the path, which
 is what the scan's own non-git fallback (``_iter_glob_source_files``) does.
@@ -44,6 +51,7 @@
     "exclusion_rule",
     "free_tier_selection",
     "git_ignored",
+    "git_tracked",
     "indexable_paths",
     "load_lemoncrow_ignore_spec",
     "run_git",
@@ -64,7 +72,8 @@
 
 #: The indexer's file-selection rules, first match wins. The free-tier cap only
 #: ever applies to a path the scan selected, and every other rule only to a path
-#: it did not. ``source-file-scan`` closes the ladder: the scan's own glob gate
+#: it did not. ``skipped-directory`` and ``git-ignore`` never fire for a tracked
+#: file. ``source-file-scan`` closes the ladder: the scan's own glob gate
 #: does not take the path, and no narrower rule says why (an extension in the
 #: wrong case, an untracked file inside a submodule).
 EXCLUSION_RULES: tuple[str, ...] = (
@@ -76,6 +85,9 @@
     RULE_SOURCE_FILE_SCAN,
 )
 
+#: Directory names whose files are never indexed unless git tracks them. Only
+#: untracked files meet this list: a tracked file under ``build/`` or ``data/``
+#: is source its owner committed (a Next.js ``build/[id]/page.tsx`` route).
 _SKIP_PARTS = {
     ".git",
     ".lemoncrow",
@@ -242,11 +254,57 @@ def git_ignored(root: Path, candidates: Sequence[str]) -> frozenset[str]:
     return frozenset(entry for entry in output.split("\0") if entry)
 
 
+#: Pathspecs per ``git ls-files`` call in :func:`git_tracked`, so a batch of
+#: thousands of touched files stays well under the OS argument-length limit.
+_TRACKED_QUERY_CHUNK = 1000
+
+
+def git_tracked(root: Path, candidates: Sequence[str]) -> frozenset[str]:
+    """The repo-relative *candidates* git tracks in this checkout, submodules included.
+
+    The answer the scan's ``ls-files --cached --recurse-submodules`` pass gives,
+    asked only about *candidates* so an edit's re-index does not list the whole
+    repository. Pathspecs are literal: ``app/[id]/page.tsx`` is a path, not a glob.
+
+    Fail-open to the empty set: every candidate is then judged as untracked, by
+    the full rule ladder -- what the scan's own non-git fallback applies.
+    """
+    tracked: set[str] = set()
+    for start in range(0, len(candidates), _TRACKED_QUERY_CHUNK):
+        chunk = candidates[start : start + _TRACKED_QUERY_CHUNK]
+        try:
+            completed = subprocess.run(
+                [
+                    "git",
+                    "--literal-pathspecs",
+                    "-C",
+                    str(root),
+                    "ls-files",
+                    "-z",
+                    "--cached",
+                    "--recurse-submodules",
+                    "--",
+                    *chunk,
+                ],
+                capture_output=True,
+                check=False,
+                timeout=30,
+            )
+        except (OSError, subprocess.SubprocessError):
+            return frozenset()
+        if completed.returncode != 0:
+            return frozenset()
+        # Bytes, not text: a non-UTF-8 name must not raise out of an edit's re-index.
+        tracked.update(entry.decode("utf-8", errors="replace") for entry in completed.stdout.split(b"\0") if entry)
+    return frozenset(tracked)
+
+
 def exclusion_rule(
     rel: str,
     *,
     ignore_spec: Any | None,
     ignored: Container[str],
+    tracked: Container[str],
     patterns: Sequence[str] | None = None,
 ) -> tuple[str, str] | None:
     """``(rule, reason)`` for the first rule that passes *rel* over, else ``None``.
@@ -258,12 +316,14 @@ def exclusion_rule(
     scan drops on its patterns alone -- ``src/SHOUT.PY`` -- was admitted by the
     incremental entry point while coverage called it ``excluded``.
 
-    *patterns* defaults to the scan's own :data:`SOURCE_FILE_PATTERNS`; pass the
-    list once when classifying a batch.
+    A path in *tracked* (see :func:`git_tracked`) skips the skipped-directory
+    rung: git vouches for it. *patterns* defaults to the scan's own
+    :data:`SOURCE_FILE_PATTERNS`; pass the list once when classifying a batch.
     """
-    for part in Path(rel).parts:
-        if should_skip_relative_path(part):
-            return RULE_SKIPPED_DIRECTORY, f"skipped directory: {part}"
+    if rel not in tracked:
+        for part in Path(rel).parts:
+            if should_skip_relative_path(part):
+                return RULE_SKIPPED_DIRECTORY, f"skipped directory: {part}"
     if ignore_spec is not None and ignore_spec.match_file(rel):
         return RULE_LEMONCROW_IGNORE, ".lemoncrow/.ignore"
     if rel in ignored:
@@ -280,9 +340,10 @@ def indexable_paths(repo_root: Path, paths: Iterable[Path]) -> list[Path]:
 
     For the incremental entry point, where the candidates are the handful of
     files an edit touched rather than a whole scan: the ``.lemoncrow/.ignore``
-    spec is loaded once and git is asked once for the batch, so the cost is one
-    subprocess per reindex, not one per file. A path outside *repo_root* is
-    dropped -- no scan of this checkout would ever list it.
+    spec is loaded once and git is asked about the batch as a whole -- which
+    candidates it ignores, which it tracks -- so the cost is two subprocesses
+    per reindex, not per file. A path outside *repo_root* is dropped -- no scan
+    of this checkout would ever list it.
     """
     root = repo_root.resolve()
     pairs: list[tuple[Path, str]] = []
@@ -294,10 +355,12 @@ def indexable_paths(repo_root: Path, paths: Iterable[Path]) -> list[Path]:
     if not pairs:
         return []
     ignore_spec = load_lemoncrow_ignore_spec(root)
-    ignored = git_ignored(root, [rel for _path, rel in pairs])
+    rels = [rel for _path, rel in pairs]
+    ignored = git_ignored(root, rels)
+    tracked = git_tracked(root, rels)
     patterns = source_file_patterns()
     return [
         path
         for path, rel in pairs
-        if exclusion_rule(rel, ignore_spec=ignore_spec, ignored=ignored, patterns=patterns) is None
+        if exclusion_rule(rel, ignore_spec=ignore_spec, ignored=ignored, tracked=tracked, patterns=patterns) is None
     ]

src/lemoncrow/pro/capabilities/code_context/engine.py

@@ -8409,7 +8409,14 @@ def _fetch_cf() -> None:
                     )
         if file_glob:
             hits = [hit for hit in hits if _matches_file_glob(hit.file_path, file_glob)]
-        hits = [hit for hit in hits if not should_skip_relative_path(hit.file_path)]
+        # Repo symbols already passed the index's file-selection rules, which take a
+        # tracked file whatever its directory is called; the directory-name filter
+        # still judges commit-history and external-dependency hits.
+        hits = [
+            hit
+            for hit in hits
+            if (scope == "repo" and hit.provenance != "commit") or not should_skip_relative_path(hit.file_path)
+        ]
         if provenance_filter is not None:
             hits = [h for h in hits if h.provenance == provenance_filter]
         if _is_precise_symbol_query(query):

src/lemoncrow/pro/capabilities/repo_map/graph.py

@@ -51,11 +51,12 @@ def _iter_git_visible_source_files(
 ) -> list[Path]:
     # ``--recurse-submodules`` lists tracked files inside submodules too, but git
     # rejects it combined with ``--others``, so untracked files are fetched in a
-    # separate top-level-only call.
-    entries: list[bytes] = []
-    for extra_args in (
-        ("--cached", "--recurse-submodules"),
-        ("--others", "--exclude-standard"),
+    # separate top-level-only call. Which call listed an entry is kept: only an
+    # untracked file meets the skipped-directory list (see inclusion._SKIP_PARTS).
+    entries: list[tuple[bytes, bool]] = []
+    for extra_args, tracked in (
+        (("--cached", "--recurse-submodules"), True),
+        (("--others", "--exclude-standard"), False),
     ):
         try:
             completed = subprocess.run(
@@ -69,10 +70,10 @@ def _iter_git_visible_source_files(
             return []
         if completed.returncode != 0:
             return []
-        entries.extend(entry for entry in completed.stdout.split(b"\x00") if entry)
+        entries.extend((entry, tracked) for entry in completed.stdout.split(b"\x00") if entry)
     files: list[Path] = []
     total_raw = len(entries)
-    for i, raw_entry in enumerate(entries):
+    for i, (raw_entry, tracked) in enumerate(entries):
         if progress_callback is not None:
             progress_callback(i, total_raw)
         rel = raw_entry.decode("utf-8", errors="replace")
@@ -81,7 +82,7 @@ def _iter_git_visible_source_files(
         path = (repo_root / rel).resolve()
         if not path.is_file():
             continue
-        if should_skip_path(path, repo_root=repo_root):
+        if not tracked and should_skip_path(path, repo_root=repo_root):
             continue
         if ignore_spec is not None and ignore_spec.match_file(rel):
             continue

tests/core/test_code_context.py

@@ -1666,6 +1666,21 @@ def test_auto_mode_keeps_identifier_queries_on_exact_lexical_order(tmp_path: Pat
     assert payload["mode"] == "lexical"
 
 
+def test_search_symbols_keeps_tracked_files_under_skip_list_directory_names(tmp_path: Path) -> None:
+    """A tracked file under ``build/`` is indexed, so repo search must not drop it by directory name."""
+    _init_git_fixture_repo(tmp_path / "repo")
+    root = tmp_path / "repo"
+    (root / "build").mkdir()
+    (root / "build" / "pipeline.py").write_text("def assemble_release_bundle() -> int:\n    return 1\n")
+    _git(["add", "build/pipeline.py"], root)
+    engine = CodeContextEngine(root, db_path=tmp_path / "code.sqlite")
+    engine.index_repo()
+
+    hits = engine.search_symbols("assemble_release_bundle", limit=5, mode="lexical")
+
+    assert [(hit.file_path, hit.symbol_name) for hit in hits][:1] == [("build/pipeline.py", "assemble_release_bundle")]
+
+
 def test_search_symbols_lexical_planner_prioritizes_exact_and_case_insensitive_matches(
     tmp_path: Path,
 ) -> None:

tests/core/test_repo_map.py

@@ -210,3 +210,27 @@ def test_iter_source_files_respects_lemoncrow_ignore_git_tracked(tmp_path: Path)
     files = {path.relative_to(tmp_path).as_posix() for path in iter_source_files(tmp_path)}
 
     assert files == {"src/keep.py"}
+
+
+def test_iter_source_files_takes_tracked_files_whatever_their_directory_is_called(tmp_path: Path) -> None:
+    """A tracked file is indexed under any directory name; the skip list judges only untracked files."""
+    import subprocess
+
+    subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True)
+    tracked = [
+        "src/keep.py",
+        "app/[orgSlug]/build/[id]/page.ts",
+        "insights/data/source.ts",
+        "compute/results/route.ts",
+        "node_modules/vendored/index.js",
+        "dist/action.js",
+    ]
+    untracked = ["build/scratch.py", "data/dump.py", ".venv/lib/site.py", "node_modules/dep/index.js"]
+    for rel in [*tracked, *untracked]:
+        (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)
+        (tmp_path / rel).write_text("export const x = 1;\n" if rel.endswith((".ts", ".js")) else "X = 1\n")
+    subprocess.run(["git", "--literal-pathspecs", "add", "--", *tracked], cwd=tmp_path, check=True)
+
+    files = {path.relative_to(tmp_path).as_posix() for path in iter_source_files(tmp_path)}
+
+    assert files == set(tracked)