Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ uvx srcly report . --out .srcly --verbose

Use `uvx srcly scan . --out .srcly/tree.json` or `uvx srcly report . --include-tree` when you need the full raw tree. Use `--include-dependencies` to add the TypeScript/TSX dependency graph artifact.

In a Git worktree, the tree, Imports and Symbols start from the same Git-listed files: tracked files stay visible even if they match an ignore rule; ignored untracked files do not. Imports and Symbols then limit the list to supported JavaScript, TypeScript and Python source up to 1 MB, excluding `.d.ts`, entries in hidden or pruned directories such as `node_modules`, and symlinks that leave the selected root. A file can appear in the tree without being eligible for these views. These filters do not define which files changed between revisions.

`explain --at` reads one scope of the analysis: the repo or a folder (child folders ranked by their worst file, then the worst files below), a file (its functions and classes), or `file::symbol` (its lines and every measured metric). Commands share the app's analysis cache, so after the first scan each call takes about a second while no file has changed; `--refresh` rescans. Each answer ends with an `Analysis:` line: reused or fresh scan, the Git HEAD and uncommitted files, the checkers that ran, and how many code files none covers. Same-named symbols are `file::name@line`, and any printed "Start at" value (`path:line name`) works as `--at`. A symbol answer splits its CCN into statements and `&&`/`||`/`?:` operators with the deepest lines, so a flat router reads differently from nested control flow, and lists test files that import its module and name it (a name match, not coverage).

Findings rank **code** files. Each file is also classed as test, docs, style (CSS and other stylesheets), fixture, generated, vendored, config or data from its path, and the report names how many of each it left out and why (`--include-class test` ranks one too). A long README or a test snapshot is never a refactor target by accident, and never silently dropped.
Expand Down
9 changes: 9 additions & 0 deletions client/e2e/symbols.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,15 @@ test.describe("Symbols mode on solid-route", () => {
await page.goto(url({ view: "graph", symbols: "src/components/RecordTable.tsx#Nope" }));
await expect(page.getByText("Not in the index")).toBeVisible({ timeout: 90_000 });
});

test("explain when a tree file isn't eligible for structural views", async ({ page }) => {
await page.goto(url({ view: "graph", at: "README.md" }));
const notice = page.locator("[data-imports-excluded]");
await expect(notice).toContainText("README.md is in the tree but not in Imports", { timeout: 90_000 });
await page.goto(url({ view: "graph", symbols: "README.md" }));
await expect(page.getByText("Not in the index")).toBeVisible({ timeout: 90_000 });
await expect(page.getByText(/Symbols indexes JavaScript, TypeScript and Python source/)).toBeVisible();
});
});

test.describe("Symbols mode paths on solid-route", () => {
Expand Down
18 changes: 17 additions & 1 deletion client/src/viz/deps/DependencyView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import { layoutWithHubs } from "./hubs";
import type { ViewLayout } from "./layout";
import { viewContents, viewMetrics, type MetricsIndex } from "./metrics";
import { buildViewModel, DEFAULT_BUDGET, type ViewNode } from "./viewModel";
import { folderOf, graphIdFor, isDataFlow, isSymbols } from "../../navigation/location";
import { folderOf, graphIdFor, isDataFlow, isSymbols, splitAt } from "../../navigation/location";
import { addAnchorPatch, refsPatch, tracePatch } from "../../navigation/nodeActions";
import NodeActionsMenu, { type NodeMenuTarget } from "../../navigation/NodeActionsMenu";
import { go, graphFilters as filters, location, replace, setGraphFilters as setFilters } from "../../navigation/store";
Expand Down Expand Up @@ -88,6 +88,15 @@ function ImportView(props: {
return g ? buildIndex(applyFilters(g, filters())) : null;
});

const unindexedFile = createMemo(() => {
const nodes = graph()?.nodes;
const { file } = splitAt(location().at);
if (!nodes || !file || !props.metricsIndex) return null;
const separator = props.rootPath.includes("\\") ? "\\" : "/";
const full = `${props.rootPath.replace(/[\\/]$/, "")}${separator}${file.replaceAll("/", separator)}`;
return props.metricsIndex.get(full)?.type === "file" && !nodes.some((n) => n.id === file) ? file : null;
});

const effectiveFocus = createMemo(() => {
const idx = index();
if (!idx) return null;
Expand Down Expand Up @@ -420,6 +429,13 @@ function ImportView(props: {
</IconButton>
</ViewTools>

<Show when={unindexedFile()}>
{(file) => (
<p data-imports-excluded role="status" class="border-b border-[var(--plc-border)] px-3 py-2 text-xs text-[var(--plc-on-muted)]">
<code>{file()}</code> is in the tree but not in Imports. Imports covers JavaScript, TypeScript and Python source up to 1 MB. It excludes <code>.d.ts</code> files, hidden or pruned folders, and links outside this root.
</p>
)}
</Show>
<div ref={host} class="relative min-h-0 flex-1">
<Show when={graph.loading}>
<LoadingState label="Loading dependencies…" class="absolute inset-0" />
Expand Down
2 changes: 1 addition & 1 deletion client/src/viz/deps/symbols/SymbolsView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,7 @@ export default function SymbolsView(props: { rootPath: string; onOpen: (at: stri
<EmptyState
class="absolute inset-0"
title="Not in the index"
description={`${message()}. The index holds every function, class, method, field, variable and type of the TypeScript and Python files under this root.`}
description={`${message()}. Symbols indexes JavaScript, TypeScript and Python source up to 1 MB. It excludes .d.ts files, hidden or pruned folders, and links outside this root.`}
actions={
<Button variant="ghost" class="gap-1 px-2" onClick={() => setPalette(true)} title="Find a symbol (/)">
<Search size={13} aria-hidden="true" />
Expand Down
2 changes: 1 addition & 1 deletion docs/CODE_MAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ Browser
| Feature | Where |
| --- | --- |
| Scan orchestration (walk, filter, analyze, build tree) | `server/app/services/analysis.py` (`scan_codebase`) |
| File discovery (scan + landing-page estimate): `git ls-files` in repos, walk fallback | `server/app/services/analysis.py` (`discover_files`); estimate in `server/app/routers/analysis.py` (`_estimate_counts`) |
| File discovery: Tree and the landing-page estimate share `discover_files`; Imports and Symbols share its Git candidates, then apply their structural source filters | `server/app/services/analysis.py` (`discover_files`, `git_inventory_files`); `server/app/services/deps/common.py` (`walk_source_files`); estimate in `server/app/routers/analysis.py` (`_estimate_counts`) |
| Ignore rules: built-in dirs/files/extensions, binary + minified skips | `server/app/config.py`; `analysis.py` (`classify_candidate`) |
| Ignore rules: nested `.gitignore` translation (walk fallback only) | `server/app/services/analysis.py` (`_load_gitignore_spec`, `_translate_gitignore_pattern`) |
| Worker pool: reuse, hard per-file timeouts, crash recovery | `server/app/services/scan_workers.py` (`run_file_analyses`), wrapped by `analysis.py` (`_run_file_analyses_with_hard_timeouts`) |
Expand Down
44 changes: 35 additions & 9 deletions server/app/services/analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -562,22 +562,27 @@ class _GitListing:
"""Ignored entries relative to the root; wholly ignored trees end in "/"."""


def _list_with_git(root_path: Path, cancel: CancelToken | None = None) -> _GitListing | None:
def _list_with_git(
root_path: Path, cancel: CancelToken | None = None, *, include_ignored: bool = True
) -> _GitListing | None:
"""
List ``root_path`` with ``git ls-files --cached --others --exclude-standard``.

Returns None if git cannot list the tree. Nested repositories and
submodules show up as directories and are returned in ``nested``.
"""
with ThreadPoolExecutor(max_workers=1) as pool:
# --directory collapses wholly ignored trees (node_modules, build output)
# into one "dir/" entry; listing every file in them costs seconds on big
# repos. (--no-empty-directory is avoided: some git versions then list nothing.)
ignored_future = pool.submit(
_git_ls_files, root_path, "--others", "--ignored", "--exclude-standard", "--directory"
)
if include_ignored:
with ThreadPoolExecutor(max_workers=1) as pool:
# --directory collapses wholly ignored trees (node_modules, build output)
# into one "dir/" entry; listing every file in them costs seconds on big repos.
ignored_future = pool.submit(
_git_ls_files, root_path, "--others", "--ignored", "--exclude-standard", "--directory"
)
listed = _git_ls_files(root_path, "--cached", "--others", "--exclude-standard")
ignored = ignored_future.result()
else:
listed = _git_ls_files(root_path, "--cached", "--others", "--exclude-standard")
ignored = ignored_future.result()
ignored = []
if listed is None:
return None

Expand Down Expand Up @@ -619,6 +624,27 @@ def _list_with_git(root_path: Path, cancel: CancelToken | None = None) -> _GitLi
return listing


def git_inventory_files(root_path: Path) -> list[str] | None:
"""The tree scan's Git candidates, before per-view analysis filters.

Tracked files stay visible even when they match an ignore rule. Structural
views don't need the tree's ignored counts or its binary/minified sniff.
None signals that callers should use their non-Git discovery fallback.
"""
root_path = root_path.resolve()
if not _is_inside_git_work_tree(root_path):
return None
pending = [root_path]
files: list[str] = []
while pending:
listing = _list_with_git(pending.pop(), include_ignored=False)
if listing is None:
return None
files.extend(listing.files)
pending.extend(listing.nested)
return files


def _discover_with_git(
root_path: Path,
result: DiscoveryResult,
Expand Down
57 changes: 46 additions & 11 deletions server/app/services/deps/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from app.config import IGNORE_DIRS

MAX_SOURCE_BYTES = 1_000_000
SourceSignature = tuple[tuple[Path, int, int], ...]

_TEST_DIRS = {"test", "tests", "__tests__", "__mocks__", "spec", "e2e"}
_TEST_FILE = re.compile(
Expand Down Expand Up @@ -51,8 +52,36 @@ def _gitignore_spec(root: Path):


def walk_source_files(root: Path, extensions: Set[str]) -> List[Path]:
"""Source files under root, skipping ignored, hidden and gitignored paths."""
"""Eligible Imports/Symbols source files, with the tree's Git visibility."""
from app.services.analysis import git_inventory_files

root = root.resolve()
inventory = git_inventory_files(root)
if inventory is not None:
out: list[Path] = []
for name in inventory:
path = Path(name)
if any(part.startswith(".") for part in path.relative_to(root).parts[:-1]):
continue
source = _eligible_source(root, path, extensions)
if source is not None:
out.append(source)
return sorted(set(out))
return _walk_source_files(root, extensions)


def _eligible_source(root: Path, path: Path, extensions: Set[str]) -> Path | None:
if path.name.endswith(".d.ts") or path.suffix not in extensions:
return None
try:
source = path.resolve()
return source if source.is_relative_to(root) and source.stat().st_size <= MAX_SOURCE_BYTES else None
except OSError:
return None


def _walk_source_files(root: Path, extensions: Set[str]) -> List[Path]:
"""Keep the existing pathspec discovery for folders without a Git listing."""
repo_root, spec = _gitignore_spec(root)

def ignored(p: Path, is_dir: bool) -> bool:
Expand All @@ -73,19 +102,25 @@ def ignored(p: Path, is_dir: bool) -> bool:
if d not in IGNORE_DIRS and not d.startswith(".") and not ignored(base / d, True)
)
for name in sorted(filenames):
if name.endswith(".d.ts"):
continue
p = base / name
if ignored(p, False):
continue
if p.suffix in extensions:
try:
if p.stat().st_size > MAX_SOURCE_BYTES:
continue
except OSError:
continue
out.append(p.resolve())
return out
source = _eligible_source(root, p, extensions)
if source is not None:
out.append(source)
return list(dict.fromkeys(out))


def source_signature(files: List[Path]) -> SourceSignature:
"""Exact source identities and file stats, independent of inventory order."""
stamps = []
for path in sorted(files):
try:
st = path.stat()
stamps.append((path, st.st_mtime_ns, st.st_size))
except OSError:
stamps.append((path, -1, -1))
return tuple(stamps)


@dataclass
Expand Down
35 changes: 12 additions & 23 deletions server/app/services/deps/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@
from typing import Dict, List, Optional, Tuple

from app.models import DepGraph, DepGraphEdge, DepGraphNode
from app.services.deps.common import DepEdge, is_test_path, unique, walk_source_files
from app.services.deps.common import DepEdge, SourceSignature, is_test_path, source_signature, unique, walk_source_files
from app.services.deps.typescript_deps import TS_EXTENSIONS, build_ts_edges

EXTERNAL_ROOT_ID = "ext:"

_cache: Dict[Path, Tuple[Tuple[int, int], DepGraph]] = {}
_cache: Dict[Path, Tuple[SourceSignature, DepGraph]] = {}
_lock = threading.Lock()


Expand All @@ -24,22 +24,11 @@ def _language(path: Path) -> str:
return "other"


def _signature(files: List[Path]) -> Tuple[int, int]:
total = 0
for f in files:
try:
total ^= hash((str(f), f.stat().st_mtime_ns))
except OSError:
continue
return len(files), total


def collect_edges(root: Path) -> Tuple[List[Path], List[DepEdge], Dict[str, list]]:
"""Run every language extractor. Returns files, edges and per-file symbol defs."""
ts_files = walk_source_files(root, TS_EXTENSIONS)
def collect_edges(root: Path, files: List[Path]) -> Tuple[List[DepEdge], Dict[str, list]]:
"""Run every language extractor against one inventory snapshot."""
ts_files = [f for f in files if f.suffix in TS_EXTENSIONS]
edges: List[DepEdge] = []
symbols: Dict[str, list] = {}
files: List[Path] = list(ts_files)

if ts_files:
ts_edges, ts_infos = build_ts_edges(root, ts_files)
Expand All @@ -53,26 +42,26 @@ def collect_edges(root: Path) -> Tuple[List[Path], List[DepEdge], Dict[str, list
build_py_edges = None
PY_EXTENSIONS = set()
if build_py_edges is not None:
py_files = walk_source_files(root, PY_EXTENSIONS)
py_files = [f for f in files if f.suffix in PY_EXTENSIONS]
if py_files:
py_edges, py_symbols, _infos, _index = build_py_edges(root, py_files)
edges.extend(py_edges)
symbols.update(py_symbols)
files.extend(py_files)
return files, edges, symbols
return edges, symbols


def build_dependency_graph(root: Path, use_cache: bool = True) -> DepGraph:
root = root.resolve()
files = walk_source_files(root, TS_EXTENSIONS | {".py"})
if use_cache:
files_now = walk_source_files(root, TS_EXTENSIONS | {".py"})
sig = _signature(files_now)
sig = source_signature(files)
with _lock:
hit = _cache.get(root)
if hit is not None and hit[0] == sig:
return hit[1]
files, edges, symbols = collect_edges(root)
graph = _assemble(root, files, edges, symbols)
edges, symbols = collect_edges(root, files)
# Keep the existing TS-first language order in the graph payload.
graph = _assemble(root, sorted(files, key=lambda f: f.suffix == ".py"), edges, symbols)
if use_cache:
with _lock:
_cache[root] = (sig, graph)
Expand Down
16 changes: 3 additions & 13 deletions server/app/services/symbols/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from typing import Dict, Iterable, List, Optional, Set, Tuple

from app.models import SymbolGraph, SymbolGraphEdge, SymbolGraphNode, SymbolIndexInfo, SymbolPath, SymbolSearchHit, SymbolStats
from app.services.deps.common import is_test_path, walk_source_files
from app.services.deps.common import SourceSignature, is_test_path, source_signature, walk_source_files
from app.services.deps.typescript_deps import TS_EXTENSIONS
from app.services.symbols.model import BUILTIN_MEMBERS, REF_KINDS, FileSymbols, PendingMember, RefKind, Reference, Symbol

Expand Down Expand Up @@ -550,27 +550,17 @@ def _demote(kind: RefKind) -> RefKind:
return "write" if kind == "write" else "read"


_cache: Dict[Path, Tuple[Tuple[int, int], SymbolIndex]] = {}
_cache: Dict[Path, Tuple[SourceSignature, SymbolIndex]] = {}
_lock = threading.Lock()

SOURCE_EXTENSIONS = TS_EXTENSIONS | {".py"}


def _signature(files: List[Path]) -> Tuple[int, int]:
total = 0
for f in files:
try:
total ^= hash((str(f), f.stat().st_mtime_ns))
except OSError:
continue
return len(files), total


def build_symbol_index(root: Path, use_cache: bool = True) -> SymbolIndex:
root = root.resolve()
files = walk_source_files(root, SOURCE_EXTENSIONS)
sig = _signature(files)
if use_cache:
sig = source_signature(files)
with _lock:
hit = _cache.get(root)
if hit is not None and hit[0] == sig:
Expand Down
Loading
Loading