diff --git a/README.md b/README.md index 798d3ea..34b6071 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,20 @@ concurrency, and whether to compress. `Result` carries `[]File` and a rendered `Tree` string. `Result.Markdown(w)` and `Result.XML(w)` write the packed document. +`Build(root string, opts Options) (*Graph, error)` walks `root` the same way as +`Pack` and returns a directed code graph. Nodes are files, modules, +declarations, and unresolved external call targets; each edge has a relation +(`contains`, `imports`, `calls`) and a confidence (`extracted` for facts read +directly from one AST, `inferred` for cross-file name resolution). `Graph` +methods query the result: `Def(name)` looks up by node ID, bare name, or +qualified name; `Callers`/`Callees` return one-hop call edges; +`Affected(seeds, opts)` returns reverse-reachable evidence paths from a set of +sinks; `Path(from, to, opts)` returns the shortest forward call chain; +`JSON(w)` writes sorted output so repeated builds of unchanged input are +byte-identical. Call extraction and cross-file resolution currently cover Go +and Python; for other supported languages the graph contains file, symbol, +module, and import structure with call edges omitted. + `Tree(paths []string) string` renders a box-drawing directory tree from a flat path list. @@ -90,6 +104,41 @@ language with an outlining query. `SetParseTimeout(d time.Duration)` overrides the per-file parse timeout (default 1s). Must be called before the first `Outline` or `Pack` call. +## outline binary + +``` +go install github.com/git-pkgs/outline/cmd/outline@latest +``` + +`outline graph [dir]` builds the graph and prints a summary; `-json` writes it +to stdout and `-o file` to disk. `outline def `, +`outline callers `, `outline callees `, +`outline affected ...`, and `outline path ` query it. `-g +file` reads a saved graph instead of rebuilding; `-inferred` includes +cross-file name-matched edges alongside extracted ones; `-depth N` caps +traversal hops; `-relations r,...` selects which edge kinds to follow (default +`calls`); `-budget N` caps output size in tokens. Flags precede positional +arguments. + +Seeds take the same forms as `Def`: `Handler`, `Handler.process`, or a full +node ID. External targets use `ext:::`, so tracing every +declaration that reaches `exec.Command` from the `testdata/cli-go` fixture +produces: + +``` +$ outline affected -inferred -dir testdata/cli-go ext:go:os/exec:Command +2 paths, 3 nodes +NODE Handler func main.go:9 exported=true sig=func Handler(name string) error +NODE os/exec.Command external exported=false sig= +NODE main func main.go:17 exported=false sig=func main() +EDGE Handler --calls[inferred]--> Run at main.go:14 +NODE Run external exported=false sig= +EDGE Handler --calls[inferred]--> os/exec.Command at main.go:14 +EDGE Handler --calls[inferred]--> Load at main.go:10 +NODE Load func store/store.go:7 exported=true sig=func Load(name string) (Record, error) +EDGE main --calls[extracted]--> Handler at main.go:18 +``` + ## Languages 35 languages have body-stripping queries: Go, Ruby, Python, JavaScript, diff --git a/analyse.go b/analyse.go new file mode 100644 index 0000000..151af9d --- /dev/null +++ b/analyse.go @@ -0,0 +1,213 @@ +package outline + +import ( + "sort" + + ts "github.com/odvcencio/gotreesitter" +) + +// decl is a declaration captured for graph construction. Unlike Symbol it +// retains the full definition span and nesting so callers can compute node +// identities and containment before the tree is released. +type decl struct { + Name string + Kind string + Line int + Exported bool + NameAt uint32 + Start uint32 + End uint32 + SigEnd uint32 + Parent int + Params []string +} + +func (d decl) symID(path string) string { + return SymID(path, d.NameAt) +} + +// analysis is the per-file fact set that graph resolution consumes. +type analysis struct { + Lang string + Decls []decl + Imports []Import + Calls []Call +} + +// analyse parses src once and returns every fact the graph builder needs +// from a single file. The bool is false when the file's language is +// unsupported or the parse failed. +func analyse(src []byte, filename string) (*analysis, bool) { + l, tree, ok := parseSource(src, filename) + if !ok { + return nil, false + } + defer tree.Release() + + root := tree.RootNode() + matches := l.query.Execute(tree) + + a := &analysis{Lang: l.name} + a.Decls = extractDecls(src, l, root, matches) + a.Imports, _ = importsFor(src, l, root) + a.Calls, _ = callsFor(src, l, root, a.Decls) + return a, true +} + +func extractDecls(src []byte, l *lang, root *ts.Node, matches []ts.QueryMatch) []decl { + raw := make([]decl, 0, len(matches)) + for _, m := range matches { + raw = append(raw, declsFromMatch(src, l, m)...) + } + if exported, merge, ok := explicitExports(l, root, src); ok { + for i := range raw { + if merge { + raw[i].Exported = raw[i].Exported || exported[raw[i].Name] + } else { + raw[i].Exported = exported[raw[i].Name] + } + } + } + if len(raw) == 0 { + return nil + } + sort.Slice(raw, func(i, j int) bool { + if raw[i].Start != raw[j].Start { + return raw[i].Start < raw[j].Start + } + if raw[i].End != raw[j].End { + return raw[i].End > raw[j].End + } + return raw[i].NameAt < raw[j].NameAt + }) + assignParents(raw) + return raw +} + +func declsFromMatch(src []byte, l *lang, m ts.QueryMatch) []decl { + var definition *ts.Node + var kind string + exported := false + names := make([]*ts.Node, 0, 1) + for _, cap := range m.Captures { + switch cap.Name { + case "symbol.name": + names = append(names, cap.Node) + case "symbol.exported": + exported = true + case "symbol.func", "symbol.type", "symbol.class", "symbol.const", "symbol.var": + definition = cap.Node + kind = cap.Name[len("symbol."):] + } + } + if definition == nil || kind == "" { + return nil + } + start := definition.StartByte() + end := definition.EndByte() + sigEnd := end + if body := definition.ChildByFieldName("body", l.language); body != nil { + sigEnd = body.StartByte() + } + params := extractParams(src, l, definition) + out := make([]decl, 0, len(names)) + for _, n := range names { + if n == nil { + continue + } + name := n.Text(src) + if name == "" || name == "_" { + continue + } + out = append(out, decl{ + Name: name, + Kind: normalizeSymbolKind(l.name, kind, name, definition, src, l.language), + Line: int(n.StartPoint().Row) + 1, + Exported: exported || symbolExported(l.name, name, definition, src, l.language), + NameAt: n.StartByte(), + Start: start, + End: end, + SigEnd: sigEnd, + Parent: -1, + Params: params, + }) + } + return out +} + +// extractParams returns the parameter (and, for Go, receiver and named +// result) identifiers introduced by a function declaration, so lexical +// resolution can detect when a call target is shadowed. +func extractParams(src []byte, l *lang, def *ts.Node) []string { + var out []string + collect := func(list *ts.Node) { + if list == nil { + return + } + for i := range list.NamedChildCount() { + p := list.NamedChild(i) + switch p.Type(l.language) { + case "identifier": + out = append(out, p.Text(src)) + case "parameter_declaration", "variadic_parameter_declaration": + for j := range p.NamedChildCount() { + if c := p.NamedChild(j); c.Type(l.language) == "identifier" { + out = append(out, c.Text(src)) + } + } + default: + if n := p.ChildByFieldName("name", l.language); n != nil { + out = append(out, n.Text(src)) + } else if p.NamedChildCount() > 0 { + if c := p.NamedChild(0); c.Type(l.language) == "identifier" { + out = append(out, c.Text(src)) + } + } + } + } + } + switch l.name { + case "go": + collect(def.ChildByFieldName("receiver", l.language)) + collect(def.ChildByFieldName("parameters", l.language)) + collect(def.ChildByFieldName("result", l.language)) + case "python": + collect(def.ChildByFieldName("parameters", l.language)) + } + return out +} + +// assignParents sets Parent on each decl to the index of the innermost +// enclosing decl. Input must be sorted by Start ascending, End descending. +func assignParents(decls []decl) { + var stack []int + for i := range decls { + for len(stack) > 0 && decls[stack[len(stack)-1]].End <= decls[i].Start { + stack = stack[:len(stack)-1] + } + if len(stack) > 0 { + top := stack[len(stack)-1] + if decls[top].Start == decls[i].Start && decls[top].End == decls[i].End { + decls[i].Parent = decls[top].Parent + } else { + decls[i].Parent = top + } + } + stack = append(stack, i) + } +} + +// enclosing returns the index of the innermost decl whose span contains pos, +// or -1 if none does. +func enclosing(decls []decl, pos uint32) int { + best := -1 + for i := range decls { + if decls[i].Start > pos { + break + } + if decls[i].End > pos { + best = i + } + } + return best +} diff --git a/analyse_test.go b/analyse_test.go new file mode 100644 index 0000000..8e18f47 --- /dev/null +++ b/analyse_test.go @@ -0,0 +1,203 @@ +package outline + +import ( + "slices" + "testing" +) + +func declByName(t *testing.T, decls []decl, name string) (int, decl) { + t.Helper() + for i, d := range decls { + if d.Name == name { + return i, d + } + } + t.Fatalf("decl %q not found in %v", name, decls) + return -1, decl{} +} + +func callByName(t *testing.T, calls []Call, name string) Call { + t.Helper() + for _, c := range calls { + if c.Name == name { + return c + } + } + t.Fatalf("call %q not found in %v", name, calls) + return Call{} +} + +func TestAnalysePythonNesting(t *testing.T) { + src := []byte(`import os + +class Handler: + def process(self, path): + def helper(): + return os.getcwd() + return helper() + +def top(): + Handler().process(".") +`) + a, ok := analyse(src, "h.py") + if !ok { + t.Fatal("analyse returned false") + } + if a.Lang != "python" { + t.Fatalf("lang = %q", a.Lang) + } + + iH, _ := declByName(t, a.Decls, "Handler") + iP, dP := declByName(t, a.Decls, "process") + iHelp, dHelp := declByName(t, a.Decls, "helper") + _, dTop := declByName(t, a.Decls, "top") + + if a.Decls[iH].Parent != -1 { + t.Errorf("Handler parent = %d, want -1", a.Decls[iH].Parent) + } + if dP.Parent != iH { + t.Errorf("process parent = %d, want %d (Handler)", dP.Parent, iH) + } + if dHelp.Parent != iP { + t.Errorf("helper parent = %d, want %d (process)", dHelp.Parent, iP) + } + if dTop.Parent != -1 { + t.Errorf("top parent = %d, want -1", dTop.Parent) + } + if dP.Kind != "func" || a.Decls[iH].Kind != "class" { + t.Errorf("kinds: Handler=%q process=%q", a.Decls[iH].Kind, dP.Kind) + } + if a.Decls[iH].Start >= dP.Start || dP.End > a.Decls[iH].End { + t.Errorf("process span [%d,%d) not inside Handler [%d,%d)", + dP.Start, dP.End, a.Decls[iH].Start, a.Decls[iH].End) + } + + if len(a.Imports) != 1 || a.Imports[0].Module != "os" { + t.Errorf("imports = %v", a.Imports) + } + + getcwd := callByName(t, a.Calls, "getcwd") + if getcwd.Receiver != "os" || getcwd.In != iHelp { + t.Errorf("getcwd: receiver=%q in=%d, want os/%d", getcwd.Receiver, getcwd.In, iHelp) + } + helperCall := callByName(t, a.Calls, "helper") + if helperCall.Receiver != "" || helperCall.In != iP { + t.Errorf("helper call: receiver=%q in=%d, want \"\"/%d", helperCall.Receiver, helperCall.In, iP) + } +} + +func TestAnalyseGoCalls(t *testing.T) { + src := []byte(`package main + +import "os/exec" + +type Runner struct{} + +func (r Runner) Run(name string) error { + cmd := exec.Command(name) + return cmd.Run() +} + +func main() { + check(Runner{}.Run("ls")) +} +`) + a, ok := analyse(src, "main.go") + if !ok { + t.Fatal("analyse returned false") + } + + iRun, dRun := declByName(t, a.Decls, "Run") + iMain, _ := declByName(t, a.Decls, "main") + _, dRunner := declByName(t, a.Decls, "Runner") + + if dRun.Parent != -1 || dRunner.Parent != -1 { + t.Errorf("Go top-level decls should have parent -1: Run=%d Runner=%d", dRun.Parent, dRunner.Parent) + } + if dRun.SigEnd >= dRun.End || dRun.SigEnd <= dRun.Start { + t.Errorf("Run SigEnd=%d not between Start=%d End=%d", dRun.SigEnd, dRun.Start, dRun.End) + } + + if len(a.Imports) != 1 || a.Imports[0].Module != "os/exec" { + t.Errorf("imports = %v", a.Imports) + } + + cmd := callByName(t, a.Calls, "Command") + if cmd.Receiver != "exec" || cmd.In != iRun { + t.Errorf("Command: receiver=%q in=%d, want exec/%d", cmd.Receiver, cmd.In, iRun) + } + check := callByName(t, a.Calls, "check") + if check.Receiver != "" || check.In != iMain { + t.Errorf("check: receiver=%q in=%d, want \"\"/%d", check.Receiver, check.In, iMain) + } +} + +func TestAnalyseParams(t *testing.T) { + a, ok := analyse([]byte(`package m +func F(a, b int, c ...string) (err error) { return } +`), "m.go") + if !ok { + t.Fatal("analyse failed") + } + _, d := declByName(t, a.Decls, "F") + for _, p := range []string{"a", "b", "c", "err"} { + if !slices.Contains(d.Params, p) { + t.Errorf("Go params missing %q: %v", p, d.Params) + } + } + + a, ok = analyse([]byte("def f(x, y=1, *args, z: int, **kw): pass\n"), "m.py") + if !ok { + t.Fatal("analyse failed") + } + _, d = declByName(t, a.Decls, "f") + for _, p := range []string{"x", "y", "args", "z", "kw"} { + if !slices.Contains(d.Params, p) { + t.Errorf("Python params missing %q: %v", p, d.Params) + } + } +} + +func TestAnalyseExplicitExports(t *testing.T) { + a, ok := analyse([]byte("function f() {}\nfunction g() {}\nexport { f };\n"), "m.js") + if !ok { + t.Fatal("analyse failed") + } + _, df := declByName(t, a.Decls, "f") + _, dg := declByName(t, a.Decls, "g") + if !df.Exported { + t.Errorf("f should be exported via export list") + } + if dg.Exported { + t.Errorf("g should not be exported") + } + + a, ok = analyse([]byte("proc p*() = discard\nproc q() = discard\n"), "m.nim") + if !ok { + t.Fatal("analyse failed") + } + _, dp := declByName(t, a.Decls, "p") + _, dq := declByName(t, a.Decls, "q") + if !dp.Exported || dq.Exported { + t.Errorf("Nim export marker: p=%v q=%v", dp.Exported, dq.Exported) + } +} + +func TestEnclosing(t *testing.T) { + decls := []decl{ + {Name: "a", Start: 0, End: 100}, + {Name: "b", Start: 10, End: 50}, + {Name: "c", Start: 60, End: 90}, + } + cases := []struct { + pos uint32 + want int + }{ + {5, 0}, {20, 1}, {55, 0}, {70, 2}, {200, -1}, + } + for _, tc := range cases { + if got := enclosing(decls, tc.pos); got != tc.want { + t.Errorf("enclosing(%d) = %d, want %d", tc.pos, got, tc.want) + } + } +} diff --git a/build.go b/build.go new file mode 100644 index 0000000..b1a3d79 --- /dev/null +++ b/build.go @@ -0,0 +1,243 @@ +package outline + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "unicode/utf8" + + "github.com/git-pkgs/gitignore" +) + +// ResolutionHints carries caller-supplied context that improves module +// resolution beyond what the repository files alone reveal. +type ResolutionHints struct { + SourceRoots []string + Ecosystems []string + Packages []PackageHint +} + +// PackageHint maps an installed distribution to the import names it +// provides, for ecosystems where the two differ. +type PackageHint struct { + Ecosystem string + Name string + Imports []string +} + +// ToolVersion is stamped into every emitted graph. +var ToolVersion = "dev" + +const sigCap = 240 + +// fileAnalysis holds one file's raw bytes and extracted facts. +type fileAnalysis struct { + path string + src []byte + a *analysis + skipped string +} + +// Build walks root, analyses every supported source file, resolves +// cross-file references for supported languages, and returns a Graph. +func Build(root string, opts Options) (*Graph, error) { + abs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if opts.MaxFileSize == 0 { + opts.MaxFileSize = defaultMaxFileSize + } + if opts.MaxFiles == 0 { + opts.MaxFiles = defaultMaxFiles + } + if opts.Concurrency == 0 { + opts.Concurrency = runtime.NumCPU() + } + + m := gitignore.New(abs) + m.AddPatterns(defaultIgnore, "") + for _, p := range opts.Ignore { + m.AddPatterns([]byte(p+"\n"), "") + } + paths, truncated, err := collect(abs, m, opts.MaxFiles) + if err != nil { + return nil, err + } + sort.Strings(paths) + + files := analyseAll(abs, paths, opts) + + g := &Graph{ + SchemaVersion: SchemaVersion, + ToolVersion: ToolVersion, + SourceDigest: sourceDigest(files), + Complete: !truncated, + } + if truncated { + g.Warnings = append(g.Warnings, fmt.Sprintf("file limit %d reached; graph is partial", opts.MaxFiles)) + } + + emitFileNodes(g, files) + r := newResolver(abs, files, opts.Resolution) + r.emitModules(g) + r.emitCalls(g) + + g.sortStable() + return g, nil +} + +func analyseAll(root string, paths []string, opts Options) []fileAnalysis { + files := make([]fileAnalysis, len(paths)) + work := make(chan int) + var wg sync.WaitGroup + for range opts.Concurrency { + wg.Go(func() { + for i := range work { + files[i] = analyseOne(root, paths[i], opts) + } + }) + } + for i := range paths { + work <- i + } + close(work) + wg.Wait() + return files +} + +func analyseOne(root, path string, opts Options) fileAnalysis { + fa := fileAnalysis{path: path} + full := filepath.Join(root, filepath.FromSlash(path)) + info, err := os.Lstat(full) + if err != nil { + fa.skipped = "unreadable" + return fa + } + if info.Size() > opts.MaxFileSize { + fa.skipped = "too-large" + return fa + } + if _, ok := detect(path); !ok { + fa.skipped = "unsupported" + return fa + } + src, err := os.ReadFile(full) + if err != nil { + fa.skipped = "unreadable" + return fa + } + a, ok := analyse(src, path) + if !ok { + fa.skipped = "parse-failed" + return fa + } + fa.src = src + fa.a = a + return fa +} + +func emitFileNodes(g *Graph, files []fileAnalysis) { + for _, f := range files { + fid := FileID(f.path) + g.Nodes = append(g.Nodes, Node{ID: fid, Kind: KindFile, Name: f.path, File: f.path}) + if f.a == nil { + if f.skipped != "" && f.skipped != "unsupported" { + g.Warnings = append(g.Warnings, f.path+": "+f.skipped) + g.Complete = false + } + continue + } + for i, d := range f.a.Decls { + sid := d.symID(f.path) + g.Nodes = append(g.Nodes, Node{ + ID: sid, + Kind: d.Kind, + Name: d.Name, + Qualified: qualified(f.a.Decls, i), + File: f.path, + Line: d.Line, + Start: int(d.Start), + End: int(d.End), + Exported: d.Exported, + Sig: signature(f.src, d), + }) + from := fid + if d.Parent >= 0 { + from = f.a.Decls[d.Parent].symID(f.path) + } + g.Edges = append(g.Edges, Edge{ + From: from, To: sid, Rel: RelContains, Conf: ConfExtracted, + File: f.path, Line: d.Line, + }) + } + } +} + +func qualified(decls []decl, i int) string { + parts := []string{decls[i].Name} + for p := decls[i].Parent; p >= 0; p = decls[p].Parent { + parts = append(parts, decls[p].Name) + } + for l, r := 0, len(parts)-1; l < r; l, r = l+1, r-1 { + parts[l], parts[r] = parts[r], parts[l] + } + return strings.Join(parts, ".") +} + +func signature(src []byte, d decl) string { + if int(d.Start) >= len(src) { + return "" + } + end := min(d.SigEnd, uint32(len(src))) + s := sanitise(string(src[d.Start:end])) + if len(s) > sigCap { + cut := sigCap + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + s = s[:cut] + } + return s +} + +// sanitise collapses whitespace runs to a single space and drops control +// bytes so signatures and text output stay one-line safe. +func sanitise(s string) string { + var b strings.Builder + b.Grow(len(s)) + space := false + for i := 0; i < len(s); i++ { + c := s[i] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + space = true + continue + } + if c < ' ' { + continue + } + if space && b.Len() > 0 { + b.WriteByte(' ') + } + space = false + b.WriteByte(c) + } + return b.String() +} + +func sourceDigest(files []fileAnalysis) string { + h := sha256.New() + for _, f := range files { + h.Write([]byte(f.path)) + h.Write([]byte{0}) + fh := sha256.Sum256(f.src) + h.Write(fh[:]) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/build_test.go b/build_test.go new file mode 100644 index 0000000..b583c0c --- /dev/null +++ b/build_test.go @@ -0,0 +1,479 @@ +package outline + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" +) + +func writeFiles(t *testing.T, root string, files map[string]string) { + t.Helper() + for p, content := range files { + full := filepath.Join(root, filepath.FromSlash(p)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func edge(g *Graph, from, to, rel string) *Edge { + for i := range g.Edges { + e := &g.Edges[i] + if e.From == from && e.To == to && e.Rel == rel { + return e + } + } + return nil +} + +func nodeByName(g *Graph, kind, name string) *Node { + for i := range g.Nodes { + if g.Nodes[i].Kind == kind && g.Nodes[i].Name == name { + return &g.Nodes[i] + } + } + return nil +} + +func TestBuildGoCrossPackage(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module example.com/app\n\ngo 1.26\n", + "main.go": `package main + +import ( + "os/exec" + "example.com/app/util" +) + +func main() { + util.Run("ls") + exec.Command("ls") +} +`, + "util/util.go": `package util + +func Run(name string) error { + return nil +} +`, + }) + + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + + mainFn := nodeByName(g, KindFunc, "main") + runFn := nodeByName(g, KindFunc, "Run") + execMod := nodeByName(g, KindModule, "os/exec") + if mainFn == nil || runFn == nil || execMod == nil { + t.Fatalf("missing nodes: main=%v Run=%v exec=%v", mainFn, runFn, execMod) + } + if !runFn.Exported { + t.Errorf("Run should be exported") + } + if runFn.Sig == "" || !bytes.Contains([]byte(runFn.Sig), []byte("name string")) { + t.Errorf("Run sig = %q, want to contain parameter name", runFn.Sig) + } + + utilMod := ModID("go", "example.com/app/util") + if g.Node(utilMod) == nil { + t.Fatalf("missing module node %s", utilMod) + } + if e := edge(g, FileID("main.go"), utilMod, RelImports); e == nil { + t.Error("missing main.go -> util imports edge") + } + if e := edge(g, utilMod, FileID("util/util.go"), RelContains); e == nil { + t.Error("util module did not resolve to util/util.go") + } + + call := edge(g, mainFn.ID, runFn.ID, RelCalls) + if call == nil { + t.Fatalf("missing main -> util.Run calls edge; edges from main: %v", g.Out(mainFn.ID)) + } + if call.Conf != ConfInferred { + t.Errorf("cross-package call conf = %q, want inferred", call.Conf) + } + + extCmd := ExtID("go", "os/exec", "Command") + if g.Node(extCmd) == nil { + t.Fatalf("missing external node %s", extCmd) + } + if e := edge(g, mainFn.ID, extCmd, RelCalls); e == nil { + t.Error("missing main -> exec.Command external edge") + } +} + +func TestBuildPythonCrossModule(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "app.py": `import subprocess +from util import run + +def handler(name): + run(name) + subprocess.call(name) +`, + "util.py": `def run(name): + pass +`, + }) + + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + + handler := nodeByName(g, KindFunc, "handler") + runFn := nodeByName(g, KindFunc, "run") + if handler == nil || runFn == nil { + t.Fatalf("missing nodes: handler=%v run=%v", handler, runFn) + } + + utilMod := ModID("python", "util") + if e := edge(g, utilMod, FileID("util.py"), RelContains); e == nil { + t.Error("util module did not resolve to util.py") + } + + call := edge(g, handler.ID, runFn.ID, RelCalls) + if call == nil { + t.Fatalf("missing handler -> run calls edge; edges: %v", g.Out(handler.ID)) + } + if call.Conf != ConfInferred { + t.Errorf("cross-module call conf = %q, want inferred", call.Conf) + } + + extCall := ExtID("python", "subprocess", "call") + if g.Node(extCall) == nil { + t.Fatalf("missing external node %s", extCall) + } + if e := edge(g, handler.ID, extCall, RelCalls); e == nil { + t.Error("missing handler -> subprocess.call external edge") + } +} + +func TestBuildSameFileExtracted(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "a.py": "def a():\n b()\n\ndef b():\n pass\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + aFn := nodeByName(g, KindFunc, "a") + bFn := nodeByName(g, KindFunc, "b") + e := edge(g, aFn.ID, bFn.ID, RelCalls) + if e == nil { + t.Fatal("missing a -> b calls edge") + } + if e.Conf != ConfExtracted { + t.Errorf("same-file call conf = %q, want extracted", e.Conf) + } +} + +func TestBuildGoSamePackage(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module m\n", + "a.go": "package m\nfunc A() { helper() }\n", + "b.go": "package m\nfunc helper() {}\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + aFn := nodeByName(g, KindFunc, "A") + hFn := nodeByName(g, KindFunc, "helper") + e := edge(g, aFn.ID, hFn.ID, RelCalls) + if e == nil { + t.Fatalf("missing A -> helper same-package edge; edges from A: %v", g.Out(aFn.ID)) + } + if e.Conf != ConfInferred { + t.Errorf("same-package cross-file conf = %q, want inferred", e.Conf) + } + if n := nodeByName(g, KindExternal, "helper"); n != nil { + t.Errorf("spurious external node for resolved same-package call: %v", n) + } +} + +func TestBuildGoMultiName(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module m\n", + "m.go": "package m\n\nvar a, b int\n\nconst c, d = 1, 2\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + seen := make(map[string]string) + for _, n := range g.Nodes { + if other, dup := seen[n.ID]; dup { + t.Errorf("duplicate node ID %s: %q and %q", n.ID, other, n.Name) + } + seen[n.ID] = n.Name + } + for _, name := range []string{"a", "b", "c", "d"} { + if nodeByName(g, KindVar, name) == nil && nodeByName(g, KindConst, name) == nil { + t.Errorf("missing decl node %q", name) + } + } + for _, e := range g.Edges { + if e.Rel == RelContains && e.From != FileID("m.go") && g.Node(e.From).Kind != KindFile { + t.Errorf("multi-name decl got non-file parent: %v", e) + } + } +} + +func TestBuildPythonRelativeDistinct(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "a/__init__.py": "", + "a/app.py": "from .util import run\ndef fa(): run()\n", + "a/util.py": "def run(): pass\n", + "b/__init__.py": "", + "b/app.py": "from .util import run\ndef fb(): run()\n", + "b/util.py": "def run(): pass\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + fa := nodeByName(g, KindFunc, "fa") + fb := nodeByName(g, KindFunc, "fb") + if fa == nil || fb == nil { + t.Fatalf("missing fa=%v fb=%v", fa, fb) + } + target := func(from string) string { + for _, e := range g.Out(from) { + if e.Rel == RelCalls { + return e.To + } + } + return "" + } + ta, tb := target(fa.ID), target(fb.ID) + if !strings.Contains(ta, "a/util.py") { + t.Errorf("fa resolved to %q, want a/util.py", ta) + } + if !strings.Contains(tb, "b/util.py") { + t.Errorf("fb resolved to %q, want b/util.py", tb) + } + if ta == tb { + t.Errorf("both packages resolved to same target %q", ta) + } +} + +func TestBuildExtQualifiedCanonical(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "a.py": "import subprocess as one\ndef fa(): one.run('x')\n", + "b.py": "import subprocess as two\ndef fb(): two.run('x')\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + ext := g.Node(ExtID("python", "subprocess", "run")) + if ext == nil { + t.Fatal("missing ext:python:subprocess:run node") + } + if ext.Qualified != "subprocess.run" { + t.Errorf("ext Qualified = %q, want subprocess.run", ext.Qualified) + } + if hits := g.Def("subprocess.run"); len(hits) != 1 || hits[0].ID != ext.ID { + t.Errorf("Def(subprocess.run) = %v", hits) + } +} + +func TestBuildGoModuleBoundary(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module example.com/svc\n", + "main.go": "package main\nimport \"example.com/svc2/util\"\nfunc main() { util.Run() }\n", + "2/util/util.go": "package util\nfunc Run() {}\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + mid := ModID("go", "example.com/svc2/util") + if e := edge(g, mid, FileID("2/util/util.go"), RelContains); e != nil { + t.Errorf("svc2 import wrongly resolved into svc module: %v", e) + } +} + +func TestBuildGoVersionedAlias(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module m\n", + "main.go": "package main\nimport \"github.com/foo/bar/v3\"\nfunc main() { bar.Do() }\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + want := ExtID("go", "github.com/foo/bar/v3", "Do") + if g.Node(want) == nil { + t.Fatalf("bar.Do() did not resolve via versioned alias; want %s, ext nodes: %v", want, g.Def("Do")) + } + if defaultAlias("go", "gopkg.in/vault") != "vault" { + t.Errorf("non-numeric v-prefix segment stripped: %q", defaultAlias("go", "gopkg.in/vault")) + } +} + +func TestBuildPythonRelativeExtDistinct(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "a/__init__.py": "", + "a/app.py": "from .missing import run\ndef fa(): run()\n", + "b/__init__.py": "", + "b/app.py": "from .missing import run\ndef fb(): run()\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + fa := nodeByName(g, KindFunc, "fa") + fb := nodeByName(g, KindFunc, "fb") + target := func(from string) string { + for _, e := range g.Out(from) { + if e.Rel == RelCalls { + return e.To + } + } + return "" + } + ta, tb := target(fa.ID), target(fb.ID) + if ta == tb { + t.Errorf("unresolved relative imports collided: fa->%q fb->%q", ta, tb) + } + if !strings.HasPrefix(ta, "ext:") || !strings.HasPrefix(tb, "ext:") { + t.Errorf("expected ext: targets, got %q %q", ta, tb) + } +} + +func TestBuildLexicalScope(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "n.py": `def outer(): + def inner(): + pass + inner() + +def helper(): + pass + +class C: + def m(self): + m() + helper() + +def caller(helper): + helper() +`, + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + outer := nodeByName(g, KindFunc, "outer") + inner := nodeByName(g, KindFunc, "inner") + helperFn := nodeByName(g, KindFunc, "helper") + m := nodeByName(g, KindFunc, "m") + caller := nodeByName(g, KindFunc, "caller") + + if e := edge(g, outer.ID, inner.ID, RelCalls); e == nil || e.Conf != ConfExtracted { + t.Errorf("nested inner() should resolve extracted: %v", e) + } + if e := edge(g, m.ID, m.ID, RelCalls); e != nil { + t.Errorf("m() inside method should not resolve to the method: %v", e) + } + if e := edge(g, m.ID, helperFn.ID, RelCalls); e == nil || e.Conf != ConfExtracted { + t.Errorf("helper() from method should resolve to top-level extracted: %v", e) + } + if e := edge(g, caller.ID, helperFn.ID, RelCalls); e != nil { + t.Errorf("helper() shadowed by param should not resolve to top-level: %v", e) + } + extHelper := ExtID("python", "", "helper") + if e := edge(g, caller.ID, extHelper, RelCalls); e == nil { + t.Errorf("shadowed helper() should produce ext edge; edges from caller: %v", g.Out(caller.ID)) + } +} + +func TestBuildGoParamShadow(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module example.com/app\n", + "main.go": `package main +import "example.com/app/util" +func Handler() {} +func A(Handler func()) { Handler() } +func B(util T) { util.Run() } +`, + "util/util.go": "package util\nfunc Run() {}\n", + }) + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + a := nodeByName(g, KindFunc, "A") + handler := nodeByName(g, KindFunc, "Handler") + if e := edge(g, a.ID, handler.ID, RelCalls); e != nil { + t.Errorf("Handler() shadowed by param should not resolve to top-level: %v", e) + } + b := nodeByName(g, KindFunc, "B") + runFn := nodeByName(g, KindFunc, "Run") + if e := edge(g, b.ID, runFn.ID, RelCalls); e != nil { + t.Errorf("util.Run() with util param should not resolve via import: %v", e) + } +} + +func TestBuildDeterministic(t *testing.T) { + root := t.TempDir() + // pkg defines D in two files, as build-tag variants do, so resolution + // must pick the same declaration on every build. + writeFiles(t, root, map[string]string{ + "go.mod": "module m\n", + "a.go": "package m\n\nimport \"m/pkg\"\n\nfunc A() { B(); pkg.D() }\n", + "b.go": "package m\nfunc B() {}\n", + "pkg/d1.go": "package pkg\nfunc D() {}\n", + "pkg/d2.go": "package pkg\nfunc D() {}\n", + "c/c.py": "def c(): pass\n", + }) + var out [4]bytes.Buffer + for i := range out { + g, err := Build(root, Options{}) + if err != nil { + t.Fatal(err) + } + if err := g.JSON(&out[i]); err != nil { + t.Fatal(err) + } + } + for i := 1; i < len(out); i++ { + if !bytes.Equal(out[0].Bytes(), out[i].Bytes()) { + t.Fatalf("Build not deterministic:\n%s\n---\n%s", out[0].String(), out[i].String()) + } + } +} + +func TestSignatureRuneBoundary(t *testing.T) { + src := []byte("func " + strings.Repeat("a", sigCap-6) + "é()") + d := decl{Start: 0, End: uint32(len(src)), SigEnd: uint32(len(src))} + s := signature(src, d) + if !utf8.ValidString(s) { + t.Errorf("signature is not valid UTF-8: %q", s) + } + if len(s) > sigCap { + t.Errorf("signature length %d exceeds cap %d", len(s), sigCap) + } +} diff --git a/call.go b/call.go new file mode 100644 index 0000000..6c5988c --- /dev/null +++ b/call.go @@ -0,0 +1,109 @@ +package outline + +import ts "github.com/odvcencio/gotreesitter" + +// Call is one call site. Receiver is the leftmost identifier of a member +// call (a.b()) or empty for a bare call (b()). In is the index into the +// file's decl slice of the innermost enclosing declaration, or -1 for +// top-level calls. +type Call struct { + Receiver string + Name string + Line int + Start uint32 + In int +} + +func callsFor(src []byte, l *lang, root *ts.Node, decls []decl) ([]Call, bool) { + var calls []Call + switch l.name { + case "go": + calls = goCalls(src, l.language, root) + case "python": + calls = pythonCalls(src, l.language, root) + default: + return nil, false + } + for i := range calls { + calls[i].In = enclosing(decls, calls[i].Start) + } + return calls, true +} + +func goCalls(src []byte, language *ts.Language, root *ts.Node) []Call { + var calls []Call + walkNamed(root, func(node *ts.Node) { + if node.Type(language) != "call_expression" { + return + } + fn := node.ChildByFieldName("function", language) + if fn == nil { + return + } + c := Call{Line: sourceLine(node), Start: node.StartByte()} + switch fn.Type(language) { + case "identifier": + c.Name = fn.Text(src) + case "selector_expression": + recv := fn.ChildByFieldName("operand", language) + member := fn.ChildByFieldName("field", language) + if member == nil { + return + } + c.Name = member.Text(src) + if recv != nil && recv.Type(language) == "identifier" { + c.Receiver = recv.Text(src) + } + default: + return + } + calls = append(calls, c) + }) + return calls +} + +func pythonCalls(src []byte, language *ts.Language, root *ts.Node) []Call { + var calls []Call + walkNamed(root, func(node *ts.Node) { + if node.Type(language) != "call" { + return + } + fn := node.ChildByFieldName("function", language) + if fn == nil { + return + } + c := Call{Line: sourceLine(node), Start: node.StartByte()} + switch fn.Type(language) { + case "identifier": + c.Name = fn.Text(src) + case "attribute": + recv := fn.ChildByFieldName("object", language) + member := fn.ChildByFieldName("attribute", language) + if member == nil { + return + } + c.Name = member.Text(src) + c.Receiver = leftmostIdentifier(src, language, recv) + default: + return + } + calls = append(calls, c) + }) + return calls +} + +// leftmostIdentifier walks a chained attribute expression (a.b.c) and +// returns the base identifier text, or "" if the base is not a plain name. +func leftmostIdentifier(src []byte, language *ts.Language, node *ts.Node) string { + for node != nil { + switch node.Type(language) { + case "identifier": + return node.Text(src) + case "attribute": + node = node.ChildByFieldName("object", language) + default: + return "" + } + } + return "" +} diff --git a/cmd/outline/main.go b/cmd/outline/main.go new file mode 100644 index 0000000..60fcaa7 --- /dev/null +++ b/cmd/outline/main.go @@ -0,0 +1,298 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "strings" + + "github.com/git-pkgs/outline" +) + +const ( + exitUsage = 2 + defaultBudget = 2000 + pathArgs = 2 +) + +const usage = `outline graph [-json] [-o file] [dir] +outline def [-g file] [-dir path] +outline callers [-g file] [-dir path] [-inferred] +outline callees [-g file] [-dir path] [-inferred] +outline affected [-g file] [-dir path] [-depth N] [-relations r,...] [-inferred] ... +outline path [-g file] [-dir path] [-relations r,...] [-inferred] + +Flags must precede positional arguments. +` + +func main() { + if len(os.Args) <= 1 { + fmt.Fprint(os.Stderr, usage) + os.Exit(exitUsage) + } + var err error + switch os.Args[1] { + case "graph": + err = cmdGraph(os.Args[2:]) + case "def": + err = cmdDef(os.Args[2:]) + case "callers": + err = cmdNeighbours(os.Args[2:], true) + case "callees": + err = cmdNeighbours(os.Args[2:], false) + case "affected": + err = cmdAffected(os.Args[2:]) + case "path": + err = cmdPath(os.Args[2:]) + default: + fmt.Fprint(os.Stderr, usage) + os.Exit(exitUsage) + } + if err != nil { + fmt.Fprintln(os.Stderr, "outline:", err) + os.Exit(1) + } +} + +type common struct { + graph string + dir string + depth int + inferred bool + rels string + budget int +} + +func (c *common) register(fs *flag.FlagSet) { + fs.StringVar(&c.graph, "g", "", "load graph from file instead of building") + fs.StringVar(&c.dir, "dir", ".", "repository root when building") + fs.IntVar(&c.depth, "depth", 0, "hop limit (0 = unbounded)") + fs.BoolVar(&c.inferred, "inferred", false, "follow inferred edges") + fs.StringVar(&c.rels, "relations", "", "comma-separated relations to follow") + fs.IntVar(&c.budget, "budget", defaultBudget, "output token budget") +} + +func (c *common) traverse() outline.TraverseOptions { + opts := outline.TraverseOptions{Depth: c.depth, IncludeInferred: c.inferred} + if c.rels != "" { + opts.Relations = strings.Split(c.rels, ",") + } + return opts +} + +func (c *common) load() (*outline.Graph, error) { + if c.graph != "" { + return loadGraph(c.graph) + } + return outline.Build(c.dir, outline.Options{}) +} + +func loadGraph(path string) (*outline.Graph, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var g outline.Graph + if err := json.Unmarshal(data, &g); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if g.SchemaVersion != outline.SchemaVersion { + return nil, fmt.Errorf("%s: schema version %d, this build supports %d", + path, g.SchemaVersion, outline.SchemaVersion) + } + return &g, nil +} + +func cmdGraph(args []string) error { + fs := flag.NewFlagSet("graph", flag.ExitOnError) + jsonOut := fs.Bool("json", false, "write JSON to stdout") + out := fs.String("o", "", "write JSON to file") + if err := fs.Parse(args); err != nil { + return err + } + dir := "." + if fs.NArg() > 0 { + dir = fs.Arg(0) + } + g, err := outline.Build(dir, outline.Options{}) + if err != nil { + return err + } + if *out != "" { + f, err := os.Create(*out) + if err != nil { + return err + } + if err := g.JSON(f); err != nil { + _ = f.Close() + return err + } + return f.Close() + } + if *jsonOut { + return g.JSON(os.Stdout) + } + fmt.Printf("%d nodes, %d edges, complete=%t\n", len(g.Nodes), len(g.Edges), g.Complete) + for _, w := range g.Warnings { + fmt.Printf("warning: %s\n", w) + } + return nil +} + +func cmdDef(args []string) error { + fs := flag.NewFlagSet("def", flag.ExitOnError) + var c common + c.register(fs) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return fmt.Errorf("def requires a name") + } + g, err := c.load() + if err != nil { + return err + } + nodes := g.Def(fs.Arg(0)) + if len(nodes) == 0 { + return fmt.Errorf("no node matches %q", fs.Arg(0)) + } + ids := make([]string, len(nodes)) + for i := range nodes { + ids[i] = nodes[i].ID + } + return g.Text(os.Stdout, ids, c.budget, nil) +} + +func cmdNeighbours(args []string, callers bool) error { + fs := flag.NewFlagSet("neighbours", flag.ExitOnError) + var c common + c.register(fs) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return fmt.Errorf("requires a name") + } + g, err := c.load() + if err != nil { + return err + } + seed, err := resolveOne(g, fs.Arg(0)) + if err != nil { + return err + } + var edges []outline.Edge + if callers { + edges = g.Callers(seed) + } else { + edges = g.Callees(seed) + } + opts := c.traverse() + ids := []string{seed} + for _, e := range edges { + if !opts.Allow(e) { + continue + } + if callers { + ids = append(ids, e.From) + } else { + ids = append(ids, e.To) + } + } + return g.Text(os.Stdout, ids, c.budget, opts.Allow) +} + +func cmdAffected(args []string) error { + fs := flag.NewFlagSet("affected", flag.ExitOnError) + var c common + c.register(fs) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return fmt.Errorf("affected requires at least one seed") + } + g, err := c.load() + if err != nil { + return err + } + var seeds []string + for _, name := range fs.Args() { + id, err := resolveOne(g, name) + if err != nil { + return err + } + seeds = append(seeds, id) + } + opts := c.traverse() + return writePaths(g, g.Affected(seeds, opts), c.budget, opts.Allow) +} + +func cmdPath(args []string) error { + fs := flag.NewFlagSet("path", flag.ExitOnError) + var c common + c.register(fs) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < pathArgs { + return fmt.Errorf("path requires ") + } + g, err := c.load() + if err != nil { + return err + } + from, err := resolveOne(g, fs.Arg(0)) + if err != nil { + return err + } + to, err := resolveOne(g, fs.Arg(1)) + if err != nil { + return err + } + opts := c.traverse() + p := g.Path(from, to, opts) + if p == nil { + return fmt.Errorf("no path from %s to %s", from, to) + } + return writePaths(g, []outline.Path{p}, c.budget, opts.Allow) +} + +// resolveOne turns a user-supplied seed into exactly one node ID, returning +// an error listing candidates if the name is ambiguous. +func resolveOne(g *outline.Graph, name string) (string, error) { + nodes := g.Def(name) + switch len(nodes) { + case 0: + return "", fmt.Errorf("no node matches %q", name) + case 1: + return nodes[0].ID, nil + } + var b strings.Builder + fmt.Fprintf(&b, "%d nodes match %q; use one of:\n", len(nodes), name) + for _, n := range nodes { + fmt.Fprintf(&b, " %s (%s %s:%d)\n", n.ID, n.Kind, n.File, n.Line) + } + return "", fmt.Errorf("%s", b.String()) +} + +func writePaths(g *outline.Graph, paths []outline.Path, budget int, allow func(outline.Edge) bool) error { + seen := make(map[string]bool) + var ids []string + for _, p := range paths { + for _, e := range p { + if !seen[e.From] { + seen[e.From] = true + ids = append(ids, e.From) + } + if !seen[e.To] { + seen[e.To] = true + ids = append(ids, e.To) + } + } + } + fmt.Printf("%d paths, %d nodes\n", len(paths), len(ids)) + return g.Text(os.Stdout, ids, budget, allow) +} diff --git a/cmd/outline/main_test.go b/cmd/outline/main_test.go new file mode 100644 index 0000000..da0388f --- /dev/null +++ b/cmd/outline/main_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +var binPath string + +func TestMain(m *testing.M) { + os.Exit(runMain(m)) +} + +func runMain(m *testing.M) int { + dir, err := os.MkdirTemp("", "outline-cli-test") + if err != nil { + return 1 + } + defer func() { _ = os.RemoveAll(dir) }() + binPath = filepath.Join(dir, "outline") + if runtime.GOOS == "windows" { + binPath += ".exe" + } + if out, err := exec.Command("go", "build", "-o", binPath, ".").CombinedOutput(); err != nil { + _, _ = os.Stderr.Write(out) + return 1 + } + return m.Run() +} + +func run(t *testing.T, args ...string) string { + t.Helper() + cmd := exec.Command(binPath, args...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("outline %v: %v\n%s", args, err, out) + } + return string(out) +} + +func testdata(rel string) string { + return filepath.Join("..", "..", "testdata", rel) +} + +func TestCLIGoAffected(t *testing.T) { + dir := testdata("cli-go") + gpath := filepath.Join(t.TempDir(), "g.json") + + run(t, "graph", "-o", gpath, dir) + if fi, err := os.Stat(gpath); err != nil || fi.Size() == 0 { + t.Fatalf("graph.json not written: %v", err) + } + + out := run(t, "affected", "-g", gpath, "-inferred", "ext:go:os/exec:Command") + if !strings.Contains(out, "Handler") { + t.Errorf("affected output missing Handler:\n%s", out) + } + if !strings.Contains(out, "main") { + t.Errorf("affected output missing main (transitive caller):\n%s", out) + } + + out = run(t, "path", "-g", gpath, "-inferred", "Handler", "Load") + if !strings.Contains(out, "--calls[inferred]--> Load") { + t.Errorf("path output missing cross-package call:\n%s", out) + } +} + +func TestCLIPythonAffected(t *testing.T) { + dir := testdata("cli-py") + + out := run(t, "affected", "-dir", dir, "-inferred", "ext:python:subprocess:run") + if !strings.Contains(out, "handler") { + t.Errorf("affected output missing handler:\n%s", out) + } + + out = run(t, "def", "-dir", dir, "load") + if !strings.Contains(out, "pkg/loader.py") { + t.Errorf("def output missing loader location:\n%s", out) + } +} + +func TestCLICallersInferred(t *testing.T) { + dir := testdata("cli-go") + out := run(t, "callers", "-dir", dir, "Load") + if strings.Contains(out, "Handler") || strings.Contains(out, "[inferred]") { + t.Errorf("callers without -inferred should not show cross-package caller:\n%s", out) + } + if strings.Contains(out, "--contains") || strings.Contains(out, "--imports") { + t.Errorf("callers output should not include non-call edges:\n%s", out) + } + out = run(t, "callers", "-dir", dir, "-inferred", "Load") + if !strings.Contains(out, "Handler") { + t.Errorf("callers -inferred should show cross-package caller:\n%s", out) + } +} + +func TestCLIUsage(t *testing.T) { + cmd := exec.Command(binPath) + out, err := cmd.CombinedOutput() + if err == nil { + t.Error("expected non-zero exit for missing subcommand") + } + if !strings.Contains(string(out), "outline graph") { + t.Errorf("usage not printed: %s", out) + } +} diff --git a/graph.go b/graph.go new file mode 100644 index 0000000..9f5c03c --- /dev/null +++ b/graph.go @@ -0,0 +1,219 @@ +package outline + +import ( + "encoding/json" + "io" + "sort" + "strconv" + "strings" +) + +// SchemaVersion is bumped whenever Node, Edge, or Graph change in a way a +// reader must understand. +const SchemaVersion = 1 + +// Node kinds. +const ( + KindFile = "file" + KindModule = "module" + KindExternal = "external" + KindFunc = "func" + KindType = "type" + KindClass = "class" + KindConst = "const" + KindVar = "var" +) + +// Edge relations. +const ( + RelContains = "contains" + RelImports = "imports" + RelCalls = "calls" + RelReferences = "references" + RelInherits = "inherits" + RelEmbeds = "embeds" + RelReexports = "reexports" +) + +// Edge confidence. +const ( + ConfExtracted = "extracted" + ConfInferred = "inferred" +) + +// Node is one graph vertex. IDs are opaque and produced only by the id +// encoder helpers below. +type Node struct { + ID string `json:"id"` + Kind string `json:"kind"` + Name string `json:"name"` + Qualified string `json:"qualified,omitempty"` + File string `json:"file,omitempty"` + Line int `json:"line,omitempty"` + Start int `json:"start,omitempty"` + End int `json:"end,omitempty"` + Exported bool `json:"exported,omitempty"` + Sig string `json:"sig,omitempty"` +} + +// Edge is one directed relationship between two nodes. +type Edge struct { + From string `json:"from"` + To string `json:"to"` + Rel string `json:"rel"` + Conf string `json:"conf"` + File string `json:"file,omitempty"` + Line int `json:"line,omitempty"` +} + +// Graph is a whole-repository code graph. +type Graph struct { + SchemaVersion int `json:"schema_version"` + ToolVersion string `json:"tool_version"` + SourceDigest string `json:"source_digest"` + Complete bool `json:"complete"` + Warnings []string `json:"warnings,omitempty"` + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` + + byID map[string]int + fwd map[string][]int + rev map[string][]int + built bool +} + +const ( + escColon = "%3A" + escPercent = "%25" +) + +// idEscape encodes one path or name segment so that ':' remains a safe +// separator in composite IDs. Only ':' and '%' are escaped. +func idEscape(s string) string { + if strings.IndexByte(s, ':') < 0 && strings.IndexByte(s, '%') < 0 { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + switch s[i] { + case ':': + b.WriteString(escColon) + case '%': + b.WriteString(escPercent) + default: + b.WriteByte(s[i]) + } + } + return b.String() +} + +// idUnescape reverses idEscape. +func idUnescape(s string) string { + if strings.IndexByte(s, '%') < 0 { + return s + } + s = strings.ReplaceAll(s, escColon, ":") + return strings.ReplaceAll(s, escPercent, "%") +} + +// FileID returns the node ID for a repository-relative file path. +func FileID(path string) string { + return "file:" + idEscape(path) +} + +// SymID returns the node ID for a declaration at a byte offset in a file. +func SymID(path string, start uint32) string { + return "sym:" + idEscape(path) + ":" + strconv.FormatUint(uint64(start), 10) +} + +// ModID returns the node ID for a module in a language. +func ModID(lang, module string) string { + return "mod:" + lang + ":" + idEscape(module) +} + +// ExtID returns the node ID for an unresolved external target. +func ExtID(lang, module, name string) string { + return "ext:" + lang + ":" + idEscape(module) + ":" + idEscape(name) +} + +// index rebuilds the adjacency maps. It is called lazily on first query. +func (g *Graph) index() { + if g.built { + return + } + g.byID = make(map[string]int, len(g.Nodes)) + for i := range g.Nodes { + g.byID[g.Nodes[i].ID] = i + } + g.fwd = make(map[string][]int, len(g.Nodes)) + g.rev = make(map[string][]int, len(g.Nodes)) + for i := range g.Edges { + g.fwd[g.Edges[i].From] = append(g.fwd[g.Edges[i].From], i) + g.rev[g.Edges[i].To] = append(g.rev[g.Edges[i].To], i) + } + g.built = true +} + +// Node returns the node with the given ID, or nil. +func (g *Graph) Node(id string) *Node { + g.index() + if i, ok := g.byID[id]; ok { + return &g.Nodes[i] + } + return nil +} + +// Out returns edges leaving id. +func (g *Graph) Out(id string) []Edge { + g.index() + return g.edgeSlice(g.fwd[id]) +} + +// In returns edges arriving at id. +func (g *Graph) In(id string) []Edge { + g.index() + return g.edgeSlice(g.rev[id]) +} + +func (g *Graph) edgeSlice(idx []int) []Edge { + if len(idx) == 0 { + return nil + } + out := make([]Edge, len(idx)) + for i, e := range idx { + out[i] = g.Edges[e] + } + return out +} + +// sortStable orders Nodes by ID and Edges by (From, To, Rel, File, Line) so +// two builds of the same input produce byte-identical JSON. +func (g *Graph) sortStable() { + sort.Slice(g.Nodes, func(i, j int) bool { return g.Nodes[i].ID < g.Nodes[j].ID }) + sort.Slice(g.Edges, func(i, j int) bool { + a, b := g.Edges[i], g.Edges[j] + if a.From != b.From { + return a.From < b.From + } + if a.To != b.To { + return a.To < b.To + } + if a.Rel != b.Rel { + return a.Rel < b.Rel + } + if a.File != b.File { + return a.File < b.File + } + return a.Line < b.Line + }) + g.built = false +} + +// JSON writes the graph as sorted, indented JSON. +func (g *Graph) JSON(w io.Writer) error { + g.sortStable() + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.SetEscapeHTML(false) + return enc.Encode(g) +} diff --git a/graph_test.go b/graph_test.go new file mode 100644 index 0000000..eb499c0 --- /dev/null +++ b/graph_test.go @@ -0,0 +1,96 @@ +package outline + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestIDEscape(t *testing.T) { + cases := map[string]string{ + "plain": "plain", + "a:b": "a%3Ab", + "100%": "100%25", + "a:b%c": "a%3Ab%25c", + "src/x.go": "src/x.go", + } + for in, want := range cases { + if got := idEscape(in); got != want { + t.Errorf("idEscape(%q) = %q, want %q", in, got, want) + } + if got := idUnescape(idEscape(in)); got != in { + t.Errorf("idUnescape(idEscape(%q)) = %q", in, got) + } + } + if got := modName(ModID("go", "a:b/c")); got != "a:b/c" { + t.Errorf("modName round-trip = %q", got) + } + if a, b := SymID("a:b.go", 10), SymID("a", 10); a == b { + t.Errorf("distinct paths collided: %q == %q", a, b) + } +} + +func TestGraphAdjacency(t *testing.T) { + g := &Graph{ + Nodes: []Node{ + {ID: "file:a.go", Kind: KindFile}, + {ID: "sym:a.go:0", Kind: KindFunc, Name: "A"}, + {ID: "sym:b.go:0", Kind: KindFunc, Name: "B"}, + }, + Edges: []Edge{ + {From: "file:a.go", To: "sym:a.go:0", Rel: RelContains, Conf: ConfExtracted}, + {From: "sym:a.go:0", To: "sym:b.go:0", Rel: RelCalls, Conf: ConfInferred}, + }, + } + if n := g.Node("sym:a.go:0"); n == nil || n.Name != "A" { + t.Fatalf("Node lookup failed: %v", n) + } + out := g.Out("sym:a.go:0") + if len(out) != 1 || out[0].To != "sym:b.go:0" || out[0].Rel != RelCalls { + t.Errorf("Out = %v", out) + } + in := g.In("sym:a.go:0") + if len(in) != 1 || in[0].From != "file:a.go" || in[0].Rel != RelContains { + t.Errorf("In = %v", in) + } + if g.Node("missing") != nil { + t.Error("Node(missing) should be nil") + } +} + +func TestGraphJSONDeterministic(t *testing.T) { + mk := func() *Graph { + return &Graph{ + SchemaVersion: SchemaVersion, + Complete: true, + Nodes: []Node{ + {ID: "sym:b.go:0", Kind: KindFunc, Name: "B"}, + {ID: "file:a.go", Kind: KindFile}, + }, + Edges: []Edge{ + {From: "sym:b.go:0", To: "file:a.go", Rel: RelReferences}, + {From: "file:a.go", To: "sym:b.go:0", Rel: RelContains}, + }, + } + } + var a, b bytes.Buffer + if err := mk().JSON(&a); err != nil { + t.Fatal(err) + } + if err := mk().JSON(&b); err != nil { + t.Fatal(err) + } + if !bytes.Equal(a.Bytes(), b.Bytes()) { + t.Errorf("non-deterministic JSON:\n%s\n---\n%s", a.String(), b.String()) + } + var round Graph + if err := json.Unmarshal(a.Bytes(), &round); err != nil { + t.Fatal(err) + } + if round.Nodes[0].ID != "file:a.go" { + t.Errorf("nodes not sorted: first = %q", round.Nodes[0].ID) + } + if round.Edges[0].From != "file:a.go" { + t.Errorf("edges not sorted: first from = %q", round.Edges[0].From) + } +} diff --git a/import.go b/import.go index 74afdba..a63f06d 100644 --- a/import.go +++ b/import.go @@ -19,46 +19,49 @@ func Imports(src []byte, filename string) ([]Import, bool) { return nil, false } defer tree.Release() + return importsFor(src, l, tree.RootNode()) +} +func importsFor(src []byte, l *lang, root *ts.Node) ([]Import, bool) { switch l.name { case "go": - return goImports(src, l.language, tree.RootNode()), true + return goImports(src, l.language, root), true case "ruby": - return rubyImports(src, l.language, tree.RootNode()), true + return rubyImports(src, l.language, root), true case "python": - return pythonImports(src, l.language, tree.RootNode()), true + return pythonImports(src, l.language, root), true case "javascript", "typescript": - return javascriptImports(src, l.language, tree.RootNode()), true + return javascriptImports(src, l.language, root), true case "rust": - return rustImports(src, l.language, tree.RootNode()), true + return rustImports(src, l.language, root), true case "php": - return phpImports(src, l.language, tree.RootNode()), true + return phpImports(src, l.language, root), true case "elixir": - return elixirImports(src, l.language, tree.RootNode()), true + return elixirImports(src, l.language, root), true case "dart": - return dartImports(src, l.language, tree.RootNode()), true + return dartImports(src, l.language, root), true case "swift": - return swiftImports(src, l.language, tree.RootNode()), true + return swiftImports(src, l.language, root), true case "haskell": - return haskellImports(src, l.language, tree.RootNode()), true + return haskellImports(src, l.language, root), true case "perl": - return perlImports(src, l.language, tree.RootNode()), true + return perlImports(src, l.language, root), true case "lua": - return luaImports(src, l.language, tree.RootNode()), true + return luaImports(src, l.language, root), true case "r": - return rImports(src, l.language, tree.RootNode()), true + return rImports(src, l.language, root), true case "julia": - return juliaImports(src, l.language, tree.RootNode()), true + return juliaImports(src, l.language, root), true case "ocaml": - return ocamlImports(src, l.language, tree.RootNode()), true + return ocamlImports(src, l.language, root), true case "crystal": - return crystalImports(src, l.language, tree.RootNode()), true + return crystalImports(src, l.language, root), true case "nim": - return nimImports(src, l.language, tree.RootNode()), true + return nimImports(src, l.language, root), true case "zig": - return zigImports(src, l.language, tree.RootNode()), true + return zigImports(src, l.language, root), true case "d": - return dImports(src, l.language, tree.RootNode()), true + return dImports(src, l.language, root), true default: return nil, false } diff --git a/pack.go b/pack.go index a50bda4..7c534c2 100644 --- a/pack.go +++ b/pack.go @@ -33,6 +33,9 @@ type Options struct { // Concurrency is the number of files processed in parallel. // Zero means runtime.NumCPU(). Concurrency int + // Resolution supplies caller-provided source roots and package hints + // for graph module resolution. Pack ignores it. + Resolution ResolutionHints } // File is one packed file. diff --git a/query.go b/query.go new file mode 100644 index 0000000..7595b26 --- /dev/null +++ b/query.go @@ -0,0 +1,288 @@ +package outline + +import ( + "fmt" + "io" + "slices" + "sort" +) + +// TraverseOptions filters graph traversal. +type TraverseOptions struct { + // Depth caps hop distance. Zero means unbounded. + Depth int + // Relations restricts which edge relations are followed. Nil means + // RelCalls only. + Relations []string + // IncludeInferred follows inferred-confidence edges as well as + // extracted ones. Extracted edges are always followed. + IncludeInferred bool +} + +// Path is an ordered chain of edges from a seed to a reached node. +type Path []Edge + +// Allow reports whether e passes the relation and confidence filter. +func (o TraverseOptions) Allow(e Edge) bool { + if !o.IncludeInferred && e.Conf != ConfExtracted { + return false + } + if o.Relations == nil { + return e.Rel == RelCalls + } + return slices.Contains(o.Relations, e.Rel) +} + +// Def returns nodes whose Name or Qualified exactly matches name, or whose +// ID is name. +func (g *Graph) Def(name string) []Node { + g.index() + if i, ok := g.byID[name]; ok { + return []Node{g.Nodes[i]} + } + var out []Node + for i := range g.Nodes { + if g.Nodes[i].Name == name || g.Nodes[i].Qualified == name { + out = append(out, g.Nodes[i]) + } + } + return out +} + +// Callers returns inbound calls edges to id. +func (g *Graph) Callers(id string) []Edge { + return filterRel(g.In(id), RelCalls) +} + +// Callees returns outbound calls edges from id. +func (g *Graph) Callees(id string) []Edge { + return filterRel(g.Out(id), RelCalls) +} + +func filterRel(edges []Edge, rel string) []Edge { + var out []Edge + for _, e := range edges { + if e.Rel == rel { + out = append(out, e) + } + } + return out +} + +// Affected walks the graph in reverse from each seed and returns one +// evidence path per reached node. The default relation set is RelCalls, +// so with default options this is the transitive caller set. +func (g *Graph) Affected(seeds []string, opts TraverseOptions) []Path { + g.index() + prev := make(map[string]int) + dist := make(map[string]int) + var frontier []string + for _, s := range seeds { + if _, ok := g.byID[s]; ok { + prev[s] = -1 + dist[s] = 0 + frontier = append(frontier, s) + } + } + for len(frontier) > 0 { + id := frontier[0] + frontier = frontier[1:] + if opts.Depth > 0 && dist[id] >= opts.Depth { + continue + } + for _, ei := range g.rev[id] { + e := g.Edges[ei] + if !opts.Allow(e) { + continue + } + if _, seen := prev[e.From]; seen { + continue + } + prev[e.From] = ei + dist[e.From] = dist[id] + 1 + frontier = append(frontier, e.From) + } + } + reached := make([]string, 0, len(prev)) + for id, ei := range prev { + if ei >= 0 { + reached = append(reached, id) + } + } + sort.Slice(reached, func(i, j int) bool { + if dist[reached[i]] != dist[reached[j]] { + return dist[reached[i]] < dist[reached[j]] + } + return reached[i] < reached[j] + }) + paths := make([]Path, len(reached)) + for i, id := range reached { + for cur := id; prev[cur] >= 0; cur = g.Edges[prev[cur]].To { + paths[i] = append(paths[i], g.Edges[prev[cur]]) + } + } + return paths +} + +// Path returns the shortest forward edge chain from from to to under opts, +// or nil if none exists. +func (g *Graph) Path(from, to string, opts TraverseOptions) Path { + g.index() + if _, ok := g.byID[from]; !ok { + return nil + } + prev := map[string]int{from: -1} + dist := map[string]int{from: 0} + frontier := []string{from} + for len(frontier) > 0 { + id := frontier[0] + frontier = frontier[1:] + if id == to { + var p Path + for cur := to; prev[cur] >= 0; cur = g.Edges[prev[cur]].From { + p = append(Path{g.Edges[prev[cur]]}, p...) + } + return p + } + if opts.Depth > 0 && dist[id] >= opts.Depth { + continue + } + for _, ei := range g.fwd[id] { + e := g.Edges[ei] + if !opts.Allow(e) { + continue + } + if _, seen := prev[e.To]; seen { + continue + } + prev[e.To] = ei + dist[e.To] = dist[id] + 1 + frontier = append(frontier, e.To) + } + } + return nil +} + +const bytesPerToken = 3 + +type textWriter struct { + w io.Writer + limit int + written int + err error + trunc bool +} + +func (t *textWriter) line(s string) { + if t.err != nil || t.trunc { + return + } + if t.limit > 0 && t.written+len(s)+1 > t.limit { + _, t.err = fmt.Fprintln(t.w, "... truncated at budget") + t.trunc = true + return + } + _, t.err = fmt.Fprintln(t.w, s) + t.written += len(s) + 1 +} + +func (t *textWriter) done() bool { return t.err != nil || t.trunc } + +func (g *Graph) display(id string) string { + if n := g.Node(id); n != nil { + return nodeLabel(n) + } + return id +} + +func nodeLabel(n *Node) string { + if n.Qualified != "" { + return n.Qualified + } + if n.Name != "" { + return n.Name + } + return n.ID +} + +func location(file string, line int) string { + if file == "" { + return "" + } + if line > 0 { + return fmt.Sprintf("%s:%d", sanitise(file), line) + } + return sanitise(file) +} + +func nodeLine(n *Node) string { + s := fmt.Sprintf("NODE %s %s", sanitise(nodeLabel(n)), sanitise(n.Kind)) + if loc := location(n.File, n.Line); loc != "" { + s += " " + loc + } + return fmt.Sprintf("%s exported=%t sig=%s", s, n.Exported, sanitise(n.Sig)) +} + +func (g *Graph) edgeLine(e Edge) string { + s := fmt.Sprintf("EDGE %s --%s[%s]--> %s", + sanitise(g.display(e.From)), sanitise(e.Rel), sanitise(e.Conf), sanitise(g.display(e.To))) + if loc := location(e.File, e.Line); loc != "" { + s += " at " + loc + } + return s +} + +// Text writes a budgeted line-oriented rendering of the seeds and their +// BFS neighbourhood, following only edges that pass allow (nil admits +// every edge). Every line is sanitised so untrusted source cannot inject +// terminal escapes or break the line protocol. +func (g *Graph) Text(w io.Writer, ids []string, budget int, allow func(Edge) bool) error { + g.index() + tw := &textWriter{w: w, limit: budget * bytesPerToken} + seen := make(map[string]bool) + edgeSeen := make(map[int]bool) + + frontier := make([]string, 0, len(ids)) + for _, id := range ids { + n := g.Node(id) + if n == nil || seen[id] { + continue + } + seen[id] = true + frontier = append(frontier, id) + tw.line(nodeLine(n)) + } + for len(frontier) > 0 && !tw.done() { + id := frontier[0] + frontier = frontier[1:] + frontier = g.textEdges(tw, id, allow, seen, edgeSeen, frontier) + } + return tw.err +} + +func (g *Graph) textEdges(tw *textWriter, id string, allow func(Edge) bool, seen map[string]bool, edgeSeen map[int]bool, frontier []string) []string { + for _, adj := range [][]int{g.fwd[id], g.rev[id]} { + for _, ei := range adj { + if edgeSeen[ei] || tw.done() { + continue + } + edgeSeen[ei] = true + e := g.Edges[ei] + if allow != nil && !allow(e) { + continue + } + tw.line(g.edgeLine(e)) + for _, next := range []string{e.From, e.To} { + if seen[next] { + continue + } + seen[next] = true + if n := g.Node(next); n != nil { + tw.line(nodeLine(n)) + } + frontier = append(frontier, next) + } + } + } + return frontier +} diff --git a/query_test.go b/query_test.go new file mode 100644 index 0000000..4f516c2 --- /dev/null +++ b/query_test.go @@ -0,0 +1,183 @@ +package outline + +import ( + "bytes" + "reflect" + "strings" + "testing" +) + +// chain builds a linear call graph a -> b -> c -> d. +func chain() *Graph { + g := &Graph{SchemaVersion: SchemaVersion} + names := []string{"a", "b", "c", "d"} + for i, n := range names { + g.Nodes = append(g.Nodes, Node{ + ID: "sym:x.go:" + n, Kind: KindFunc, Name: n, Qualified: n, + File: "x.go", Line: i + 1, + }) + } + for i := 0; i+1 < len(names); i++ { + conf := ConfExtracted + if i == 1 { + conf = ConfInferred + } + g.Edges = append(g.Edges, Edge{ + From: "sym:x.go:" + names[i], To: "sym:x.go:" + names[i+1], + Rel: RelCalls, Conf: conf, File: "x.go", Line: i + 1, + }) + } + return g +} + +func TestDef(t *testing.T) { + g := chain() + if got := g.Def("b"); len(got) != 1 || got[0].ID != "sym:x.go:b" { + t.Errorf("Def(b) = %v", got) + } + if got := g.Def("sym:x.go:c"); len(got) != 1 || got[0].Name != "c" { + t.Errorf("Def by ID = %v", got) + } + if got := g.Def("nope"); got != nil { + t.Errorf("Def(nope) = %v", got) + } +} + +func TestCallersCallees(t *testing.T) { + g := chain() + if got := g.Callers("sym:x.go:b"); len(got) != 1 || got[0].From != "sym:x.go:a" { + t.Errorf("Callers(b) = %v", got) + } + if got := g.Callees("sym:x.go:b"); len(got) != 1 || got[0].To != "sym:x.go:c" { + t.Errorf("Callees(b) = %v", got) + } +} + +func TestAffected(t *testing.T) { + g := chain() + paths := g.Affected([]string{"sym:x.go:d"}, TraverseOptions{IncludeInferred: true}) + got := make(map[string]int) + for _, p := range paths { + got[p[0].From] = len(p) + } + if got["sym:x.go:c"] != 1 || got["sym:x.go:b"] != 2 || got["sym:x.go:a"] != 3 { + t.Errorf("Affected paths = %v", got) + } + + paths = g.Affected([]string{"sym:x.go:d"}, TraverseOptions{IncludeInferred: false}) + if len(paths) != 1 || paths[0][0].From != "sym:x.go:c" { + t.Errorf("Affected without inferred should stop at c: %v", paths) + } + + paths = g.Affected([]string{"sym:x.go:d"}, TraverseOptions{Depth: 1, IncludeInferred: true}) + if len(paths) != 1 { + t.Errorf("Affected depth=1 should return 1 path: %v", paths) + } + + a := chain().Affected([]string{"sym:x.go:d"}, TraverseOptions{IncludeInferred: true}) + b := chain().Affected([]string{"sym:x.go:d"}, TraverseOptions{IncludeInferred: true}) + if !reflect.DeepEqual(a, b) { + t.Errorf("Affected order not deterministic:\n%v\n%v", a, b) + } + if len(a[0]) > len(a[len(a)-1]) { + t.Errorf("Affected not sorted by distance: %v", a) + } +} + +func TestPath(t *testing.T) { + g := chain() + p := g.Path("sym:x.go:a", "sym:x.go:d", TraverseOptions{IncludeInferred: true}) + if len(p) != 3 || p[0].From != "sym:x.go:a" || p[2].To != "sym:x.go:d" { + t.Errorf("Path a->d = %v", p) + } + if p := g.Path("sym:x.go:a", "sym:x.go:d", TraverseOptions{}); p != nil { + t.Errorf("Path without inferred should not reach d: %v", p) + } + if p := g.Path("sym:x.go:d", "sym:x.go:a", TraverseOptions{IncludeInferred: true}); p != nil { + t.Errorf("Path is directed; d->a should be nil: %v", p) + } +} + +func TestText(t *testing.T) { + g := chain() + var buf bytes.Buffer + if err := g.Text(&buf, []string{"sym:x.go:b"}, 0, nil); err != nil { + t.Fatal(err) + } + out := buf.String() + if !strings.Contains(out, "NODE b func x.go:2") { + t.Errorf("missing seed node line: %q", out) + } + if !strings.Contains(out, "EDGE a --calls[extracted]--> b") { + t.Errorf("missing inbound edge: %q", out) + } + if !strings.Contains(out, "EDGE b --calls[inferred]--> c") { + t.Errorf("missing outbound edge: %q", out) + } + + buf.Reset() + if err := g.Text(&buf, []string{"sym:x.go:a"}, 10, nil); err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "truncated") { + t.Errorf("small budget should truncate: %q", buf.String()) + } + + buf.Reset() + allow := TraverseOptions{}.Allow + if err := g.Text(&buf, []string{"sym:x.go:b"}, 0, allow); err != nil { + t.Fatal(err) + } + out = buf.String() + if strings.Contains(out, "[inferred]") { + t.Errorf("filtered Text should not print inferred edges: %q", out) + } + if !strings.Contains(out, "EDGE a --calls[extracted]--> b") { + t.Errorf("filtered Text should still print extracted edges: %q", out) + } +} + +func TestTextRendering(t *testing.T) { + g := &Graph{ + Nodes: []Node{ + {ID: "file:x.go", Kind: KindFile, Name: "x.go", File: "x.go"}, + {ID: "mod:go:m", Kind: KindModule, Name: "m"}, + }, + Edges: []Edge{ + {From: "mod:go:m", To: "file:x.go", Rel: RelContains, Conf: ConfInferred}, + }, + } + var buf bytes.Buffer + if err := g.Text(&buf, []string{"file:x.go"}, 0, nil); err != nil { + t.Fatal(err) + } + out := buf.String() + if !strings.Contains(out, "NODE x.go file x.go") { + t.Errorf("file node should fall back to Name label: %q", out) + } + if !strings.Contains(out, "NODE m module exported=") { + t.Errorf("module node should omit empty location: %q", out) + } + if strings.Contains(out, "at :0") || strings.Contains(out, " at \n") { + t.Errorf("edge without location should omit at-clause: %q", out) + } + if strings.Contains(out, "NODE ") { + t.Errorf("blank node label: %q", out) + } + + evil := &Node{ID: "x", Kind: "func\ninjected", Name: "n", File: "a\x1b[31m.go", Line: 1} + line := nodeLine(evil) + if strings.Contains(line, "\n") || strings.Contains(line, "\x1b") { + t.Errorf("nodeLine leaked control bytes: %q", line) + } + e := Edge{From: "a", To: "b", Rel: "calls\n", Conf: "x\x1b", File: "f\n.go", Line: 2} + if el := g.edgeLine(e); strings.Contains(el, "\n") || strings.Contains(el, "\x1b") { + t.Errorf("edgeLine leaked control bytes: %q", el) + } +} + +func TestSanitise(t *testing.T) { + if got := sanitise("a\n b\t\x1b[31mc"); got != "a b [31mc" { + t.Errorf("sanitise = %q", got) + } +} diff --git a/resolve.go b/resolve.go new file mode 100644 index 0000000..411f058 --- /dev/null +++ b/resolve.go @@ -0,0 +1,430 @@ +package outline + +import ( + "maps" + "os" + "path" + "path/filepath" + "slices" + "strings" +) + +// resolver joins per-file analyses into cross-file graph edges. +type resolver struct { + root string + hints ResolutionHints + paths []string + files map[string]*fileAnalysis + + goModule string + goPkgs map[string]map[string]string + pyRoots []string + + // modules maps a mod: ID to the repository-relative files that + // implement it, once resolved. + modules map[string][]string + // exports maps a mod: ID to that module's top-level name → sym ID. + exports map[string]map[string]string +} + +// scope is a file's local name table. +type scope struct { + // syms maps a bare top-level name to a repository sym or ext ID. + syms map[string]string + // mods maps a local module alias to its mod ID. + mods map[string]string + // children maps a decl index (or -1 for top level) to its direct + // child decl indices, for lexical resolution of nested calls. + children map[int][]int +} + +func newResolver(root string, files []fileAnalysis, hints ResolutionHints) *resolver { + r := &resolver{ + root: root, + hints: hints, + files: make(map[string]*fileAnalysis, len(files)), + modules: make(map[string][]string), + exports: make(map[string]map[string]string), + } + for i := range files { + r.paths = append(r.paths, files[i].path) + r.files[files[i].path] = &files[i] + } + r.goModule = readGoModule(root) + r.goPkgs = indexGoPackages(files) + r.pyRoots = append([]string{""}, hints.SourceRoots...) + return r +} + +// indexGoPackages groups top-level declarations from every Go file by +// directory so same-package calls resolve without an import edge. +func indexGoPackages(files []fileAnalysis) map[string]map[string]string { + pkgs := make(map[string]map[string]string) + for i := range files { + f := &files[i] + if f.a == nil || f.a.Lang != "go" { + continue + } + dir := path.Dir(f.path) + if pkgs[dir] == nil { + pkgs[dir] = make(map[string]string) + } + for _, d := range f.a.Decls { + if d.Parent == -1 { + pkgs[dir][d.Name] = d.symID(f.path) + } + } + } + return pkgs +} + +func readGoModule(root string) string { + data, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + return "" + } + for line := range strings.SplitSeq(string(data), "\n") { + line = strings.TrimSpace(line) + if after, ok := strings.CutPrefix(line, "module "); ok { + return strings.TrimSpace(after) + } + } + return "" +} + +// moduleID returns the mod: node ID for one import. Python relative +// imports are qualified by the importing file's directory so two +// `from .util import x` in different packages produce distinct nodes. +func (r *resolver) moduleID(f *fileAnalysis, imp Import) string { + name := imp.Module + if f.a.Lang == "python" && strings.HasPrefix(name, ".") { + name = path.Dir(f.path) + "/" + name + } + return ModID(f.a.Lang, name) +} + +// emitModules creates one mod: node per distinct import, emits file→mod +// imports edges, and resolves each module to repository files where the +// language resolver supports it. +func (r *resolver) emitModules(g *Graph) { + seen := make(map[string]bool) + for _, p := range r.paths { + f := r.files[p] + if f.a == nil { + continue + } + fid := FileID(p) + for _, imp := range f.a.Imports { + mid := r.moduleID(f, imp) + if !seen[mid] { + seen[mid] = true + g.Nodes = append(g.Nodes, Node{ + ID: mid, Kind: KindModule, Name: imp.Module, + }) + r.modules[mid] = r.moduleFiles(f.a.Lang, imp.Module, p) + } + g.Edges = append(g.Edges, Edge{ + From: fid, To: mid, Rel: RelImports, Conf: ConfExtracted, + File: p, Line: imp.Line, + }) + } + } + for mid, paths := range r.modules { + for _, p := range paths { + g.Edges = append(g.Edges, Edge{ + From: mid, To: FileID(p), Rel: RelContains, Conf: ConfInferred, + }) + } + } +} + +// moduleFiles resolves an import module string to zero or more +// repository-relative file paths already in r.files. +func (r *resolver) moduleFiles(lang, module, from string) []string { + switch lang { + case "go": + return r.goModuleFiles(module) + case "python": + return r.pyModuleFiles(module, from) + } + return nil +} + +func (r *resolver) goModuleFiles(module string) []string { + if r.goModule == "" { + return nil + } + rel, ok := strings.CutPrefix(module, r.goModule) + if !ok { + return nil + } + if rel != "" && rel[0] != '/' { + return nil + } + dir := strings.TrimPrefix(rel, "/") + var files []string + for _, p := range r.paths { + f := r.files[p] + if f.a == nil || f.a.Lang != "go" { + continue + } + if path.Dir(p) == dir || (dir == "" && path.Dir(p) == ".") { + files = append(files, p) + } + } + return files +} + +func (r *resolver) pyModuleFiles(module, from string) []string { + if strings.HasPrefix(module, ".") { + return r.pyRelative(module, from) + } + rel := strings.ReplaceAll(module, ".", "/") + for _, root := range r.pyRoots { + for _, cand := range []string{ + path.Join(root, rel+".py"), + path.Join(root, rel, "__init__.py"), + } { + if _, ok := r.files[cand]; ok { + return []string{cand} + } + } + } + return nil +} + +func (r *resolver) pyRelative(module, from string) []string { + dots := 0 + for dots < len(module) && module[dots] == '.' { + dots++ + } + dir := path.Dir(from) + for i := 1; i < dots; i++ { + dir = path.Dir(dir) + } + rest := strings.ReplaceAll(module[dots:], ".", "/") + base := dir + if rest != "" { + base = path.Join(dir, rest) + } + for _, cand := range []string{base + ".py", path.Join(base, "__init__.py")} { + if _, ok := r.files[cand]; ok { + return []string{cand} + } + } + return nil +} + +// moduleExports returns the top-level exported name → sym ID map for a +// resolved module, computing and caching it on first request. +func (r *resolver) moduleExports(mid string) map[string]string { + if m, ok := r.exports[mid]; ok { + return m + } + m := make(map[string]string) + for _, p := range r.modules[mid] { + f := r.files[p] + if f == nil || f.a == nil { + continue + } + for _, d := range f.a.Decls { + if d.Parent != -1 || !d.Exported { + continue + } + m[d.Name] = d.symID(p) + } + } + r.exports[mid] = m + return m +} + +// fileScope builds the local name table for one analysed file. +func (r *resolver) fileScope(f *fileAnalysis) scope { + sc := scope{ + syms: make(map[string]string), + mods: make(map[string]string), + children: make(map[int][]int), + } + if f.a.Lang == "go" { + maps.Copy(sc.syms, r.goPkgs[path.Dir(f.path)]) + } + for i, d := range f.a.Decls { + sc.children[d.Parent] = append(sc.children[d.Parent], i) + if d.Parent == -1 { + sc.syms[d.Name] = d.symID(f.path) + } + } + for _, imp := range f.a.Imports { + mid := r.moduleID(f, imp) + switch imp.Kind { + case ImportModule, ImportNamespace, ImportDefault: + alias := "" + if len(imp.Names) > 0 { + alias = imp.Names[0].Alias + } + if alias == "" { + alias = defaultAlias(f.a.Lang, imp.Module) + } + if alias != "" { + sc.mods[alias] = mid + } + case ImportNamed: + exp := r.moduleExports(mid) + for _, n := range imp.Names { + local := n.Alias + if local == "" { + local = n.Name + } + if sid, ok := exp[n.Name]; ok { + sc.syms[local] = sid + } else { + sc.syms[local] = ExtID(f.a.Lang, modName(mid), n.Name) + } + } + case ImportWildcard: + maps.Copy(sc.syms, r.moduleExports(mid)) + } + } + return sc +} + +func defaultAlias(lang, module string) string { + switch lang { + case "go": + seg := module + if i := strings.LastIndexByte(seg, '/'); i >= 0 && goVersionSuffix(seg[i+1:]) { + seg = seg[:i] + } + if i := strings.LastIndexByte(seg, '/'); i >= 0 { + return seg[i+1:] + } + return seg + case "python": + if i := strings.IndexByte(module, '.'); i > 0 { + return module[:i] + } + return module + } + return module +} + +// emitCalls resolves each call site to a target node and emits a calls +// edge. Unresolved targets get an ext: node. +func (r *resolver) emitCalls(g *Graph) { + ext := make(map[string]bool) + for _, p := range r.paths { + f := r.files[p] + if f.a == nil { + continue + } + sc := r.fileScope(f) + for _, c := range f.a.Calls { + from := FileID(p) + if c.In >= 0 { + from = f.a.Decls[c.In].symID(p) + } + to, conf := r.resolveCall(f, sc, c) + if strings.HasPrefix(to, "ext:") && !ext[to] { + ext[to] = true + g.Nodes = append(g.Nodes, Node{ + ID: to, Kind: KindExternal, Name: c.Name, + Qualified: extQualified(to), + }) + } + g.Edges = append(g.Edges, Edge{ + From: from, To: to, Rel: RelCalls, Conf: conf, + File: p, Line: c.Line, + }) + } + } +} + +func (r *resolver) resolveCall(f *fileAnalysis, sc scope, c Call) (string, string) { + if c.Receiver == "" { + if sid, shadowed := lexical(f, sc, c.In, c.Name); sid != "" { + return sid, ConfExtracted + } else if shadowed { + return ExtID(f.a.Lang, "", c.Name), ConfInferred + } + if sid, ok := sc.syms[c.Name]; ok { + return sid, ConfInferred + } + return ExtID(f.a.Lang, "", c.Name), ConfInferred + } + if _, shadowed := lexical(f, sc, c.In, c.Receiver); shadowed { + return ExtID(f.a.Lang, c.Receiver, c.Name), ConfInferred + } + if mid, ok := sc.mods[c.Receiver]; ok { + if sid, ok := r.moduleExports(mid)[c.Name]; ok { + return sid, ConfInferred + } + return ExtID(f.a.Lang, modName(mid), c.Name), ConfInferred + } + return ExtID(f.a.Lang, c.Receiver, c.Name), ConfInferred +} + +// lexical walks outward from the enclosing declaration, returning the +// innermost visible decl matching name. If a parameter of an enclosing +// function matches first, it reports shadowed instead. Class bodies do +// not contribute their members to enclosed functions' scopes. +func lexical(f *fileAnalysis, sc scope, in int, name string) (id string, shadowed bool) { + decls := f.a.Decls + at := in + for { + if at < 0 || at == in || decls[at].Kind != KindClass { + for _, ci := range sc.children[at] { + if decls[ci].Name == name { + return decls[ci].symID(f.path), false + } + } + } + if at < 0 { + return "", false + } + if slices.Contains(decls[at].Params, name) { + return "", true + } + at = decls[at].Parent + } +} + +func goVersionSuffix(s string) bool { + if len(s) < 2 || s[0] != 'v' { + return false + } + for i := 1; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return s != "v0" && s != "v1" +} + +// extQualified derives a stable display name from an ext: node ID so the +// same external target reached via different local aliases renders the +// same way. +func extQualified(id string) string { + rest, ok := strings.CutPrefix(id, "ext:") + if !ok { + return id + } + if i := strings.IndexByte(rest, ':'); i >= 0 { + rest = rest[i+1:] + } + mod, name, ok := strings.Cut(rest, ":") + if !ok { + return idUnescape(rest) + } + mod, name = idUnescape(mod), idUnescape(name) + if mod == "" { + return name + } + return mod + "." + name +} + +func modName(mid string) string { + if i := strings.LastIndexByte(mid, ':'); i >= 0 { + return idUnescape(mid[i+1:]) + } + return mid +} diff --git a/symbol.go b/symbol.go index 9a85256..f23e791 100644 --- a/symbol.go +++ b/symbol.go @@ -78,18 +78,25 @@ func symbolsFromMatch(src []byte, l *lang, match ts.QueryMatch) []parsedSymbol { return parsed } -func applyExplicitExports(parsed []parsedSymbol, l *lang, root *ts.Node, src []byte) { - var exported map[string]bool - merge := false +// explicitExports returns the set of names a file's export list marks as +// public, for languages that declare exports separately from definitions. +// merge is true when the list augments per-definition markers rather than +// replacing them. +func explicitExports(l *lang, root *ts.Node, src []byte) (names map[string]bool, merge, ok bool) { switch l.name { case "javascript", "typescript": - exported = javascriptExports(root, src, l.language) - merge = true + return javascriptExports(root, src, l.language), true, true case "erlang": - exported = exportsFromNodes(root, src, l.language, "export_attribute", "export_type_attribute", "atom") + return exportsFromNodes(root, src, l.language, "export_attribute", "export_type_attribute", "atom"), false, true case "julia": - exported = exportsFromNodes(root, src, l.language, "export_statement", "public_statement", "identifier") - default: + return exportsFromNodes(root, src, l.language, "export_statement", "public_statement", "identifier"), false, true + } + return nil, false, false +} + +func applyExplicitExports(parsed []parsedSymbol, l *lang, root *ts.Node, src []byte) { + exported, merge, ok := explicitExports(l, root, src) + if !ok { return } for i := range parsed { diff --git a/testdata/cli-go/go.mod b/testdata/cli-go/go.mod new file mode 100644 index 0000000..6514ee4 --- /dev/null +++ b/testdata/cli-go/go.mod @@ -0,0 +1,3 @@ +module example.com/svc + +go 1.26 diff --git a/testdata/cli-go/main.go b/testdata/cli-go/main.go new file mode 100644 index 0000000..6f46415 --- /dev/null +++ b/testdata/cli-go/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "os/exec" + + "example.com/svc/store" +) + +func Handler(name string) error { + rec, err := store.Load(name) + if err != nil { + return err + } + return exec.Command(rec.Path).Run() +} + +func main() { + _ = Handler("x") +} diff --git a/testdata/cli-go/store/store.go b/testdata/cli-go/store/store.go new file mode 100644 index 0000000..34b3cf2 --- /dev/null +++ b/testdata/cli-go/store/store.go @@ -0,0 +1,9 @@ +package store + +type Record struct { + Path string +} + +func Load(name string) (Record, error) { + return Record{Path: name}, nil +} diff --git a/testdata/cli-py/app.py b/testdata/cli-py/app.py new file mode 100644 index 0000000..ba14875 --- /dev/null +++ b/testdata/cli-py/app.py @@ -0,0 +1,8 @@ +import subprocess + +from pkg import loader + + +def handler(name): + rec = loader.load(name) + subprocess.run(rec) diff --git a/testdata/cli-py/pkg/__init__.py b/testdata/cli-py/pkg/__init__.py new file mode 100644 index 0000000..848c37a --- /dev/null +++ b/testdata/cli-py/pkg/__init__.py @@ -0,0 +1 @@ +from . import loader diff --git a/testdata/cli-py/pkg/loader.py b/testdata/cli-py/pkg/loader.py new file mode 100644 index 0000000..b2603b9 --- /dev/null +++ b/testdata/cli-py/pkg/loader.py @@ -0,0 +1,2 @@ +def load(name): + return name