Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/cg-core/src/brand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub const UPSTREAM_CONFIG_FILE: &str = "codegraph.json";
pub const CONFIG_FILE: &str = "cgr.json";
/// Version of the resolution rules: changing resolver behaviour bumps this, which
/// marks older indexes as "built by an older engine" (design doc 5.8).
pub const ENGINE_VERSION: u32 = 3;
pub const ENGINE_VERSION: u32 = 4;
/// Version of the on-disk segment format; a mismatch forces a full rebuild.
pub const FORMAT_VERSION: u32 = 2;
/// The upstream commit whose kernel is vendored (see crates/cg-kernel/UPSTREAM_COMMIT).
Expand Down
2 changes: 2 additions & 0 deletions crates/cg-extract/src/facts/js.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ fn walk(cx: &mut Cx, n: Node, scope: u32) {
if let Some(name) = n.child_by_field_name("name") {
let nm = cx.text(name).to_owned();
cx.add_binding(scope, &nm, BindKind::LocalFn, "", "", 0);
let row = cx.row_at(cx.start(n));
cx.facts.bindings.last_mut().unwrap().node_row = row.unwrap_or(NONE);
}
}
function(cx, n, scope);
Expand Down
28 changes: 24 additions & 4 deletions crates/cg-extract/src/facts/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ pub enum BindKind {
/// Function assigned to a JavaScript object member, rather than a free function.
/// This is declaration metadata and must not participate in lexical lookup.
MemberFn = 5,
/// Raw Rust return annotation, keyed by the function's node. This metadata
/// preserves generic return types the upstream kernel intentionally reduces.
/// Return annotation or a bounded inferred return expression, keyed by the
/// function's node. Metadata only; not a lexical binding.
ReturnType = 6,
}

Expand All @@ -118,8 +118,8 @@ pub struct BindingFact {
pub scope: u32,
pub name: String,
pub kind: BindKind,
/// Declared type, usually reduced to its head (`Foo<T>` -> `Foo`). Rust Result/Option
/// retain their arguments so the resolver does not mistake the container for its value.
/// Declared type, usually reduced to its head (`Foo<T>` -> `Foo`). Rust
/// retains generic arguments to distinguish containers and pointer targets.
pub decl_type: String,
/// Constructor or bounded initializer call path (`Foo::new().build()`), or empty.
pub init_ctor: String,
Expand All @@ -137,6 +137,21 @@ pub struct Facts {
/// `scopes[0]` is the file scope.
pub scopes: Vec<ScopeFact>,
pub bindings: Vec<BindingFact>,
/// Precise call paths lost by the compatibility kernel. Applied to matching
/// call references before storage; node identities and sites stay unchanged.
pub call_names: Vec<CallNameFact>,
/// A supplementary parse exhausted its deadline. The pipeline exposes this
/// as an incomplete file rather than silently advertising complete facts.
pub timed_out: bool,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CallNameFact {
pub line: u32,
pub col: u32,
pub tail: String,
pub kernel_name: String,
pub name: String,
}

impl Facts {
Expand All @@ -158,7 +173,12 @@ impl Facts {

/// Build the facts for one parsed file. `rows` are the kernel tables of the same parse.
pub fn extract(lang: Lang, tree: &Tree, src: &str, rows: FileRows<'_>) -> Facts {
extract_with_timeout(lang, tree, src, rows, std::time::Duration::from_secs(5))
}

pub fn extract_with_timeout(lang: Lang, tree: &Tree, src: &str, rows: FileRows<'_>, timeout: std::time::Duration) -> Facts {
let mut cx = Cx::new(src, rows);
cx.parse_timeout = timeout;
let root = tree.root_node();
match lang {
Lang::TypeScript | Lang::Tsx | Lang::JavaScript | Lang::Jsx => js::extract(&mut cx, root),
Expand Down
77 changes: 76 additions & 1 deletion crates/cg-extract/src/facts/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,13 @@ pub fn py_type_name(text: &str) -> String {
}
if let Some((a, b)) = t.split_once('|') {
let (a, b) = (a.trim(), b.trim());
t = if a == "None" { b } else { a };
t = if a == "None" {
b
} else if b == "None" {
a
} else {
return String::new();
};
}
let name = simple_type_name(t);
match name.as_str() {
Expand Down Expand Up @@ -196,10 +202,12 @@ fn param_names(cx: &mut Cx, params: Node, sc: u32) {
}

fn function(cx: &mut Cx, n: Node, scope: u32) {
let row = cx.row_at(cx.start(n));
if cx.facts.scopes[scope as usize].kind != ScopeKind::Class {
if let Some(name) = n.child_by_field_name("name") {
let nm = cx.text(name).to_owned();
cx.add_binding(scope, &nm, BindKind::LocalFn, "", "", 0);
cx.facts.bindings.last_mut().unwrap().node_row = row.unwrap_or(NONE);
}
}
let sc = cx.push_scope(ScopeKind::Function, scope, n);
Expand All @@ -208,12 +216,79 @@ fn function(cx: &mut Cx, n: Node, scope: u32) {
}
if let Some(body) = n.child_by_field_name("body") {
walk(cx, body, sc);
if let (Some(row), Some(name)) = (row, n.child_by_field_name("name")) {
let (decl_type, init_ctor) = if let Some(ret) = n.child_by_field_name("return_type") {
let raw = cx.text(ret);
let ty = py_type_name(raw);
(if ty.is_empty() { raw.to_owned() } else { ty }, String::new())
} else {
inferred_return(cx, n, body).unwrap_or_default()
};
if !decl_type.is_empty() || !init_ctor.is_empty() {
cx.facts.bindings.push(BindingFact {
scope: sc,
name: cx.text(name).to_owned(),
kind: BindKind::ReturnType,
decl_type,
init_ctor,
line: cx.line(n),
node_row: row,
});
}
}
}
if let Some(dec) = n.child_by_field_name("decorators") {
walk(cx, dec, scope);
}
}

/// Infer only a uniform, explicit return expression. A final return rules out
/// implicit None; nested definitions, generators and arbitrary decorators must
/// not supply evidence for the enclosing function.
fn inferred_return(cx: &Cx, function: Node, body: Node) -> Option<(String, String)> {
if kids(function).any(|n| n.kind() == "async") || function.parent().is_some_and(|p| p.kind() == "decorated_definition") {
return None;
}
if kids(body).filter(|n| n.kind() != "comment").last()?.kind() != "return_statement" {
return None;
}
fn collect(cx: &Cx, n: Node, result: &mut Option<(String, String)>, depth: u8) -> Option<()> {
if depth >= 64 {
return None;
}
match n.kind() {
"function_definition" | "class_definition" | "lambda" => return Some(()),
"yield" => return None,
"return_statement" => {
let value = n.named_child(0)?;
let inferred = match value.kind() {
"call" => {
let callee = value.child_by_field_name("function")?;
if !matches!(callee.kind(), "identifier" | "attribute") || cx.text(callee).len() > 4096 {
return None;
}
(String::new(), format!("{}()", cx.text(callee)))
}
_ => return None,
};
if result.as_ref().is_some_and(|previous| *previous != inferred) {
return None;
}
*result = Some(inferred);
return Some(());
}
_ => {}
}
for child in kids(n) {
collect(cx, child, result, depth + 1)?;
}
Some(())
}
let mut result = None;
collect(cx, body, &mut result, 0)?;
result
}

fn lambda(cx: &mut Cx, n: Node, scope: u32) {
let sc = cx.push_scope(ScopeKind::Function, scope, n);
if let Some(params) = n.child_by_field_name("parameters") {
Expand Down
84 changes: 69 additions & 15 deletions crates/cg-extract/src/facts/rust.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,18 @@ pub fn extract(cx: &mut Cx, root: Node) {
// Conditional item wrappers such as Tokio's cfg_rt! hide `pub use` from
// tree-sitter. Reparse a position-preserving view with just those wrappers
// blanked out. Never expand arbitrary macros or change the kernel's nodes.
if let Some(tree) = conditional_items(cx.src, root) {
if let Some(tree) = conditional_items(cx, root) {
walk(cx, tree.root_node(), 0);
} else {
walk(cx, root, 0);
}
}

fn conditional_items(src: &str, root: Node) -> Option<Tree> {
use std::time::{Duration, Instant};
fn conditional_items(cx: &mut Cx, root: Node) -> Option<Tree> {
use std::time::Instant;
use tree_sitter::{ParseOptions, Parser};

let src = cx.src;
if src.len() > 1024 * 1024 || !src.contains("cfg_") {
return None;
}
Expand Down Expand Up @@ -73,10 +74,22 @@ fn conditional_items(src: &str, root: Node) -> Option<Tree> {
}
let mut parser = Parser::new();
parser.set_language(&cg_kernel::hooks::language("rust")?).ok()?;
let deadline = Instant::now() + Duration::from_millis(100);
let mut cancel = |_: &tree_sitter::ParseState| Instant::now() >= deadline;
// Use the configured parsing budget, not a separate 100 ms cutoff which
// can silently change imports under normal indexing load.
if cx.parse_timeout.is_zero() {
cx.facts.timed_out = true;
return None;
}
let deadline = Instant::now() + cx.parse_timeout;
let mut expired = false;
let mut cancel = |_: &tree_sitter::ParseState| {
expired = Instant::now() >= deadline;
expired
};
let tree =
parser.parse_with_options(&mut |offset, _| masked.get(offset..).unwrap_or_default(), None, Some(ParseOptions::new().progress_callback(&mut cancel)))?;
parser.parse_with_options(&mut |offset, _| masked.get(offset..).unwrap_or_default(), None, Some(ParseOptions::new().progress_callback(&mut cancel)));
cx.facts.timed_out |= expired;
let tree = tree?;
// Non-item wrappers (cfg_if!, DSLs, malformed syntax) are left opaque.
(!tree.root_node().has_error()).then_some(tree)
}
Expand All @@ -101,6 +114,37 @@ fn walk(cx: &mut Cx, n: Node, scope: u32) {
let_decl(cx, n, scope);
walk_children(cx, n, scope);
}
"call_expression" => {
if let Some(f) = n.child_by_field_name("function").filter(|f| f.kind() == "field_expression") {
if let Some(field) = f.child_by_field_name("field") {
let name = value_path(cx, f, 0);
if !name.is_empty() {
let pos = cx.start(n);
let mut kernel_name = String::new();
let mut depth = 0u32;
for ch in name.chars() {
if ch == ')' {
depth = depth.saturating_sub(1);
}
if depth == 0 {
kernel_name.push(ch);
}
if ch == '(' {
depth += 1;
}
}
cx.facts.call_names.push(CallNameFact {
line: (pos >> 32) as u32,
col: pos as u32,
tail: cx.text(field).to_owned(),
kernel_name,
name,
});
}
}
}
walk_children(cx, n, scope);
}
"for_expression" => {
if let Some(p) = n.child_by_field_name("pattern") {
bind_pattern(cx, p, scope, BindKind::Local);
Expand All @@ -120,14 +164,7 @@ fn walk(cx: &mut Cx, n: Node, scope: u32) {
fn rust_type_name(cx: &Cx, n: Node) -> String {
match n.kind() {
"type_identifier" | "scoped_type_identifier" => cx.text(n).to_owned(),
"generic_type" => {
let base = n.child_by_field_name("type").map(|c| rust_type_name(cx, c)).unwrap_or_default();
if matches!(base.rsplit("::").next(), Some("Result" | "Option")) {
cx.text(n).to_owned()
} else {
base
}
}
"generic_type" => cx.text(n).to_owned(),
"reference_type" | "pointer_type" => n.child_by_field_name("type").map(|c| rust_type_name(cx, c)).unwrap_or_default(),
"abstract_type" | "dynamic_type" | "bounded_type" => {
n.child_by_field_name("trait").or_else(|| n.named_child(0)).map(|c| rust_type_name(cx, c)).unwrap_or_default()
Expand Down Expand Up @@ -170,7 +207,22 @@ fn value_path(cx: &Cx, v: Node, depth: u8) -> String {
"generic_function" => v.child_by_field_name("function").map(|f| value_path(cx, f, depth + 1)).unwrap_or_default(),
"struct_expression" => v.child_by_field_name("name").map(|n| cx.text(n).to_owned()).unwrap_or_default(),
"call_expression" => {
v.child_by_field_name("function").map(|f| value_path(cx, f, depth + 1)).filter(|p| !p.is_empty()).map(|p| format!("{p}()")).unwrap_or_default()
let callee = v.child_by_field_name("function").map(|f| value_path(cx, f, depth + 1)).unwrap_or_default();
if callee.is_empty() {
return String::new();
}
if callee.ends_with("::new") {
if let Some(args) = v.child_by_field_name("arguments") {
let values: Vec<_> = kids(args).filter(|n| n.is_named() && !matches!(n.kind(), "comment" | "line_comment" | "block_comment")).collect();
if values.len() == 1 {
let inner = value_path(cx, values[0], depth + 1);
if !inner.is_empty() {
return format!("{callee}({inner})");
}
}
}
}
format!("{callee}()")
}
"field_expression" => {
let base = v.child_by_field_name("value").map(|v| value_path(cx, v, depth + 1)).unwrap_or_default();
Expand Down Expand Up @@ -247,6 +299,8 @@ fn function(cx: &mut Cx, n: Node, scope: u32) {
if let Some(name) = n.child_by_field_name("name") {
let nm = cx.text(name).to_owned();
cx.add_binding(scope, &nm, BindKind::LocalFn, "", "", 0);
let row = cx.row_at(cx.start(n));
cx.facts.bindings.last_mut().unwrap().node_row = row.unwrap_or(NONE);
}
}
let sc = cx.push_scope(ScopeKind::Function, scope, n);
Expand Down
3 changes: 2 additions & 1 deletion crates/cg-extract/src/facts/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pub struct Cx<'a> {
pub src: &'a str,
pub rows: FileRows<'a>,
pub facts: Facts,
pub parse_timeout: std::time::Duration,
line_starts: Vec<u32>,
row_by_start: Option<rustc_hash::FxHashMap<u64, u32>>,
}
Expand All @@ -17,7 +18,7 @@ impl<'a> Cx<'a> {
let line_starts = text::line_starts(src);
let end_line = line_starts.len() as u32;
let facts = Facts::new(pack_pos(end_line + 1, 0));
Cx { src, rows, facts, line_starts, row_by_start: None }
Cx { src, rows, facts, parse_timeout: std::time::Duration::from_secs(5), line_starts, row_by_start: None }
}

/// Kernel node row whose span starts exactly at `pos` (packed line / UTF-16 column), skipping
Expand Down
12 changes: 8 additions & 4 deletions crates/cg-extract/src/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ pub fn file_node_only(rel: &str, line_count: u32) -> EmitOut {
}

enum KernelOutcome {
Ok { emit: EmitOut, facts: Facts, had_error_tree: bool, facts_failed: bool },
Ok { emit: EmitOut, facts: Box<Facts>, had_error_tree: bool, facts_failed: bool },
TooDeep,
TimedOut,
Failed(String),
Expand Down Expand Up @@ -161,7 +161,7 @@ fn run_kernel(rel: &str, src: &str, lang: Lang, cfg: &ExtractConfig) -> KernelOu
let mut facts_failed = false;
let facts = match &tree {
Some(t) => {
let r = catch_unwind(AssertUnwindSafe(|| facts::extract(lang, t, src, FileRows::new(&emit))));
let r = catch_unwind(AssertUnwindSafe(|| facts::extract_with_timeout(lang, t, src, FileRows::new(&emit), cfg.parse_timeout)));
match r {
Ok(f) => f,
Err(_) => {
Expand All @@ -172,7 +172,7 @@ fn run_kernel(rel: &str, src: &str, lang: Lang, cfg: &ExtractConfig) -> KernelOu
}
None => Facts::default(),
};
KernelOutcome::Ok { emit, facts, had_error_tree, facts_failed }
KernelOutcome::Ok { emit, facts: Box::new(facts), had_error_tree, facts_failed }
}
}
}
Expand Down Expand Up @@ -233,7 +233,11 @@ pub fn extract_bytes(rel: &str, bytes: &[u8], mtime_ns: i64, lang_hint: Lang, cf
if facts_failed {
errors.push("panic while building facts".to_owned());
}
(emit, facts)
if facts.timed_out {
flags |= fileflags::TIMED_OUT;
errors.push(format!("supplementary facts parse timed out after {:?}", cfg.parse_timeout));
}
(emit, *facts)
}
KernelOutcome::TooDeep => {
errors.push("nesting too deep for the walker".to_owned());
Expand Down
2 changes: 1 addition & 1 deletion crates/cg-extract/tests/facts_langs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ impl Server {
assert_eq!(bind(&f, "self").decl_type, "Self");
assert_eq!(bind(&f, "other").decl_type, "Other");
assert_eq!(bind(&f, "t").init_ctor, "T::new()");
assert_eq!(bind(&f, "m").decl_type, "HashMap");
assert_eq!(bind(&f, "m").decl_type, "HashMap<String, u32>");
assert_eq!(bind(&f, "s").init_ctor, "Server");
assert_eq!(bind(&f, "item").kind, BindKind::Local);
assert_eq!(bind(&f, "x").decl_type, "Foo");
Expand Down
Loading
Loading