Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/cg-core/src/brand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub const UPSTREAM_CONFIG_FILE: &str = "codegraph.json";
pub const CONFIG_FILE: &str = "cgr.json";
/// Version of the resolution rules: changing resolver behaviour bumps this, which
/// marks older indexes as "built by an older engine" (design doc 5.8).
pub const ENGINE_VERSION: u32 = 2;
pub const ENGINE_VERSION: u32 = 3;
/// Version of the on-disk segment format; a mismatch forces a full rebuild.
pub const FORMAT_VERSION: u32 = 2;
/// The upstream commit whose kernel is vendored (see crates/cg-kernel/UPSTREAM_COMMIT).
Expand Down
109 changes: 109 additions & 0 deletions crates/cg-extract/src/facts/js.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ fn walk(cx: &mut Cx, n: Node, scope: u32) {
export_stmt(cx, n);
walk_children(cx, n, scope);
}
"assignment_expression" => {
if scope == 0 {
commonjs_export(cx, n);
}
walk_children(cx, n, scope);
}
"function_declaration" | "generator_function_declaration" => {
if scope != 0 {
if let Some(name) = n.child_by_field_name("name") {
Expand Down Expand Up @@ -280,7 +286,110 @@ fn export_stmt(cx: &mut Cx, n: Node) {
}
}

fn commonjs_export(cx: &mut Cx, n: Node) {
let Some(left) = n.child_by_field_name("left") else { return };
let target = cx.text(left);
let root = target.split('.').next().unwrap_or(target);
if cx.lookup_binding(0, root).is_some() {
return; // A user variable named module/exports shadows CommonJS globals.
}
let name = if target == "module.exports" {
COMMONJS_MODULE
} else if let Some(name) = target.strip_prefix("module.exports.").or_else(|| target.strip_prefix("exports.")) {
if name.is_empty() || name.contains(['.', '[', ']']) {
return;
}
name
} else {
return;
};
let Some(value) = n.child_by_field_name("right") else { return };
commonjs_value(cx, name, value);
}

fn commonjs_value(cx: &mut Cx, name: &str, value: Node) {
if let Some(spec) = require_target(cx, value) {
let line = cx.line(value);
cx.add_import(ImportFact {
local: String::new(),
imported: "*".into(),
spec: spec.clone(),
kind: ImportKind::SideEffect,
type_only: false,
scope: 0,
line,
});
if name == COMMONJS_MODULE {
cx.facts.exports.push(ExportFact { name: "*".into(), node_row: NONE, from_spec: spec.clone(), from_name: "*".into(), star: true });
cx.facts.exports.push(ExportFact { name: "default".into(), node_row: NONE, from_spec: spec.clone(), from_name: "default".into(), star: false });
}
cx.facts.exports.push(ExportFact { name: name.to_owned(), node_row: NONE, from_spec: spec, from_name: COMMONJS_MODULE.into(), star: false });
return;
}
let (node_row, from_spec, from_name) = match value.kind() {
"identifier" | "member_expression" => {
let text = cx.text(value);
let (root, member) = text.split_once('.').map_or((text, None), |(a, b)| (a, Some(b)));
if let Some(imp) = cx.facts.imports.iter().find(|i| i.scope == 0 && i.local == root) {
let imported = member.unwrap_or(if imp.imported == "*" { COMMONJS_MODULE } else { &imp.imported });
(NONE, imp.spec.clone(), imported.to_owned())
} else if member.is_none() {
(NONE, String::new(), text.to_owned())
} else {
return;
}
}
"function_expression" | "function" | "generator_function" | "arrow_function" | "class" => {
(cx.row_at(cx.start(value)).unwrap_or(NONE), String::new(), String::new())
}
"object" if name == COMMONJS_MODULE => {
cx.facts.exports.push(ExportFact { name: name.to_owned(), node_row: NONE, from_spec: String::new(), from_name: "*".into(), star: false });
for prop in kids(value) {
if prop.kind() == "pair" {
if let (Some(key), Some(val)) = (prop.child_by_field_name("key"), prop.child_by_field_name("value")) {
let key = unquote(cx.text(key)).to_owned();
commonjs_value(cx, &key, val);
}
} else if prop.kind() == "shorthand_property_identifier" {
commonjs_value(cx, cx.text(prop), prop);
}
}
return;
}
"shorthand_property_identifier" => (NONE, String::new(), cx.text(value).to_owned()),
_ => return,
};
cx.facts.exports.push(ExportFact { name: name.to_owned(), node_row, from_spec, from_name, star: false });
}

fn function(cx: &mut Cx, n: Node, scope: u32) {
// The kernel labels `obj.method = function method() {}` as a Function. Keep
// evidence of the member assignment so an unknown receiver cannot match an
// unrelated, plain function with the same name.
if let Some(parent) = n.parent() {
let member = match parent.kind() {
"object" if n.kind() == "method_definition" => n.child_by_field_name("name"),
"assignment_expression" if parent.child_by_field_name("right") == Some(n) => parent
.child_by_field_name("left")
.filter(|left| matches!(left.kind(), "member_expression" | "subscript_expression"))
.and_then(|left| left.child_by_field_name("property").or_else(|| left.child_by_field_name("index").filter(|index| index.kind() == "string"))),
"pair" if parent.child_by_field_name("value") == Some(n) => parent.child_by_field_name("key"),
_ => None,
};
if let Some(member) = member {
if let Some(row) = cx.row_at(cx.start(n)) {
cx.facts.bindings.push(BindingFact {
scope,
name: cx.text(member).trim_matches(['\'', '"']).to_owned(),
kind: BindKind::MemberFn,
decl_type: String::new(),
init_ctor: String::new(),
line: cx.line(n),
node_row: row,
});
}
}
}
let sc = cx.push_scope(ScopeKind::Function, scope, n);
let line = cx.line(n);
if let Some(p) = n.child_by_field_name("parameter") {
Expand Down
15 changes: 13 additions & 2 deletions crates/cg-extract/src/facts/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ mod rust;
mod util;
pub use util::Cx;

/// Internal export key for the module value itself, distinct from a named/ES
/// `default` export. The leading NUL keeps it out of ordinary source identifiers.
pub const COMMONJS_MODULE: &str = "\0cjs:module.exports";

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[repr(u8)]
pub enum ImportKind {
Expand Down Expand Up @@ -101,16 +105,23 @@ pub enum BindKind {
Field = 3,
/// The file's declared package (Go `package x`, Java/Kotlin `package a.b`); `name` is the package.
Package = 4,
/// Function assigned to a JavaScript object member, rather than a free function.
/// This is declaration metadata and must not participate in lexical lookup.
MemberFn = 5,
/// Raw Rust return annotation, keyed by the function's node. This metadata
/// preserves generic return types the upstream kernel intentionally reduces.
ReturnType = 6,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BindingFact {
pub scope: u32,
pub name: String,
pub kind: BindKind,
/// Declared type as written (`Foo`, `Foo<T>` reduced to `Foo` by the extractor), or empty.
/// Declared type, usually reduced to its head (`Foo<T>` -> `Foo`). Rust Result/Option
/// retain their arguments so the resolver does not mistake the container for its value.
pub decl_type: String,
/// Constructor name when initialised by `new Foo()` / `Foo()` / `Foo{}`, or empty.
/// Constructor or bounded initializer call path (`Foo::new().build()`), or empty.
pub init_ctor: String,
/// 1-based declaration line; the binding is visible from here on (hoisted functions
/// are visible throughout their scope and use line 0).
Expand Down
123 changes: 115 additions & 8 deletions crates/cg-extract/src/facts/rust.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,80 @@ use super::util::*;
use super::*;

pub fn extract(cx: &mut Cx, root: Node) {
walk(cx, root, 0);
// Conditional item wrappers such as Tokio's cfg_rt! hide `pub use` from
// tree-sitter. Reparse a position-preserving view with just those wrappers
// blanked out. Never expand arbitrary macros or change the kernel's nodes.
if let Some(tree) = conditional_items(cx.src, root) {
walk(cx, tree.root_node(), 0);
} else {
walk(cx, root, 0);
}
}

fn conditional_items(src: &str, root: Node) -> Option<Tree> {
use std::time::{Duration, Instant};
use tree_sitter::{ParseOptions, Parser};

if src.len() > 1024 * 1024 || !src.contains("cfg_") {
return None;
}
fn wrapper(name: &str) -> bool {
name.starts_with("cfg_") && name != "cfg_if" && name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_')
}
fn blank(bytes: &mut [u8]) {
for b in bytes {
if *b != b'\n' && *b != b'\r' {
*b = b' ';
}
}
}
fn collect(n: Node, src: &str, ranges: &mut Vec<(usize, usize)>, depth: usize) {
if depth > 128 || n.kind() == "macro_definition" {
return;
}
if matches!(n.kind(), "macro_invocation" | "token_tree") {
let mut c = n.walk();
let children: Vec<_> = n.children(&mut c).collect();
let mut i = 0;
while i < children.len() {
if i + 2 < children.len() && children[i + 1].kind() == "!" && children[i + 2].kind() == "token_tree" {
let name = &src[children[i].byte_range()];
let body = children[i + 2];
if wrapper(name) && src.as_bytes().get(body.start_byte()) == Some(&b'{') {
ranges.push((children[i].start_byte(), body.start_byte() + 1));
ranges.push((body.end_byte() - 1, body.end_byte()));
collect(body, src, ranges, depth + 1);
}
// An unknown macro's tokens are not ordinary Rust items.
i += 3;
} else {
collect(children[i], src, ranges, depth + 1);
i += 1;
}
}
} else {
for ch in kids(n) {
collect(ch, src, ranges, depth + 1);
}
}
}
let mut ranges = Vec::new();
collect(root, src, &mut ranges, 0);
if ranges.is_empty() {
return None;
}
let mut masked = src.as_bytes().to_vec();
for (start, end) in ranges {
blank(&mut masked[start..end]);
}
let mut parser = Parser::new();
parser.set_language(&cg_kernel::hooks::language("rust")?).ok()?;
let deadline = Instant::now() + Duration::from_millis(100);
let mut cancel = |_: &tree_sitter::ParseState| Instant::now() >= deadline;
let tree =
parser.parse_with_options(&mut |offset, _| masked.get(offset..).unwrap_or_default(), None, Some(ParseOptions::new().progress_callback(&mut cancel)))?;
// Non-item wrappers (cfg_if!, DSLs, malformed syntax) are left opaque.
(!tree.root_node().has_error()).then_some(tree)
}

fn walk_children(cx: &mut Cx, n: Node, scope: u32) {
Expand Down Expand Up @@ -47,7 +120,14 @@ fn walk(cx: &mut Cx, n: Node, scope: u32) {
fn rust_type_name(cx: &Cx, n: Node) -> String {
match n.kind() {
"type_identifier" | "scoped_type_identifier" => cx.text(n).to_owned(),
"generic_type" => n.child_by_field_name("type").map(|c| rust_type_name(cx, c)).unwrap_or_default(),
"generic_type" => {
let base = n.child_by_field_name("type").map(|c| rust_type_name(cx, c)).unwrap_or_default();
if matches!(base.rsplit("::").next(), Some("Result" | "Option")) {
cx.text(n).to_owned()
} else {
base
}
}
"reference_type" | "pointer_type" => n.child_by_field_name("type").map(|c| rust_type_name(cx, c)).unwrap_or_default(),
"abstract_type" | "dynamic_type" | "bounded_type" => {
n.child_by_field_name("trait").or_else(|| n.named_child(0)).map(|c| rust_type_name(cx, c)).unwrap_or_default()
Expand Down Expand Up @@ -78,15 +158,29 @@ fn bind_pattern(cx: &mut Cx, p: Node, scope: u32, kind: BindKind) {
}

fn value_info(cx: &Cx, v: Node) -> String {
value_path(cx, v, 0)
}

fn value_path(cx: &Cx, v: Node, depth: u8) -> String {
if depth >= 16 || v.end_byte() - v.start_byte() > 4096 {
return String::new();
}
match v.kind() {
"identifier" | "scoped_identifier" => cx.text(v).to_owned(),
"generic_function" => v.child_by_field_name("function").map(|f| value_path(cx, f, depth + 1)).unwrap_or_default(),
"struct_expression" => v.child_by_field_name("name").map(|n| cx.text(n).to_owned()).unwrap_or_default(),
"call_expression" => match v.child_by_field_name("function") {
Some(f) if matches!(f.kind(), "identifier" | "scoped_identifier") => format!("{}()", cx.text(f)),
_ => String::new(),
},
"try_expression" | "await_expression" | "reference_expression" | "parenthesized_expression" => {
v.named_child(0).map(|c| value_info(cx, c)).unwrap_or_default()
"call_expression" => {
v.child_by_field_name("function").map(|f| value_path(cx, f, depth + 1)).filter(|p| !p.is_empty()).map(|p| format!("{p}()")).unwrap_or_default()
}
"field_expression" => {
let base = v.child_by_field_name("value").map(|v| value_path(cx, v, depth + 1)).unwrap_or_default();
match v.child_by_field_name("field") {
Some(field) if !base.is_empty() => format!("{base}.{}", cx.text(field)),
_ => String::new(),
}
}
"try_expression" => v.named_child(0).map(|c| value_path(cx, c, depth + 1)).filter(|p| !p.is_empty()).map(|p| format!("{p}?")).unwrap_or_default(),
"await_expression" | "reference_expression" | "parenthesized_expression" => v.named_child(0).map(|c| value_path(cx, c, depth + 1)).unwrap_or_default(),
_ => String::new(),
}
}
Expand Down Expand Up @@ -136,6 +230,19 @@ fn params(cx: &mut Cx, list: Node, sc: u32) {
}

fn function(cx: &mut Cx, n: Node, scope: u32) {
if let (Some(name), Some(ret)) = (n.child_by_field_name("name"), n.child_by_field_name("return_type")) {
if let Some(row) = cx.row_at(cx.start(n)) {
cx.facts.bindings.push(BindingFact {
scope,
name: cx.text(name).to_owned(),
kind: BindKind::ReturnType,
decl_type: cx.text(ret).to_owned(),
init_ctor: String::new(),
line: cx.line(n),
node_row: row,
});
}
}
if cx.facts.scopes[scope as usize].kind == ScopeKind::Function {
if let Some(name) = n.child_by_field_name("name") {
let nm = cx.text(name).to_owned();
Expand Down
8 changes: 7 additions & 1 deletion crates/cg-extract/src/facts/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,13 @@ impl<'a> Cx<'a> {
pub fn lookup_binding(&self, mut scope: u32, name: &str) -> Option<(String, String)> {
loop {
if self.facts.scopes[scope as usize].kind != ScopeKind::Class {
if let Some(b) = self.facts.bindings.iter().rev().find(|b| b.scope == scope && b.name == name) {
if let Some(b) = self
.facts
.bindings
.iter()
.rev()
.find(|b| b.scope == scope && b.name == name && !matches!(b.kind, BindKind::MemberFn | BindKind::ReturnType))
{
return Some((b.decl_type.clone(), b.init_ctor.clone()));
}
}
Expand Down
Loading
Loading