feat: merge AST-based code search and enhancements from develop

Cargo.toml

@@ -5,7 +5,9 @@ members = [
     "crates/reposcout-tui",
     "crates/reposcout-api",
     "crates/reposcout-cache",
-    "crates/reposcout-deps", "crates/reposcout-semantic",
+    "crates/reposcout-deps",
+    "crates/reposcout-semantic",
+    "crates/reposcout-ast",
 ]
 resolver = "2"
 

crates/reposcout-api/src/github.rs

@@ -298,6 +298,12 @@ impl GitHubClient {
                         "Response snippet: {}",
                         &response_text[..response_text.len().min(1000)]
                     );
+                    // Save full response to file for debugging
+                    if let Err(write_err) = std::fs::write("/tmp/github_response_debug.json", &response_text) {
+                        tracing::warn!("Failed to write debug response: {}", write_err);
+                    } else {
+                        tracing::error!("Full response saved to /tmp/github_response_debug.json");
+                    }
                     GitHubError::ParseError(e)
                 })?;
             Ok(search_result.items)
@@ -513,8 +519,10 @@ pub struct CodeSearchItem {
     pub name: String,
     pub path: String,
     pub sha: String,
-    pub url: String,
-    pub git_url: String,
+    #[serde(default)]
+    pub url: Option<String>,
+    #[serde(default)]
+    pub git_url: Option<String>,
     pub html_url: String,
     pub repository: CodeSearchRepository,
     #[serde(default)]
@@ -530,10 +538,20 @@ pub struct CodeSearchRepository {
     pub full_name: String,
     #[serde(default)]
     pub description: Option<String>,
-    pub html_url: String,
+    #[serde(default)]
+    pub html_url: Option<String>,
     pub owner: Owner,
     #[serde(default)]
     pub private: bool,
+    #[serde(default)]
+    pub stargazers_count: u32,
+    #[serde(default)]
+    pub language: Option<String>,
+    // Additional fields that might be null or missing
+    #[serde(default)]
+    pub fork: Option<bool>,
+    #[serde(default)]
+    pub url: Option<String>,
 }
 
 /// Text match containing the actual code snippet
@@ -553,7 +571,8 @@ pub struct TextMatch {
 /// Individual match within a text fragment
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Match {
-    pub text: String,
+    #[serde(default)]
+    pub text: Option<String>,
     pub indices: Vec<usize>,
 }
 
@@ -588,7 +607,18 @@ pub struct GitHubRepo {
 pub struct Owner {
     pub login: String,
     pub id: u64,
-    pub avatar_url: String,
+    #[serde(default)]
+    pub avatar_url: Option<String>,
+    #[serde(default)]
+    pub gravatar_id: Option<String>,
+    #[serde(default)]
+    pub url: Option<String>,
+    #[serde(default)]
+    pub html_url: Option<String>,
+    // Flatten any additional unknown fields
+    #[serde(flatten)]
+    #[serde(default)]
+    pub additional_fields: std::collections::HashMap<String, serde_json::Value>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/reposcout-ast/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "reposcout-ast"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+
+[dependencies]
+# Internal dependencies
+reposcout-core = { path = "../reposcout-core" }
+
+# Tree-sitter core
+tree-sitter = "0.22"
+
+# Language grammars
+tree-sitter-rust = "0.21"
+tree-sitter-python = "0.21"
+tree-sitter-javascript = "0.21"
+tree-sitter-typescript = "0.21"
+tree-sitter-go = "0.21"
+tree-sitter-c = "0.21"
+tree-sitter-cpp = "0.22"
+
+# Utilities
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "2.0"
+tracing = "0.1"
+regex = "1.10"
+once_cell = "1.19"
+
+[dev-dependencies]
+tokio = { version = "1", features = ["full"] }

crates/reposcout-ast/src/error.rs

@@ -0,0 +1,30 @@
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum AstError {
+    #[error("Unsupported language: {0}")]
+    UnsupportedLanguage(String),
+
+    #[error("Parse error: {0}")]
+    ParseError(String),
+
+    #[error("Tree-sitter error: {0}")]
+    TreeSitterError(String),
+
+    #[error("Extraction failed: {0}")]
+    ExtractionError(String),
+
+    #[error("Query parsing error: {0}")]
+    QueryParseError(String),
+
+    #[error("Timeout while parsing (exceeded {timeout_ms}ms)")]
+    ParseTimeout { timeout_ms: u64 },
+
+    #[error("File too large: {size} bytes (max: {max})")]
+    FileTooLarge { size: usize, max: usize },
+
+    #[error("IO error: {0}")]
+    IoError(#[from] std::io::Error),
+}
+
+pub type Result<T> = std::result::Result<T, AstError>;