From 32c0b9496643bf9402e28abc4d9daa4f269e3743 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Thu, 8 Oct 2026 08:23:54 +0700 Subject: [PATCH] fix(watch): reindex only files that changed; --filter restricts what is indexed detect_changes returned every source file on every tick and the whole project was reindexed after a trigger, so --filter only gated whether a cycle fired. The watch loop now keeps an (mtime, size) snapshot, diffs it each tick (new/modified/deleted), applies the ext/git-only/--filter/skip filters, and reindexes only the changed files via a new include predicate (IndexSession::index_matching). Deletions are pruned by the same run; idle ticks do no indexing work. Fixes both cora watch and index --watch. Closes #578 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: ajianaz --- src/commands/watch.rs | 282 +++++++++++++++++++++++++++++++++++++----- src/index/mod.rs | 24 +++- src/index/session.rs | 16 +++ 3 files changed, 286 insertions(+), 36 deletions(-) diff --git a/src/commands/watch.rs b/src/commands/watch.rs index f49108f..a68aecb 100644 --- a/src/commands/watch.rs +++ b/src/commands/watch.rs @@ -2,27 +2,34 @@ //! //! Watches the project directory for file changes and re-indexes on save. //! Supports debounce window, git-only filtering, and glob patterns. +//! +//! Each poll compares an `(mtime, size)` snapshot of the watched files with +//! the previous one; only new/modified files are re-indexed (deletions are +//! pruned by the same run), and idle ticks do no indexing work. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use colored::Colorize; +use crate::engine::path_match::{PathMatcher, PathPattern}; use crate::index::session::IndexSession; /// Entry point for `cora watch` (also backs `cora index --watch`). /// /// Runs an initial index, then polls for changes at the debounce interval. -/// On each poll cycle, re-indexes the project and reports updated files/symbols. +/// On each poll cycle, re-indexes only the files that changed since the last +/// snapshot and reports updated files/symbols. /// Config, backend, skip patterns and root all come from the [`IndexSession`]. /// /// # Arguments /// * `session` - configured index session (owns root, DB, skip patterns) /// * `debounce_ms` - Minimum time between reindex cycles (default 500ms) /// * `git_only` - If true, only process files tracked by git -/// * `filter` - Optional glob pattern (e.g. `src/**/*.rs`) +/// * `filter` - Optional glob pattern (e.g. `src/**/*.rs`); restricts which +/// files are watched and re-indexed /// * `verbose` - Verbose output pub fn run_watch( session: &IndexSession, @@ -42,14 +49,29 @@ pub fn run_watch( // Compile glob filter if provided let glob_matcher = filter.map(|p| { - crate::engine::path_match::PathPattern::new(p).unwrap_or_else(|e| { + PathPattern::new(p).unwrap_or_else(|e| { eprintln!("{} Invalid glob pattern '{p}': {e}", "⚠ ".yellow()); std::process::exit(1); }) }); + // Same exclusions as the indexer (ignore.files / index_skip_files) + let skip = session.skip_patterns().map(PathMatcher::new); + let scan = || { + scan_files( + project_root, + &git_files, + glob_matcher.as_ref(), + skip.as_ref(), + ) + }; + let debounce = Duration::from_millis(debounce_ms); + // Baseline snapshot, taken before the initial index so an edit racing + // with it still shows up as a change on the first poll. + let mut snapshot = scan()?; + // Initial index eprintln!("{}", "🔍 Initial index...".cyan()); let stats = session.index(verbose)?; @@ -82,20 +104,35 @@ pub fn run_watch( continue; } - // Check for changed files - let changed = detect_changes(project_root, &git_files, glob_matcher.as_ref())?; - if changed.is_empty() { + // Compare the tree against the last-seen snapshot + let current = scan()?; + let changes = diff_snapshots(&snapshot, ¤t); + snapshot = current; + if changes.is_empty() { continue; } last_reindex = now; if verbose { - eprintln!("{}", format!("Changed files: {}", changed.len()).dimmed()); + eprintln!( + "{}", + format!( + "Changed files: {} ({} deleted)", + changes.changed.len(), + changes.deleted.len() + ) + .dimmed() + ); } - // Re-index - let stats = session.index(verbose)?; + // Re-index only the changed files; deletions are pruned by the same run. + let changed_rel: HashSet = changes + .changed + .iter() + .map(|p| rel_string(project_root, p)) + .collect(); + let stats = session.index_matching(verbose, &|rel| changed_rel.contains(rel))?; if stats.files_indexed > 0 { eprintln!( @@ -110,13 +147,60 @@ pub fn run_watch( } } -/// Detect files that changed since last check by comparing modification times. -fn detect_changes( +/// `(mtime in ns, size)` per watched file, as last seen on disk. +type Snapshot = HashMap; + +/// Files that differ between two snapshots. +#[derive(Debug, Default)] +struct ChangeSet { + /// New or modified files. + changed: Vec, + /// Files present in the old snapshot but gone now. + deleted: Vec, +} + +impl ChangeSet { + fn is_empty(&self) -> bool { + self.changed.is_empty() && self.deleted.is_empty() + } +} + +/// Root-relative path in the form the indexer stores. +fn rel_string(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .to_string() +} + +/// New or (mtime,size)-different files are `changed`; files missing from +/// `new` are `deleted`. +fn diff_snapshots(old: &Snapshot, new: &Snapshot) -> ChangeSet { + let mut set = ChangeSet::default(); + for (path, sig) in new { + if old.get(path) != Some(sig) { + set.changed.push(path.clone()); + } + } + for path in old.keys() { + if !new.contains_key(path) { + set.deleted.push(path.clone()); + } + } + set.changed.sort(); + set.deleted.sort(); + set +} + +/// Snapshot every watched file: source extension, git-only, `--filter` glob +/// and the session's skip patterns (same exclusions as the indexer). +fn scan_files( project_root: &Path, git_files: &Option>, - glob_matcher: Option<&crate::engine::path_match::PathPattern>, -) -> Result> { - let mut changed = Vec::new(); + glob_matcher: Option<&PathPattern>, + skip: Option<&PathMatcher>, +) -> Result { + let mut snapshot = Snapshot::new(); let extensions: &[&str] = &[ "rs", "py", "js", "ts", "go", "java", "c", "cpp", "h", "rb", "php", "scala", "cs", "kt", "svelte", "jsx", "tsx", @@ -155,11 +239,25 @@ fn detect_changes( } } - changed.push(path.to_path_buf()); + // Config skip patterns (ignore.files / index_skip_files) + if skip.is_some_and(|m| m.is_match(&rel.to_string_lossy())) { + return; + } + + let Ok(meta) = std::fs::metadata(path) else { + return; + }; + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_nanos()) + .unwrap_or(0); + snapshot.insert(path.to_path_buf(), (mtime, meta.len())); }; walk_files(project_root, &mut walker)?; - Ok(changed) + Ok(snapshot) } /// Recursively walk directory and call `f` for each file path. @@ -253,8 +351,6 @@ mod tests { assert!(found.iter().any(|p| p.ends_with("main.rs"))); assert!(found.iter().any(|p| p.ends_with("lib.rs"))); - // Hidden files should NOT be found (directory skip) - // Note: walk_files itself doesn't skip hidden at top-level, only in subdirs } #[test] @@ -265,36 +361,154 @@ mod tests { assert!(result.is_err() || result.unwrap().is_empty()); } + fn scan(root: &Path) -> Snapshot { + scan_files(root, &None, None, None).unwrap() + } + + /// Rewrite a file with a clearly different mtime and size. + fn touch_modify(path: &Path, body: &str) { + fs::write(path, body).unwrap(); + let t = std::time::SystemTime::now() + Duration::from_secs(5); + fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(t) + .unwrap(); + } + #[test] - fn test_detect_changes_empty_dir() { + fn empty_dir_has_no_changes() { let tmp = TempDir::new().unwrap(); - let changed = detect_changes(tmp.path(), &None, None).unwrap(); - assert!(changed.is_empty()); + let snap = scan(tmp.path()); + assert!(snap.is_empty()); + assert!(diff_snapshots(&snap, &scan(tmp.path())).is_empty()); } #[test] - fn test_detect_changes_with_source_file() { + fn filters_non_source() { let tmp = TempDir::new().unwrap(); let root = tmp.path(); - + fs::write(root.join("README.md"), "# readme").unwrap(); fs::write(root.join("main.rs"), "fn main() {}").unwrap(); + let snap = scan(root); + assert!(snap.keys().any(|p| p.ends_with("main.rs"))); + assert!(!snap.keys().any(|p| p.ends_with("README.md"))); + } - let changed = detect_changes(root, &None, None).unwrap(); - assert!(!changed.is_empty()); - assert!(changed.iter().any(|p| p.ends_with("main.rs"))); + #[test] + fn unchanged_tree_is_empty_change_set() { + let tmp = TempDir::new().unwrap(); + fs::write(tmp.path().join("a.rs"), "fn a() {}").unwrap(); + // The first scan is the baseline: a non-empty tree is not "all changed". + let baseline = scan(tmp.path()); + assert!(!baseline.is_empty()); + assert!(diff_snapshots(&baseline, &scan(tmp.path())).is_empty()); } #[test] - fn test_detect_changes_filters_non_source() { + fn modified_file_is_the_only_change() { let tmp = TempDir::new().unwrap(); let root = tmp.path(); + fs::write(root.join("a.rs"), "fn a() {}").unwrap(); + fs::write(root.join("b.rs"), "fn b() {}").unwrap(); + let before = scan(root); + touch_modify(&root.join("b.rs"), "fn b() {}\nfn b2() {}\n"); + let d = diff_snapshots(&before, &scan(root)); + assert_eq!(d.changed, vec![root.join("b.rs")]); + assert!(d.deleted.is_empty()); + } - fs::write(root.join("README.md"), "# readme").unwrap(); - fs::write(root.join("main.rs"), "fn main() {}").unwrap(); + #[test] + fn new_and_deleted_files_detected() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + fs::write(root.join("a.rs"), "fn a() {}").unwrap(); + fs::write(root.join("b.rs"), "fn b() {}").unwrap(); + let before = scan(root); + fs::write(root.join("c.rs"), "fn c() {}").unwrap(); + fs::remove_file(root.join("b.rs")).unwrap(); + let d = diff_snapshots(&before, &scan(root)); + assert_eq!(d.changed, vec![root.join("c.rs")]); + assert_eq!(d.deleted, vec![root.join("b.rs")]); + } + + #[test] + fn glob_filter_ignores_non_matching_changes() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write(root.join("src/a.rs"), "fn a() {}").unwrap(); + fs::write(root.join("src/b.rs"), "fn b() {}").unwrap(); + let pat = PathPattern::new("a.rs").unwrap(); + let s = || scan_files(root, &None, Some(&pat), None).unwrap(); + let before = s(); + assert_eq!(before.len(), 1); + touch_modify(&root.join("src/b.rs"), "fn b() {}\nfn b2() {}\n"); + assert!(diff_snapshots(&before, &s()).is_empty()); + touch_modify(&root.join("src/a.rs"), "fn a() {}\nfn a2() {}\n"); + let d = diff_snapshots(&before, &s()); + assert_eq!(d.changed, vec![root.join("src/a.rs")]); + } - let changed = detect_changes(root, &None, None).unwrap(); - // .md should not be detected, .rs should - assert!(changed.iter().any(|p| p.ends_with("main.rs"))); - assert!(!changed.iter().any(|p| p.ends_with("README.md"))); + #[test] + fn skip_patterns_are_respected() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + fs::create_dir_all(root.join("gen")).unwrap(); + fs::write(root.join("gen/x.rs"), "fn x() {}").unwrap(); + fs::write(root.join("a.rs"), "fn a() {}").unwrap(); + let skip = PathMatcher::new(&["gen/**".to_string()]); + let before = scan_files(root, &None, None, Some(&skip)).unwrap(); + assert_eq!(before.len(), 1); + touch_modify(&root.join("gen/x.rs"), "fn x() {}\nfn y() {}\n"); + let after = scan_files(root, &None, None, Some(&skip)).unwrap(); + assert!(diff_snapshots(&before, &after).is_empty()); + } + + /// End to end: a non-matching change yields no change set (the loop never + /// indexes); a matching one reindexes exactly that file; deletions prune. + #[test] + fn filtered_reindex_touches_only_matching_changed_files() { + use crate::index::session::{ConfigSource, IndexSession}; + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write(root.join("src/a.rs"), "pub fn alpha() {}\n").unwrap(); + fs::write(root.join("src/b.rs"), "pub fn beta() {}\n").unwrap(); + let session = IndexSession::open_at(root, ConfigSource::ProjectOnlyAt(root)).unwrap(); + session.index(false).unwrap(); + let pat = PathPattern::new("a.rs").unwrap(); + let s = || scan_files(root, &None, Some(&pat), None).unwrap(); + let snap = s(); + + // b.rs changes: nothing detected. + touch_modify(&root.join("src/b.rs"), "pub fn beta() {}\npub fn b2() {}\n"); + let cur = s(); + assert!(diff_snapshots(&snap, &cur).is_empty()); + // Even a forced run with an empty include set leaves b.rs stale. + let st = session.index_matching(false, &|_| false).unwrap(); + assert_eq!(st.files_indexed, 0); + + // a.rs changes: exactly one file reindexed. + touch_modify( + &root.join("src/a.rs"), + "pub fn alpha() {}\npub fn a2() {}\n", + ); + let next = s(); + let d = diff_snapshots(&cur, &next); + let rel: HashSet = d.changed.iter().map(|p| rel_string(root, p)).collect(); + let st = session.index_matching(false, &|r| rel.contains(r)).unwrap(); + assert_eq!(st.files_indexed, 1); + assert_eq!(st.files_pruned, 0); + assert_eq!(session.summary().unwrap().total_files, 2); + + // Deleting a.rs prunes only it. + fs::remove_file(root.join("src/a.rs")).unwrap(); + let d = diff_snapshots(&next, &s()); + assert_eq!(d.deleted.len(), 1); + let st = session.index_matching(false, &|_| false).unwrap(); + assert_eq!(st.files_pruned, 1); + assert_eq!(session.summary().unwrap().total_files, 1); } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 0e459d1..02cfb85 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -305,7 +305,7 @@ pub fn skip_patterns_from_config( /// `index_project_with_skip` with the resolved config. #[cfg(test)] pub fn index_project(conn: &Connection, root: &Path, verbose: bool) -> anyhow::Result { - index_project_with_id(conn, ensure_project(conn, root)?, root, verbose, None) + index_project_with_id(conn, ensure_project(conn, root)?, root, verbose, None, None) } /// Index a project directory, honoring skip patterns (glob `*`/`**`, matched @@ -321,6 +321,20 @@ pub fn index_project_with_skip( root: &Path, verbose: bool, skip_patterns: Option<&[String]>, +) -> anyhow::Result { + index_project_filtered(conn, root, verbose, skip_patterns, None) +} + +/// [`index_project_with_skip`] restricted to files for which `include` +/// (called with the root-relative path) returns true. Files that fail the +/// predicate are neither read nor re-indexed, but still count as present on +/// disk, so their stored rows are not pruned. `None` means every file. +pub fn index_project_filtered( + conn: &Connection, + root: &Path, + verbose: bool, + skip_patterns: Option<&[String]>, + include: Option<&dyn Fn(&str) -> bool>, ) -> anyhow::Result { let project_id = ensure_project(conn, root)?; @@ -366,7 +380,7 @@ pub fn index_project_with_skip( } } - index_project_with_id(conn, project_id, root, verbose, skip_patterns) + index_project_with_id(conn, project_id, root, verbose, skip_patterns, include) } /// Internal: index a project with an already-resolved `project_id`. @@ -376,6 +390,7 @@ fn index_project_with_id( root: &Path, verbose: bool, skip_patterns: Option<&[String]>, + include: Option<&dyn Fn(&str) -> bool>, ) -> anyhow::Result { let mut stats = IndexStats::default(); let skip_matcher = skip_patterns.map(crate::engine::path_match::PathMatcher::new); @@ -421,6 +436,11 @@ fn index_project_with_id( stats.files_scanned += 1; walked.insert(rel_str.clone()); + // Caller-supplied restriction (watch --filter / changed files only). + if include.is_some_and(|f| !f(&rel_str)) { + continue; + } + // Compute mtime:size fingerprint — cheap, no file read needed. // metadata() is a stat() call, ~microseconds per file. let metadata = match std::fs::metadata(path) { diff --git a/src/index/session.rs b/src/index/session.rs index f4a3571..804a086 100644 --- a/src/index/session.rs +++ b/src/index/session.rs @@ -146,6 +146,22 @@ impl IndexSession { super::index_project_with_skip(&self.conn, &self.root, verbose, self.skip_patterns()) } + /// Incremental index limited to files `include` accepts (root-relative + /// path). Skip patterns still apply; unrelated stored rows are kept. + pub fn index_matching( + &self, + verbose: bool, + include: &dyn Fn(&str) -> bool, + ) -> Result { + super::index_project_filtered( + &self.conn, + &self.root, + verbose, + self.skip_patterns(), + Some(include), + ) + } + /// Stored totals for this project. pub fn summary(&self) -> Result { super::index_stats(&self.conn, self.project_id)