From 5e80eaa25da5a726bf116147219cba6ce0155668 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 16:46:30 +0800 Subject: [PATCH 1/5] feat(scan): follow symlinked files and directories (#935) --- crates/codegraph-extract/src/engine.rs | 298 +++++++++++----- crates/codegraph-extract/src/lib.rs | 1 + crates/codegraph-extract/src/links.rs | 124 +++++++ .../codegraph-extract/tests/scan_symlinks.rs | 331 ++++++++++++++++++ 4 files changed, 670 insertions(+), 84 deletions(-) create mode 100644 crates/codegraph-extract/src/links.rs create mode 100644 crates/codegraph-extract/tests/scan_symlinks.rs diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index a40b808..82a1490 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -17,6 +17,7 @@ use std::time::Instant; use tree_sitter::Parser; use crate::ext_config::ExtensionOverrides; +use crate::links::LinkWalk; use crate::lang::{cpp_code_mask, spec_for_language}; use crate::walker::TreeSitterWalker; use codegraph_core::source_file::{SourceText, read_source_file}; @@ -466,6 +467,29 @@ pub fn extract_project( pub struct ScanProjectResult { pub files: Vec, pub unsupported_by_extension: BTreeMap, + /// Every symlink the scan followed, sorted by logical path (#935). + pub links: Vec, + /// Logical path of every directory scanned through a symlink, the links + /// themselves included, sorted. The watcher watches only these below a link. + pub linked_dirs: Vec, + /// For each file link whose target sits in a scanned directory: the + /// target's logical path → the link paths that alias it, sorted. + pub file_aliases: BTreeMap>, +} + +/// A symlink the scan followed: its logical path, the canonical path it +/// resolved to, and whether it named a file or a directory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FollowedLink { + pub relative: String, + pub canonical: PathBuf, + pub kind: LinkKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum LinkKind { + File, + Dir, } pub fn scan_project(root: &Path, options: &ExtractOptions) -> Result> { @@ -473,8 +497,6 @@ pub fn scan_project(root: &Path, options: &ExtractOptions) -> Result } pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result { - let mut files = Vec::new(); - let mut unsupported_by_extension = BTreeMap::new(); let ignored_dirs = options .ignore_dirs .iter() @@ -493,98 +515,206 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< root, std::env::var("CODEGRAPH_DIR").ok().as_deref(), ); - scan_dir( - root, + // No symlink may lead into the project's `.git` or a reserved index root. + let blocked = std::iter::once(root.join(".git")).chain(reserved_roots.iter().cloned()); + let mut walk = ScanWalk { root, - &ignored_dirs, - &reserved_roots, - &pattern_sets, - (&gitignore, GitignoreAncestry::default()), - &include, - &options.extensions, - &mut files, - &mut unsupported_by_extension, - )?; - files.sort(); - Ok(ScanProjectResult { - files, - unsupported_by_extension, - }) + ignored_dirs: &ignored_dirs, + reserved_roots: &reserved_roots, + pattern_sets: &pattern_sets, + gitignore: &gitignore, + include: &include, + overrides: &options.extensions, + links: LinkWalk::new(root, blocked), + files: Vec::new(), + unsupported_by_extension: BTreeMap::new(), + followed: Vec::new(), + linked_dirs: Vec::new(), + file_links: Vec::new(), + }; + let canonical_root = walk + .links + .canonical_root() + .map_or_else(|| root.to_path_buf(), Path::to_path_buf); + walk.links.enter(canonical_root.clone(), ""); + walk.scan_dir(root, &canonical_root, GitignoreAncestry::default(), 0)?; + // Symlinked directories, fewest hops first, then by logical path (#935). + while let Some(linked) = walk.links.next() { + walk.followed.push(FollowedLink { + relative: linked.relative.clone(), + canonical: linked.canonical.clone(), + kind: LinkKind::Dir, + }); + walk.linked_dirs.push(linked.relative); + walk.scan_dir(&linked.path, &linked.canonical, linked.state, linked.hops)?; + } + Ok(walk.finish()) } -#[allow(clippy::too_many_arguments)] -fn scan_dir( - root: &Path, - dir: &Path, - ignored_dirs: &HashSet<&str>, - reserved_roots: &std::collections::BTreeSet, - pattern_sets: &[&[String]], - (gitignore, above): (&RootGitignore, GitignoreAncestry), - include: &IncludeSet, - overrides: &ExtensionOverrides, - files: &mut Vec, - unsupported_by_extension: &mut BTreeMap, -) -> Result<()> { - let entries = fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))?; - for entry in entries { - let entry = entry?; - let path = entry.path(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - // Skip `.git` (a direct child of the scan root) and any directory whose - // FULL path is a resolved reserved index root — matched at any depth, so - // a nested configured root like `/cache/index` is pruned exactly, - // while a same-basename user directory elsewhere is not. - let is_reserved_root_here = - (dir == root && name == ".git") || reserved_roots.contains(&path); - let relative = normalize_path(path.strip_prefix(root).unwrap_or(&path)); - // `build` is also a legal JVM package segment: keep it under a - // conventional source root while still pruning build output (#1642). - let jvm_package = name == "build" - && codegraph_core::config::is_jvm_source_build_dir(&relative) - && entry.file_type().is_ok_and(|kind| kind.is_dir()); - if is_reserved_root_here || (ignored_dirs.contains(name.as_ref()) && !jvm_package) { - continue; - } - let file_type = entry.file_type()?; - let own = gitignore.matched(&relative, file_type.is_dir()); - let ignored = is_path_ignored(&relative, pattern_sets, above.decide(own)); - if file_type.is_dir() { - // A model-ignored dir is normally pruned before descent, so a FILE - // include under a gitignored ancestor would never be reached. - // Descend anyway when this dir is an ancestor of (or matches) an - // include pattern; files inside are still pruned unless force-included. - if ignored && !include.wants_descend(&relative) { +/// One scan's fixed inputs and accumulated output. +struct ScanWalk<'a> { + root: &'a Path, + ignored_dirs: &'a HashSet<&'a str>, + reserved_roots: &'a std::collections::BTreeSet, + pattern_sets: &'a [&'a [String]], + gitignore: &'a RootGitignore, + include: &'a IncludeSet<'a>, + overrides: &'a ExtensionOverrides, + links: LinkWalk, + files: Vec, + unsupported_by_extension: BTreeMap, + followed: Vec, + linked_dirs: Vec, + /// Indexed file links: logical path → canonical target. + file_links: Vec<(String, PathBuf)>, +} + +impl ScanWalk<'_> { + /// Scan `dir`, whose canonical path is `canonical_dir`, reached through + /// `hops` symlinks. Real subdirectories are scanned at once; a symlinked + /// one is queued for the next hop level. + fn scan_dir( + &mut self, + dir: &Path, + canonical_dir: &Path, + above: GitignoreAncestry, + hops: usize, + ) -> Result<()> { + let entries = fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))?; + for entry in entries { + let entry = entry?; + let path = entry.path(); + let file_name = entry.file_name(); + let name = file_name.to_string_lossy(); + // Skip `.git` (a direct child of the scan root) and any directory whose + // FULL path is a resolved reserved index root — matched at any depth, so + // a nested configured root like `/cache/index` is pruned exactly, + // while a same-basename user directory elsewhere is not. + let is_reserved_root_here = + (dir == self.root && name == ".git") || self.reserved_roots.contains(&path); + let relative = normalize_path(path.strip_prefix(self.root).unwrap_or(&path)); + // `build` is also a legal JVM package segment: keep it under a + // conventional source root while still pruning build output (#1642). + let jvm_package = name == "build" + && codegraph_core::config::is_jvm_source_build_dir(&relative) + && resolves_to_dir(&entry, &path); + if is_reserved_root_here || (self.ignored_dirs.contains(name.as_ref()) && !jvm_package) + { continue; } - scan_dir( - root, - &path, - ignored_dirs, - reserved_roots, - pattern_sets, - (gitignore, above.child(own)), - include, - overrides, - files, - unsupported_by_extension, - )?; - } else if file_type.is_file() { - // Post-model include decision: a model-ignored file is force-included - // iff it matches `include` and is NOT overridden by an explicit - // `exclude` (checked inside `IncludeSet::forces`). Built-in dir skips - // are already handled structurally above, so include can never - // resurface node_modules/dist/.git/etc. - if !ignored || include.forces(&relative) { - if is_extractable_source_path(&relative, overrides) { - files.push(relative); + let file_type = entry.file_type()?; + // A symlink is judged at its logical path by what it resolves to + // (#935). A broken one, or one to anything but a file or a + // directory, is skipped. + let (is_dir, is_link) = if file_type.is_symlink() { + match fs::metadata(&path) { + Ok(target) if target.is_dir() => (true, true), + Ok(target) if target.is_file() => (false, true), + _ => continue, + } + } else if file_type.is_dir() { + (true, false) + } else if file_type.is_file() { + (false, false) + } else { + continue; + }; + let own = self.gitignore.matched(&relative, is_dir); + let ignored = is_path_ignored(&relative, self.pattern_sets, above.decide(own)); + if is_dir { + // A model-ignored dir is normally pruned before descent, so a FILE + // include under a gitignored ancestor would never be reached. + // Descend anyway when this dir is an ancestor of (or matches) an + // include pattern; files inside are still pruned unless force-included. + if ignored && !self.include.wants_descend(&relative) { + continue; + } + if is_link { + self.links + .queue(hops + 1, relative, path, above.child(own)); + continue; + } + let canonical = canonical_dir.join(&file_name); + if !self.links.enter(canonical.clone(), &relative) { + continue; + } + if hops > 0 { + self.linked_dirs.push(relative); + } + self.scan_dir(&path, &canonical, above.child(own), hops)?; + } else if !ignored || self.include.forces(&relative) { + // Post-model include decision: a model-ignored file is force-included + // iff it matches `include` and is NOT overridden by an explicit + // `exclude` (checked inside `IncludeSet::forces`). Built-in dir skips + // are already handled structurally above, so include can never + // resurface node_modules/dist/.git/etc. + let target = if is_link { + match self.links.file_target(&path) { + Some(target) => Some(target), + None => continue, + } + } else { + None + }; + if is_extractable_source_path(&relative, self.overrides) { + if let Some(target) = target { + self.followed.push(FollowedLink { + relative: relative.clone(), + canonical: target.clone(), + kind: LinkKind::File, + }); + self.file_links.push((relative.clone(), target)); + } + self.files.push(relative); } else if let Some(extension) = unsupported_extension(&relative) { - *unsupported_by_extension.entry(extension).or_default() += 1; + *self.unsupported_by_extension.entry(extension).or_default() += 1; } } } + Ok(()) + } + + fn finish(mut self) -> ScanProjectResult { + self.files.sort(); + self.followed + .sort_by(|left, right| left.relative.cmp(&right.relative)); + self.linked_dirs.sort(); + // A file link aliases its target's logical path when the target sits + // in a scanned directory, so an edit there can re-index the link too. + let mut file_aliases = BTreeMap::>::new(); + for (alias, target) in self.file_links { + let (Some(parent), Some(name)) = (target.parent(), target.file_name()) else { + continue; + }; + let Some(dir) = self.links.logical_dir(parent) else { + continue; + }; + let name = name.to_string_lossy(); + let primary = if dir.is_empty() { + name.into_owned() + } else { + format!("{dir}/{name}") + }; + file_aliases.entry(primary).or_default().push(alias); + } + for aliases in file_aliases.values_mut() { + aliases.sort(); + } + ScanProjectResult { + files: self.files, + unsupported_by_extension: self.unsupported_by_extension, + links: self.followed, + linked_dirs: self.linked_dirs, + file_aliases, + } } - Ok(()) +} + +/// Whether a directory entry is a directory, or a symlink to one. +fn resolves_to_dir(entry: &fs::DirEntry, path: &Path) -> bool { + entry.file_type().is_ok_and(|kind| { + kind.is_dir() || (kind.is_symlink() && fs::metadata(path).is_ok_and(|meta| meta.is_dir())) + }) } fn unsupported_extension(relative: &str) -> Option { diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index 4472355..dcbc261 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -13,6 +13,7 @@ pub mod engine; pub mod ext_config; pub mod function_ref; pub mod lang; +mod links; pub mod spec; pub mod walker; diff --git a/crates/codegraph-extract/src/links.rs b/crates/codegraph-extract/src/links.rs new file mode 100644 index 0000000..b2f5240 --- /dev/null +++ b/crates/codegraph-extract/src/links.rs @@ -0,0 +1,124 @@ +//! Deterministic symlink following for the project scan (upstream #935). +//! +//! The project's real tree is scanned first. A symlinked directory met during +//! a walk is queued under `(hops, logical path)` and followed only after every +//! walk with fewer hops, so a canonical directory is scanned once, under the +//! logical path that reaches it through the fewest symlinks, ties going to the +//! lexicographically smallest path. Within one hop level no queued link is a +//! logical prefix of another, so popping the queue in order realizes exactly +//! that rule, and the outcome depends on the filesystem alone, never on +//! `read_dir` order. +//! +//! A link is never followed to the project root or one of its ancestors (that +//! would rescan the project, or the tree above it, under an alias), nor into +//! the canonical `.git` or a reserved index root. A target that cannot be +//! canonicalized or read is skipped like a broken link, so a stray link never +//! fails an index. + +use std::collections::hash_map::Entry; +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; + +/// A symlinked directory to scan next, already recorded as scanned. +pub(crate) struct LinkedDir { + pub(crate) hops: usize, + pub(crate) relative: String, + pub(crate) path: PathBuf, + pub(crate) canonical: PathBuf, + pub(crate) state: T, +} + +pub(crate) struct LinkWalk { + canonical_root: Option, + blocked: Vec, + /// Canonical path of every scanned directory → the logical path it was + /// scanned under. + scanned: HashMap, + queue: BTreeMap<(usize, String), (PathBuf, T)>, +} + +impl LinkWalk { + /// `blocked` names the directories no link may lead into. Those that do + /// not exist are dropped; the rest are compared canonically. + pub(crate) fn new(root: &Path, blocked: impl IntoIterator) -> Self { + Self { + canonical_root: fs::canonicalize(root).ok(), + blocked: blocked + .into_iter() + .filter_map(|path| fs::canonicalize(path).ok()) + .collect(), + scanned: HashMap::new(), + queue: BTreeMap::new(), + } + } + + /// The project root's canonical path. `None` when it cannot be resolved, + /// and then no link is ever followed. + pub(crate) fn canonical_root(&self) -> Option<&Path> { + self.canonical_root.as_deref() + } + + /// Record a directory about to be scanned under `relative`. `false` when + /// its canonical path was scanned already or lies in a blocked prefix, and + /// the directory is then skipped. + pub(crate) fn enter(&mut self, canonical: PathBuf, relative: &str) -> bool { + if self.is_blocked(&canonical) { + return false; + } + match self.scanned.entry(canonical) { + Entry::Occupied(_) => false, + Entry::Vacant(slot) => { + slot.insert(relative.to_string()); + true + } + } + } + + pub(crate) fn queue(&mut self, hops: usize, relative: String, path: PathBuf, state: T) { + self.queue.entry((hops, relative)).or_insert((path, state)); + } + + /// The next queued directory link to follow. + pub(crate) fn next(&mut self) -> Option> { + let root = self.canonical_root.clone()?; + while let Some(((hops, relative), (path, state))) = self.queue.pop_first() { + let Ok(canonical) = fs::canonicalize(&path) else { + continue; + }; + if root.starts_with(&canonical) || fs::read_dir(&path).is_err() { + continue; + } + if !self.enter(canonical.clone(), &relative) { + continue; + } + return Some(LinkedDir { + hops, + relative, + path, + canonical, + state, + }); + } + None + } + + /// The canonical target of a file link, unless it cannot be resolved or + /// lies in a blocked prefix. + pub(crate) fn file_target(&self, path: &Path) -> Option { + self.canonical_root.as_ref()?; + let canonical = fs::canonicalize(path).ok()?; + (!self.is_blocked(&canonical)).then_some(canonical) + } + + /// The logical path a canonical directory was scanned under. + pub(crate) fn logical_dir(&self, canonical: &Path) -> Option<&str> { + self.scanned.get(canonical).map(String::as_str) + } + + fn is_blocked(&self, canonical: &Path) -> bool { + self.blocked + .iter() + .any(|blocked| canonical.starts_with(blocked)) + } +} diff --git a/crates/codegraph-extract/tests/scan_symlinks.rs b/crates/codegraph-extract/tests/scan_symlinks.rs new file mode 100644 index 0000000..5b201b2 --- /dev/null +++ b/crates/codegraph-extract/tests/scan_symlinks.rs @@ -0,0 +1,331 @@ +//! The project scan follows symlinked files and directories (upstream #935). +//! +//! Precedence is deterministic: a canonical directory is scanned once, under +//! the logical path that reaches it through the fewest symlinks, ties going to +//! the lexicographically smallest path. Links to the project root or one of its +//! ancestors, into `.git` or a reserved index root, broken links, and +//! unreadable targets are not followed. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::SystemTime; + +use codegraph_extract::engine::{LinkKind, ScanProjectResult, scan_project_with_stats}; +use codegraph_extract::ExtractOptions; + +fn sandbox(tag: &str) -> PathBuf { + static COUNTER: AtomicU32 = AtomicU32::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "cg_scan_links_{tag}_{}_{nanos}_{n}", + std::process::id() + )); + fs::create_dir_all(&dir).expect("create sandbox"); + dir +} + +fn touch(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).expect("create parent dirs"); + fs::write(&path, contents).expect("write file"); +} + +/// Create a symlink. Without the privilege (Windows) a local run skips the +/// test, but CI must exercise it, so there a refusal fails loudly. +fn link(target: &Path, link: &Path, dir: bool) -> bool { + fs::create_dir_all(link.parent().unwrap()).expect("create link parent"); + #[cfg(unix)] + let made = { + let _ = dir; + std::os::unix::fs::symlink(target, link) + }; + #[cfg(windows)] + let made = if dir { + std::os::windows::fs::symlink_dir(target, link) + } else { + std::os::windows::fs::symlink_file(target, link) + }; + match made { + Ok(()) => true, + Err(error) if std::env::var_os("CI").is_some() => { + panic!("CI must be able to create symlinks: {error}") + } + Err(_) => false, + } +} + +fn link_dir(target: &Path, at: &Path) -> bool { + link(target, at, true) +} + +fn link_file(target: &Path, at: &Path) -> bool { + link(target, at, false) +} + +fn scan(project: &Path) -> ScanProjectResult { + scan_project_with_stats(project, &ExtractOptions::default()).expect("scan") +} + +fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_string()).collect() +} + +fn link_summary(result: &ScanProjectResult) -> Vec<(String, LinkKind)> { + result + .links + .iter() + .map(|link| (link.relative.clone(), link.kind)) + .collect() +} + +#[test] +fn file_links_and_out_of_root_directory_links_index_at_their_logical_paths() { + let root = sandbox("basic"); + let project = root.join("proj"); + let outside = root.join("outside"); + touch(&project, "src/real/a.ts", "export const a = 1;"); + touch(&outside, "lib/out.ts", "export const out = 2;"); + touch(&outside, "lib/deep/inner.ts", "export const inner = 3;"); + if !link_file(&project.join("src/real/a.ts"), &project.join("src/afile.ts")) + || !link_dir(&outside.join("lib"), &project.join("src/extlink")) + { + return; + } + + let result = scan(&project); + assert_eq!( + result.files, + strings(&[ + "src/afile.ts", + "src/extlink/deep/inner.ts", + "src/extlink/out.ts", + "src/real/a.ts", + ]) + ); + assert_eq!( + link_summary(&result), + vec![ + ("src/afile.ts".to_string(), LinkKind::File), + ("src/extlink".to_string(), LinkKind::Dir), + ] + ); + let extlink = result + .links + .iter() + .find(|link| link.relative == "src/extlink") + .unwrap(); + assert_eq!(extlink.canonical, outside.join("lib").canonicalize().unwrap()); + assert_eq!( + result.linked_dirs, + strings(&["src/extlink", "src/extlink/deep"]) + ); + fs::remove_dir_all(&root).ok(); +} + +#[test] +fn the_real_tree_wins_and_the_smallest_logical_path_breaks_ties() { + let root = sandbox("precedence"); + let project = root.join("proj"); + let outside = root.join("outside"); + touch(&project, "src/real/a.ts", "export const a = 1;"); + touch(&outside, "x.ts", "export const x = 1;"); + // `src/alias` sorts before `src/real` but cannot displace the real tree; + // `a/ext` and `b/ext` reach one target and `a/ext` sorts first. + if !link_dir(&project.join("src/real"), &project.join("src/alias")) + || !link_dir(&outside, &project.join("b/ext")) + || !link_dir(&outside, &project.join("a/ext")) + { + return; + } + + let result = scan(&project); + assert_eq!(result.files, strings(&["a/ext/x.ts", "src/real/a.ts"])); + assert_eq!( + link_summary(&result), + vec![("a/ext".to_string(), LinkKind::Dir)] + ); + fs::remove_dir_all(&root).ok(); +} + +/// The round-1 review counterexample: `a -> A`, `z -> B`, `A/x -> B/sub`, and +/// a real `B/sub`. `B/sub` is one hop away through `z` but two through +/// `a/x`, so it is scanned as `z/sub`, and only once. +#[test] +fn fewer_symlink_hops_win_over_an_earlier_logical_path() { + let root = sandbox("hops"); + let project = root.join("proj"); + let a = root.join("A"); + let b = root.join("B"); + fs::create_dir_all(&project).unwrap(); + touch(&a, "a.ts", "export const a = 1;"); + touch(&b, "sub/s.ts", "export const s = 1;"); + if !link_dir(&a, &project.join("a")) + || !link_dir(&b, &project.join("z")) + || !link_dir(&b.join("sub"), &a.join("x")) + { + return; + } + + let result = scan(&project); + assert_eq!(result.files, strings(&["a/a.ts", "z/sub/s.ts"])); + assert_eq!( + link_summary(&result), + vec![ + ("a".to_string(), LinkKind::Dir), + ("z".to_string(), LinkKind::Dir), + ] + ); + assert_eq!(result.linked_dirs, strings(&["a", "z", "z/sub"])); + fs::remove_dir_all(&root).ok(); +} + +#[test] +fn roots_ancestors_git_index_roots_and_broken_links_are_not_followed() { + let root = sandbox("skipped"); + let project = root.join("proj"); + touch(&project, "src/app.ts", "export const app = 1;"); + touch(&project, ".codegraph/stray.ts", "export const stray = 1;"); + touch(&project, ".git/hooks/hook.ts", "export const hook = 1;"); + touch(&root, "sibling.ts", "export const sibling = 1;"); + if !link_dir(&project, &project.join("src/loop")) + || !link_dir(&root, &project.join("up")) + || !link_dir(&project.join(".codegraph"), &project.join("cg")) + || !link_dir(&project.join(".git"), &project.join("gitdir")) + || !link_dir(&project.join(".git/hooks"), &project.join("hooks")) + || !link_file(&project.join(".git/hooks/hook.ts"), &project.join("hook.ts")) + || !link_dir(&root.join("missing"), &project.join("broken")) + { + return; + } + + let result = scan(&project); + assert_eq!(result.files, strings(&["src/app.ts"])); + assert!(result.links.is_empty(), "{:?}", result.links); + fs::remove_dir_all(&root).ok(); +} + +#[test] +fn ignore_rules_judge_a_link_at_its_logical_path_and_nested_links_follow() { + let root = sandbox("rules"); + let project = root.join("proj"); + let outside = root.join("outside"); + let inner = root.join("inner"); + touch(&project, ".gitignore", "hidden/\n"); + touch(&outside, "o.ts", "export const o = 1;"); + touch(&inner, "i.ts", "export const i = 1;"); + if !link_dir(&outside, &project.join("node_modules")) + || !link_dir(&outside, &project.join("hidden")) + || !link_dir(&outside, &project.join("linked")) + || !link_dir(&inner, &outside.join("deeper")) + { + return; + } + + let result = scan(&project); + assert_eq!( + result.files, + strings(&["linked/deeper/i.ts", "linked/o.ts"]), + "an ignored link name or path is pruned; a link inside a followed target is followed" + ); + assert_eq!( + link_summary(&result), + vec![ + ("linked".to_string(), LinkKind::Dir), + ("linked/deeper".to_string(), LinkKind::Dir), + ] + ); + fs::remove_dir_all(&root).ok(); +} + +#[test] +fn the_outcome_does_not_depend_on_link_creation_order() { + let mut outcomes = Vec::new(); + for order in [[0, 1, 2], [2, 1, 0]] { + let root = sandbox("order"); + let project = root.join("proj"); + let shared = root.join("shared"); + touch(&shared, "s.ts", "export const s = 1;"); + touch(&project, "m.ts", "export const m = 1;"); + let names = ["c/l", "a/l", "b/l"]; + for index in order { + if !link_dir(&shared, &project.join(names[index])) { + return; + } + } + let result = scan(&project); + let summary = link_summary(&result); + outcomes.push((result.files, summary)); + fs::remove_dir_all(&root).ok(); + } + assert_eq!(outcomes[0], outcomes[1]); + assert_eq!(outcomes[0].0, strings(&["a/l/s.ts", "m.ts"])); +} + +#[test] +fn file_aliases_map_an_indexed_target_to_its_link_paths() { + let root = sandbox("aliases"); + let project = root.join("proj"); + let outside = root.join("outside"); + touch(&project, "src/real/a.ts", "export const a = 1;"); + touch(&outside, "lib/b.ts", "export const b = 1;"); + touch(&root, "lonely/c.ts", "export const c = 1;"); + if !link_file(&project.join("src/real/a.ts"), &project.join("src/afile.ts")) + || !link_file(&project.join("src/real/a.ts"), &project.join("other/a2.ts")) + || !link_dir(&outside.join("lib"), &project.join("ext")) + || !link_file(&outside.join("lib/b.ts"), &project.join("bfile.ts")) + || !link_file(&root.join("lonely/c.ts"), &project.join("cfile.ts")) + { + return; + } + + let result = scan(&project); + let aliases: Vec<(String, Vec)> = result + .file_aliases + .iter() + .map(|(target, links)| (target.clone(), links.clone())) + .collect(); + assert_eq!( + aliases, + vec![ + ("ext/b.ts".to_string(), strings(&["bfile.ts"])), + ( + "src/real/a.ts".to_string(), + strings(&["other/a2.ts", "src/afile.ts"]) + ), + ], + "a target outside every scanned directory has no alias entry" + ); + assert!(result.files.contains(&"cfile.ts".to_string())); + fs::remove_dir_all(&root).ok(); +} + +#[cfg(unix)] +#[test] +fn an_unreadable_link_target_is_skipped_without_failing_the_scan() { + use std::os::unix::fs::PermissionsExt; + + let root = sandbox("unreadable"); + let project = root.join("proj"); + let locked = root.join("locked"); + touch(&project, "src/app.ts", "export const app = 1;"); + touch(&locked, "secret.ts", "export const secret = 1;"); + assert!(link_dir(&locked, &project.join("locked"))); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::read_dir(&locked).is_ok() { + // Running as root: permissions do not apply, nothing to prove. + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap(); + fs::remove_dir_all(&root).ok(); + return; + } + + let result = scan(&project); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(result.files, strings(&["src/app.ts"])); + assert!(result.links.is_empty()); + fs::remove_dir_all(&root).ok(); +} From 9294ab85a611a9fd1353417e43b7a82e9f9a9424 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 16:54:43 +0800 Subject: [PATCH 2/5] fix(sync): re-read files behind new, retargeted or unrecorded symlinks --- Cargo.lock | 2 + crates/codegraph-cli/Cargo.toml | 1 + crates/codegraph-cli/src/main.rs | 3 + crates/codegraph-cli/tests/symlink_sync.rs | 314 ++++++++++++++++++ crates/codegraph-extract/src/engine.rs | 5 +- .../codegraph-extract/tests/scan_symlinks.rs | 24 +- crates/codegraph-watch/Cargo.toml | 1 + crates/codegraph-watch/src/lib.rs | 2 + crates/codegraph-watch/src/link_state.rs | 186 +++++++++++ crates/codegraph-watch/src/migrate.rs | 4 +- crates/codegraph-watch/src/sync.rs | 24 +- 11 files changed, 552 insertions(+), 14 deletions(-) create mode 100644 crates/codegraph-cli/tests/symlink_sync.rs create mode 100644 crates/codegraph-watch/src/link_state.rs diff --git a/Cargo.lock b/Cargo.lock index 02f2792..d4a03f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,6 +617,7 @@ dependencies = [ "rayon", "regex", "rstest", + "rusqlite", "self_update", "serde", "serde_json", @@ -653,6 +654,7 @@ dependencies = [ "notify", "rusqlite", "serde", + "serde_json", "windows-sys 0.52.0", ] diff --git a/crates/codegraph-cli/Cargo.toml b/crates/codegraph-cli/Cargo.toml index a9266cf..7e7170a 100644 --- a/crates/codegraph-cli/Cargo.toml +++ b/crates/codegraph-cli/Cargo.toml @@ -44,6 +44,7 @@ tracing-subscriber = { workspace = true } time = { workspace = true } [dev-dependencies] +rusqlite = { workspace = true } codegraph-bench = { path = "../codegraph-bench" } # Feature-unifies the Store's lease barrier into `CARGO_BIN_EXE_codegraph` only # while this package's test targets are built. A normal binary build uses the diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index 65ca747..e355e9d 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -5289,6 +5289,8 @@ fn index_project_inner( }); top_unsupported_extensions.truncate(5); let files = scan.files; + // Recorded with the index so a later sync can tell a retargeted link (#935). + let followed_links = scan.links; let scan_duration = scan_started.elapsed(); let mut diagnostic_run = DiagnosticRun::start( project, @@ -5671,6 +5673,7 @@ fn index_project_inner( json!({}), ); store.set_project_metadata("indexed_with_version", VERSION)?; + codegraph_watch::record_followed_links(&store, &followed_links)?; let after = store.counts()?; // Explicit fallible finalization: pragma restore -> checkpoint + compaction -> // extraction stamp -> stamp checkpoint -> close the final connection -> diff --git a/crates/codegraph-cli/tests/symlink_sync.rs b/crates/codegraph-cli/tests/symlink_sync.rs new file mode 100644 index 0000000..316607b --- /dev/null +++ b/crates/codegraph-cli/tests/symlink_sync.rs @@ -0,0 +1,314 @@ +//! `sync` stays equal to `index --force` when the project is indexed through +//! symlinks (upstream #935). +//! +//! The scan follows symlinked files and directories, so retargeting a link can +//! point an indexed logical path at a different file with the same size and +//! mtime. The stat pre-filter must not keep the old graph then: every full +//! build records the links it followed, and a full sync re-reads whatever a +//! new, retargeted or unrecorded link reaches. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use codegraph_core::IndexPaths; +use codegraph_store::Store; + +struct TestDir { + path: PathBuf, +} + +impl TestDir { + fn new(label: &str) -> Self { + let path = std::env::temp_dir().join(format!( + "codegraph-cli-symlink-sync-{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&path).unwrap(); + Self { path } + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn cli(args: &[&str]) -> String { + let output = Command::new(env!("CARGO_BIN_EXE_codegraph")) + .args(args) + .env("CODEGRAPH_NO_DAEMON", "1") + .output() + .expect("run codegraph binary"); + assert!( + output.status.success(), + "codegraph {args:?} failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn touch(root: &Path, relative: &str, contents: &str) -> PathBuf { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, contents).unwrap(); + path +} + +/// Create a symlink. Without the privilege (Windows) a local run skips the +/// test, but CI must exercise it, so there a refusal fails loudly. +fn link(target: &Path, at: &Path, dir: bool) -> bool { + fs::create_dir_all(at.parent().unwrap()).unwrap(); + #[cfg(unix)] + let made = { + let _ = dir; + std::os::unix::fs::symlink(target, at) + }; + #[cfg(windows)] + let made = if dir { + std::os::windows::fs::symlink_dir(target, at) + } else { + std::os::windows::fs::symlink_file(target, at) + }; + match made { + Ok(()) => true, + Err(error) if std::env::var_os("CI").is_some() => { + panic!("CI must be able to create symlinks: {error}") + } + Err(_) => false, + } +} + +fn remove_link(at: &Path) { + // A directory symlink is a directory entry on Windows, a file on Unix. + if fs::remove_file(at).is_err() { + fs::remove_dir(at).expect("remove link"); + } +} + +/// Give `path` exactly `like`'s modification time. +fn copy_mtime(like: &Path, path: &Path) { + let modified = fs::metadata(like).unwrap().modified().unwrap(); + fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(modified) + .unwrap(); +} + +fn store(project: &Path) -> Store { + let paths = IndexPaths::resolve(project, None).expect("resolve index paths"); + Store::open_for_read(&paths, Instant::now() + Duration::from_secs(30), || false) + .expect("open the index for reading") +} + +/// Files (path, hash) and nodes (file, kind, name, line), sorted. +fn snapshot(project: &Path) -> (Vec<(String, String)>, Vec<(String, String, String, i64)>) { + let store = store(project); + let mut files: Vec<(String, String)> = store + .all_files() + .unwrap() + .into_iter() + .map(|file| (file.path, file.content_hash)) + .collect(); + files.sort(); + let mut nodes: Vec<(String, String, String, i64)> = store + .all_nodes() + .unwrap() + .into_iter() + .map(|node| { + ( + node.file_path, + format!("{:?}", node.kind), + node.name, + node.start_line, + ) + }) + .collect(); + nodes.sort(); + (files, nodes) +} + +fn has_symbol(project: &Path, name: &str) -> bool { + snapshot(project).1.iter().any(|node| node.2 == name) +} + +fn modified_paths(project: &str) -> Vec { + let value: serde_json::Value = + serde_json::from_str(&cli(&["status", project, "--json"])).expect("status JSON"); + value["pendingChanges"]["modifiedPaths"] + .as_array() + .expect("modifiedPaths") + .iter() + .map(|path| path.as_str().unwrap().to_string()) + .collect() +} + +/// `sync` then `index --force` must leave identical graphs. +fn assert_sync_equals_index_force(project: &Path) { + let p = project.to_str().unwrap(); + cli(&["sync", p]); + let synced = snapshot(project); + cli(&["index", "--force", p]); + assert_eq!(synced, snapshot(project), "sync must equal index --force"); +} + +#[test] +fn init_indexes_through_links_and_a_removed_link_leaves_the_index() { + let dir = TestDir::new("basic"); + let project = dir.path.join("proj"); + let outside = dir.path.join("outside"); + touch( + &project, + "src/app.ts", + "export function app() { return 1; }\n", + ); + touch( + &outside, + "lib/util.ts", + "export function util() { return 2; }\n", + ); + touch( + &project, + "src/real.ts", + "export function real() { return 3; }\n", + ); + if !link(&outside.join("lib"), &project.join("vendored"), true) + || !link( + &project.join("src/real.ts"), + &project.join("alias.ts"), + false, + ) + { + return; + } + let p = project.to_str().unwrap(); + cli(&["init", p]); + let files: Vec = snapshot(&project).0.into_iter().map(|f| f.0).collect(); + assert_eq!( + files, + vec!["alias.ts", "src/app.ts", "src/real.ts", "vendored/util.ts"] + ); + let recorded = store(&project) + .get_project_metadata(codegraph_watch::FOLLOWED_LINKS_KEY) + .unwrap() + .expect("a full build records the links it followed"); + assert!( + recorded.contains("vendored") && recorded.contains("alias.ts"), + "{recorded}" + ); + + remove_link(&project.join("vendored")); + assert_sync_equals_index_force(&project); + assert!( + !has_symbol(&project, "util"), + "a removed link's files leave the index" + ); +} + +#[test] +fn a_retargeted_directory_link_is_reread_despite_an_equal_stat() { + let dir = TestDir::new("retarget-dir"); + let project = dir.path.join("proj"); + let outside = dir.path.join("outside"); + touch(&project, "src/app.ts", "export const app = 1;\n"); + let old = touch(&outside, "v1/x.ts", "export const aa = 1;\n"); + let new = touch(&outside, "v2/x.ts", "export const bb = 1;\n"); + copy_mtime(&old, &new); + if !link(&outside.join("v1"), &project.join("lib"), true) { + return; + } + let p = project.to_str().unwrap(); + cli(&["init", p]); + assert!(has_symbol(&project, "aa")); + + remove_link(&project.join("lib")); + assert!(link(&outside.join("v2"), &project.join("lib"), true)); + assert_eq!(modified_paths(p), vec!["lib/x.ts".to_string()]); + cli(&["sync", p]); + assert!(has_symbol(&project, "bb") && !has_symbol(&project, "aa")); + assert_sync_equals_index_force(&project); +} + +#[test] +fn a_retargeted_file_link_is_reread_despite_an_equal_stat() { + let dir = TestDir::new("retarget-file"); + let project = dir.path.join("proj"); + let outside = dir.path.join("outside"); + touch(&project, "src/app.ts", "export const app = 1;\n"); + let old = touch(&outside, "f1.ts", "export const ff = 1;\n"); + let new = touch(&outside, "f2.ts", "export const gg = 1;\n"); + copy_mtime(&old, &new); + if !link(&old, &project.join("f.ts"), false) { + return; + } + let p = project.to_str().unwrap(); + cli(&["init", p]); + assert!(has_symbol(&project, "ff")); + + remove_link(&project.join("f.ts")); + assert!(link(&new, &project.join("f.ts"), false)); + assert_eq!(modified_paths(p), vec!["f.ts".to_string()]); + cli(&["sync", p]); + assert!(has_symbol(&project, "gg") && !has_symbol(&project, "ff")); + assert_sync_equals_index_force(&project); +} + +/// An index without the record — built before it existed — may track a +/// regular path that was later replaced by a link to a same-stat file. +#[test] +fn an_index_without_the_link_record_rereads_every_link() { + let dir = TestDir::new("no-record"); + let project = dir.path.join("proj"); + let other = dir.path.join("other"); + let x = touch(&project, "src/x.ts", "export const xx = 1;\n"); + let y = touch(&project, "lib/y.ts", "export const yy = 1;\n"); + let x2 = touch(&other, "x.ts", "export const x2 = 1;\n"); + let y2 = touch(&other, "lib/y.ts", "export const y2 = 1;\n"); + copy_mtime(&x, &x2); + copy_mtime(&y, &y2); + let p = project.to_str().unwrap(); + cli(&["init", p]); + + // Forget the record, as an index from before this change never wrote it. + let paths = IndexPaths::resolve(&project, None).unwrap(); + { + let db = rusqlite::Connection::open(paths.current_db()).unwrap(); + db.execute( + "DELETE FROM project_metadata WHERE key = ?1", + [codegraph_watch::FOLLOWED_LINKS_KEY], + ) + .unwrap(); + } + fs::remove_file(&x).unwrap(); + fs::remove_dir_all(project.join("lib")).unwrap(); + if !link(&x2, &project.join("src/x.ts"), false) + || !link(&other.join("lib"), &project.join("lib"), true) + { + return; + } + + assert_eq!( + modified_paths(p), + vec!["lib/y.ts".to_string(), "src/x.ts".to_string()] + ); + cli(&["sync", p]); + assert!(has_symbol(&project, "x2") && has_symbol(&project, "y2")); + assert!(!has_symbol(&project, "xx") && !has_symbol(&project, "yy")); + assert!( + store(&project) + .get_project_metadata(codegraph_watch::FOLLOWED_LINKS_KEY) + .unwrap() + .is_some(), + "the first full sync records the links" + ); + assert_sync_equals_index_force(&project); +} diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index 82a1490..ae0316c 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -17,8 +17,8 @@ use std::time::Instant; use tree_sitter::Parser; use crate::ext_config::ExtensionOverrides; -use crate::links::LinkWalk; use crate::lang::{cpp_code_mask, spec_for_language}; +use crate::links::LinkWalk; use crate::walker::TreeSitterWalker; use codegraph_core::source_file::{SourceText, read_source_file}; @@ -630,8 +630,7 @@ impl ScanWalk<'_> { continue; } if is_link { - self.links - .queue(hops + 1, relative, path, above.child(own)); + self.links.queue(hops + 1, relative, path, above.child(own)); continue; } let canonical = canonical_dir.join(&file_name); diff --git a/crates/codegraph-extract/tests/scan_symlinks.rs b/crates/codegraph-extract/tests/scan_symlinks.rs index 5b201b2..17f26c7 100644 --- a/crates/codegraph-extract/tests/scan_symlinks.rs +++ b/crates/codegraph-extract/tests/scan_symlinks.rs @@ -11,8 +11,8 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; use std::time::SystemTime; -use codegraph_extract::engine::{LinkKind, ScanProjectResult, scan_project_with_stats}; use codegraph_extract::ExtractOptions; +use codegraph_extract::engine::{LinkKind, ScanProjectResult, scan_project_with_stats}; fn sandbox(tag: &str) -> PathBuf { static COUNTER: AtomicU32 = AtomicU32::new(0); @@ -91,8 +91,10 @@ fn file_links_and_out_of_root_directory_links_index_at_their_logical_paths() { touch(&project, "src/real/a.ts", "export const a = 1;"); touch(&outside, "lib/out.ts", "export const out = 2;"); touch(&outside, "lib/deep/inner.ts", "export const inner = 3;"); - if !link_file(&project.join("src/real/a.ts"), &project.join("src/afile.ts")) - || !link_dir(&outside.join("lib"), &project.join("src/extlink")) + if !link_file( + &project.join("src/real/a.ts"), + &project.join("src/afile.ts"), + ) || !link_dir(&outside.join("lib"), &project.join("src/extlink")) { return; } @@ -119,7 +121,10 @@ fn file_links_and_out_of_root_directory_links_index_at_their_logical_paths() { .iter() .find(|link| link.relative == "src/extlink") .unwrap(); - assert_eq!(extlink.canonical, outside.join("lib").canonicalize().unwrap()); + assert_eq!( + extlink.canonical, + outside.join("lib").canonicalize().unwrap() + ); assert_eq!( result.linked_dirs, strings(&["src/extlink", "src/extlink/deep"]) @@ -197,7 +202,10 @@ fn roots_ancestors_git_index_roots_and_broken_links_are_not_followed() { || !link_dir(&project.join(".codegraph"), &project.join("cg")) || !link_dir(&project.join(".git"), &project.join("gitdir")) || !link_dir(&project.join(".git/hooks"), &project.join("hooks")) - || !link_file(&project.join(".git/hooks/hook.ts"), &project.join("hook.ts")) + || !link_file( + &project.join(".git/hooks/hook.ts"), + &project.join("hook.ts"), + ) || !link_dir(&root.join("missing"), &project.join("broken")) { return; @@ -274,8 +282,10 @@ fn file_aliases_map_an_indexed_target_to_its_link_paths() { touch(&project, "src/real/a.ts", "export const a = 1;"); touch(&outside, "lib/b.ts", "export const b = 1;"); touch(&root, "lonely/c.ts", "export const c = 1;"); - if !link_file(&project.join("src/real/a.ts"), &project.join("src/afile.ts")) - || !link_file(&project.join("src/real/a.ts"), &project.join("other/a2.ts")) + if !link_file( + &project.join("src/real/a.ts"), + &project.join("src/afile.ts"), + ) || !link_file(&project.join("src/real/a.ts"), &project.join("other/a2.ts")) || !link_dir(&outside.join("lib"), &project.join("ext")) || !link_file(&outside.join("lib/b.ts"), &project.join("bfile.ts")) || !link_file(&root.join("lonely/c.ts"), &project.join("cfile.ts")) diff --git a/crates/codegraph-watch/Cargo.toml b/crates/codegraph-watch/Cargo.toml index a349676..8d88a34 100644 --- a/crates/codegraph-watch/Cargo.toml +++ b/crates/codegraph-watch/Cargo.toml @@ -23,6 +23,7 @@ serde = { workspace = true } anyhow = { workspace = true } notify = { workspace = true } rusqlite = { workspace = true } +serde_json = { workspace = true } codegraph-core = { path = "../codegraph-core" } codegraph-extract = { path = "../codegraph-extract" } codegraph-resolve = { path = "../codegraph-resolve" } diff --git a/crates/codegraph-watch/src/lib.rs b/crates/codegraph-watch/src/lib.rs index 97080c2..0fe2526 100644 --- a/crates/codegraph-watch/src/lib.rs +++ b/crates/codegraph-watch/src/lib.rs @@ -1,4 +1,5 @@ mod git; +mod link_state; mod migrate; mod policy; mod sync; @@ -9,6 +10,7 @@ pub use git::{ DEFAULT_SYNC_HOOKS, GitHookName, GitHookResult, install_git_sync_hooks, is_git_repo, is_sync_hook_installed, remove_git_sync_hooks, }; +pub use link_state::{FOLLOWED_LINKS_KEY, record_followed_links}; pub use policy::{ CODEGRAPH_NO_WATCH, TooBroadRoot, WatchPolicy, too_broad_root_reason, watch_disabled_reason, }; diff --git a/crates/codegraph-watch/src/link_state.rs b/crates/codegraph-watch/src/link_state.rs new file mode 100644 index 0000000..717cd29 --- /dev/null +++ b/crates/codegraph-watch/src/link_state.rs @@ -0,0 +1,186 @@ +//! The symlinks an index was built through, and which files must be re-read. +//! +//! The scan follows symlinks (upstream #935), so retargeting a link can point +//! an indexed logical path at a different file with the same size and mtime. +//! Sync's stat pre-filter would then keep the old graph although +//! `index --force` reads the new target. Every full build therefore records +//! the links it followed under [`FOLLOWED_LINKS_KEY`], and a full sync (and +//! the pending inventory) bypasses the stat pre-filter at or under each +//! current link whose identity is new or changed. A missing or malformed +//! record means the identity is unknown — any index built before this record +//! existed, including one that tracked a regular path later replaced by a +//! link — so every current link is re-read until a full sync records it. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use codegraph_extract::engine::{FollowedLink, LinkKind}; +use codegraph_store::Store; + +/// `project_metadata` key holding the links the index was built through: a +/// JSON array of `[relative, canonical, "file" | "dir"]`, sorted by path. +pub const FOLLOWED_LINKS_KEY: &str = "followed_links"; + +fn kind_name(kind: LinkKind) -> &'static str { + match kind { + LinkKind::File => "file", + LinkKind::Dir => "dir", + } +} + +fn encode(links: &[FollowedLink]) -> String { + let rows = links + .iter() + .map(|link| { + serde_json::json!([ + link.relative, + link.canonical.to_string_lossy(), + kind_name(link.kind) + ]) + }) + .collect::>(); + serde_json::Value::Array(rows).to_string() +} + +/// `None` for anything that is not exactly the format [`encode`] writes. +fn decode(value: &str) -> Option> { + let rows: Vec<(String, String, String)> = serde_json::from_str(value).ok()?; + let mut links = BTreeMap::new(); + for (relative, canonical, kind) in rows { + let kind = match kind.as_str() { + "file" => LinkKind::File, + "dir" => LinkKind::Dir, + _ => return None, + }; + links.insert(relative, (PathBuf::from(canonical), kind)); + } + Some(links) +} + +/// Record the links a full build or full sync just indexed through. +pub fn record_followed_links(store: &Store, links: &[FollowedLink]) -> Result<()> { + store + .set_project_metadata(FOLLOWED_LINKS_KEY, &encode(links)) + .context("record followed links")?; + Ok(()) +} + +/// The logical paths whose stored stat proves nothing because the link that +/// reaches them is new, retargeted, or of unknown identity. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(crate) struct RehashUnder { + links: Vec<(String, LinkKind)>, +} + +impl RehashUnder { + /// Compare the current scan's links with the record in `store`. + pub(crate) fn for_scan(store: &Store, current: &[FollowedLink]) -> Result { + let recorded = store + .get_project_metadata(FOLLOWED_LINKS_KEY) + .context("read followed links")? + .as_deref() + .and_then(decode); + Ok(Self::compare(recorded.as_ref(), current)) + } + + fn compare( + recorded: Option<&BTreeMap>, + current: &[FollowedLink], + ) -> Self { + let links = current + .iter() + .filter(|link| { + recorded.is_none_or(|recorded| { + recorded.get(&link.relative) != Some(&(link.canonical.clone(), link.kind)) + }) + }) + .map(|link| (link.relative.clone(), link.kind)) + .collect(); + Self { links } + } + + /// Whether `relative` must be re-read regardless of its stored stat. + pub(crate) fn covers(&self, relative: &str) -> bool { + self.links.iter().any(|(link, kind)| match kind { + LinkKind::File => relative == link, + LinkKind::Dir => { + relative == link + || relative + .strip_prefix(link.as_str()) + .is_some_and(|rest| rest.starts_with('/')) + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn link(relative: &str, canonical: &str, kind: LinkKind) -> FollowedLink { + FollowedLink { + relative: relative.to_string(), + canonical: PathBuf::from(canonical), + kind, + } + } + + #[test] + fn the_record_round_trips_and_rejects_anything_else() { + let links = vec![ + link("a.ts", "/x/a.ts", LinkKind::File), + link("lib", "/x/lib", LinkKind::Dir), + ]; + let decoded = decode(&encode(&links)).expect("round trip"); + assert_eq!( + decoded.get("lib"), + Some(&(PathBuf::from("/x/lib"), LinkKind::Dir)) + ); + assert_eq!(decoded.len(), 2); + for malformed in ["", "{}", "[[\"a\",\"/x\"]]", "[[\"a\",\"/x\",\"socket\"]]"] { + assert_eq!(decode(malformed), None, "{malformed:?}"); + } + } + + #[test] + fn only_new_retargeted_or_retyped_links_are_rehashed() { + let recorded = decode(&encode(&[ + link("same", "/t/same", LinkKind::Dir), + link("moved", "/t/old", LinkKind::Dir), + link("retyped", "/t/retyped", LinkKind::Dir), + ])) + .unwrap(); + let current = [ + link("same", "/t/same", LinkKind::Dir), + link("moved", "/t/new", LinkKind::Dir), + link("retyped", "/t/retyped", LinkKind::File), + link("fresh", "/t/fresh", LinkKind::Dir), + ]; + let rehash = RehashUnder::compare(Some(&recorded), ¤t); + assert!(!rehash.covers("same/a.ts")); + assert!(rehash.covers("moved/a.ts")); + assert!(rehash.covers("retyped")); + assert!( + !rehash.covers("retyped/a.ts"), + "a file link covers itself only" + ); + assert!(rehash.covers("fresh/deep/b.ts")); + assert!( + !rehash.covers("freshly.ts"), + "a prefix match needs a segment boundary" + ); + } + + #[test] + fn an_unknown_record_rehashes_under_every_current_link() { + let current = [ + link("lib", "/t/lib", LinkKind::Dir), + link("x.ts", "/t/x.ts", LinkKind::File), + ]; + let rehash = RehashUnder::compare(None, ¤t); + assert!(rehash.covers("lib/a.ts")); + assert!(rehash.covers("x.ts")); + assert!(!rehash.covers("src/plain.ts")); + } +} diff --git a/crates/codegraph-watch/src/migrate.rs b/crates/codegraph-watch/src/migrate.rs index 1299ed1..5225555 100644 --- a/crates/codegraph-watch/src/migrate.rs +++ b/crates/codegraph-watch/src/migrate.rs @@ -73,7 +73,8 @@ pub(crate) fn migrate_project( let options = &scope.options; // `scan_project` returns a SORTED list, and every downstream pass keeps that // order, so no HashSet iteration order can reach the database or the outcome. - let candidates = codegraph_extract::engine::scan_project(project_root, options)?; + let scan = codegraph_extract::engine::scan_project_with_stats(project_root, options)?; + let candidates = scan.files; let total = candidates.len(); // Publishes `phase=building` BEFORE deleting a database byte, removes only @@ -180,6 +181,7 @@ pub(crate) fn migrate_project( rebuild .store() .set_project_metadata(INDEXED_WITH_VERSION_KEY, env!("CARGO_PKG_VERSION"))?; + crate::link_state::record_followed_links(rebuild.store(), &scan.links)?; // Explicit fallible finalization: pragma restore, checkpoint + compaction, // extraction stamp, stamp checkpoint, connection close, and only then the diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index 5b06fbb..75d1ba2 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -22,6 +22,7 @@ use codegraph_resolve::frameworks::godot_dsl_config::GodotDslConfig; use codegraph_store::queries::{FileReferenceSite, ReferenceSite}; use codegraph_store::{IndexLease, Store, StoreWriteOpen, StoreWritePurpose}; +use crate::link_state::{RehashUnder, record_followed_links}; use crate::policy::WatchPolicy; /// Bounded wall-clock budget for acquiring the ONE outer exclusive lease a sync @@ -147,7 +148,11 @@ pub fn pending_project_changes( let project_root = project_root.as_ref(); let paths = index_paths(project_root)?; let scope = ProjectScope::load(project_root, &paths)?; - let on_disk = codegraph_extract::engine::scan_project(project_root, &scope.options)?; + let scan = codegraph_extract::engine::scan_project_with_stats(project_root, &scope.options)?; + // A path reached through a new, retargeted or unrecorded link is re-read: + // its stored stat may describe a different file (#935). + let rehash = RehashUnder::for_scan(store, &scan.links)?; + let on_disk = scan.files; let tracked = store .all_files()? .into_iter() @@ -169,7 +174,9 @@ pub fn pending_project_changes( continue; }; let metadata = fs::metadata(&full).with_context(|| format!("stat {}", full.display()))?; - if stored.size == metadata.len() as i64 && stored.modified_at == modified_millis(&metadata) + if stored.size == metadata.len() as i64 + && stored.modified_at == modified_millis(&metadata) + && !rehash.covers(&relative) { continue; } @@ -247,7 +254,11 @@ fn sync_project_once_with_scope( let _active = cancel.map(SyncCancellation::enter); match open_sync_writer(paths, cancel)? { SyncWriter::Incremental(mut store) => { - let mut candidates = codegraph_extract::engine::scan_project(project_root, options)?; + let scan = codegraph_extract::engine::scan_project_with_stats(project_root, options)?; + // Re-read whatever a new, retargeted or unrecorded link reaches, so a + // same-stat file behind a moved link cannot keep its old graph (#935). + let rehash = RehashUnder::for_scan(&store, &scan.links)?; + let mut candidates = scan.files; // Cold CLI sync has no watcher event list, so removals are found by // diffing tracked files against scan_project's current in-scope set. // This includes both physically absent files and still-present files @@ -270,12 +281,14 @@ fn sync_project_once_with_scope( project_root, candidates, Some(&on_disk), + &rehash, scope, &include, &exclude, started, on_progress, )?; + record_followed_links(&store, &scan.links)?; store.finish_current_mutation()?; Ok(outcome) } @@ -425,6 +438,7 @@ fn sync_changed_paths_current_scope( project_root, changed, None, + &RehashUnder::default(), &scope, &include, &exclude, @@ -451,6 +465,7 @@ fn sync_paths_with_store( project_root: &Path, paths: impl IntoIterator>, scanned: Option<&HashSet>, + rehash: &RehashUnder, scope: &ProjectScope, include: &[String], exclude: &[String], @@ -506,6 +521,7 @@ fn sync_paths_with_store( store, &relative, scope, + rehash, &mut outcome, &mut dependent_sites, &mut dependent_fallbacks, @@ -759,6 +775,7 @@ fn sync_one( store: &mut Store, relative: &str, scope: &ProjectScope, + rehash: &RehashUnder, outcome: &mut SyncOutcome, dependent_sites: &mut BTreeMap>, dependent_fallbacks: &mut BTreeSet, @@ -793,6 +810,7 @@ fn sync_one( if let Some(file) = &stored && file.size == metadata.len() as i64 && file.modified_at == modified_millis(&metadata) + && !rehash.covers(relative) { outcome.files_skipped_unchanged += 1; return Ok(false); From 9bc7896a7360613bfa1cb710035cfb99809d507f Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 17:02:39 +0800 Subject: [PATCH 3/5] feat(watch): follow the scan's symlinks with live watches (#770) --- crates/codegraph-watch/src/watcher.rs | 654 +++++++++++++++++++++++++- 1 file changed, 640 insertions(+), 14 deletions(-) diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index 024cf55..e3703fc 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -15,10 +15,12 @@ use notify::{Event, RecommendedWatcher, RecursiveMode, Watcher}; use codegraph_core::IndexPaths; use codegraph_core::config::Config; use codegraph_extract::ExtensionOverrides; +use codegraph_extract::engine::{LinkKind, ScanProjectResult}; use crate::policy::{WatchPolicy, watch_disabled_reason}; use crate::sync::{ - SyncCancellation, SyncOutcome, sync_changed_paths_cancellable, sync_project_once_cancellable, + ProjectScope, SyncCancellation, SyncOutcome, sync_changed_paths_cancellable, + sync_project_once_cancellable, }; type SyncCallback = Arc; @@ -322,7 +324,16 @@ fn watch_registration(backend: WatchBackend) -> WatchRegistration { /// and `read_dir`/metadata errors on any subdir are tolerated (that subdir is /// skipped, the walk continues) so a transient FS error never panics startup. fn collect_watch_dirs(root: &Path, policy: &WatchPolicy) -> Vec { + collect_watch_tree(root, policy).0 +} + +/// [`collect_watch_dirs`], also reporting whether a walked directory holds a +/// symlink. Symlinks are never followed here: which ones the index follows, +/// and under which logical path, is the scan's decision (#935), which +/// [`LinkState::load`] reads. +fn collect_watch_tree(root: &Path, policy: &WatchPolicy) -> (Vec, bool) { let mut dirs = Vec::new(); + let mut saw_link = false; // Explicit stack DFS (no recursion) so a deep tree can't blow the stack. let mut stack = vec![root.to_path_buf()]; while let Some(dir) = stack.pop() { @@ -335,9 +346,13 @@ fn collect_watch_dirs(root: &Path, policy: &WatchPolicy) -> Vec { for entry in entries.flatten() { let path = entry.path(); // Only directories add inotify watches. Use `file_type()` (no extra - // stat syscall via DirEntry) and skip symlinks to avoid cycles. + // stat syscall via DirEntry); a symlink is only noted. let is_dir = match entry.file_type() { - Ok(ft) => ft.is_dir() && !ft.is_symlink(), + Ok(ft) if ft.is_symlink() => { + saw_link = true; + continue; + } + Ok(ft) => ft.is_dir(), Err(_) => continue, }; if !is_dir { @@ -354,7 +369,133 @@ fn collect_watch_dirs(root: &Path, policy: &WatchPolicy) -> Vec { } } dirs.sort(); - dirs + (dirs, saw_link) +} + +/// The symlinks the index follows (upstream #770), exactly as the scan chose +/// them, narrowed to what the stricter watch policy watches. The watcher never +/// watches a directory the scan did not walk through that very logical path, +/// so an incremental sync can only ever be handed a path `index --force` keeps. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct LinkState { + /// Logical paths of every followed link, files and directories. + link_paths: BTreeSet, + /// Directories scanned through a link whose whole logical chain is watched. + watched_dirs: BTreeSet, + /// Watched directory links: logical path → canonical target. + dir_links: Vec<(String, PathBuf)>, + /// A file link's indexed target → the links aliasing it. + aliases: BTreeMap>, +} + +impl LinkState { + fn load(project_root: &Path, paths: &IndexPaths, policy: &WatchPolicy) -> Self { + let Ok(scope) = ProjectScope::load(project_root, paths) else { + return Self::default(); + }; + match codegraph_extract::engine::scan_project_with_stats(project_root, &scope.options) { + Ok(scan) => Self::from_scan(&scan, policy), + Err(_) => Self::default(), + } + } + + fn from_scan(scan: &ScanProjectResult, policy: &WatchPolicy) -> Self { + let watched = |relative: &str| { + let mut prefix = String::new(); + relative.split('/').all(|segment| { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(segment); + policy.should_watch_dir(&prefix) + }) + }; + Self { + link_paths: scan.links.iter().map(|link| link.relative.clone()).collect(), + watched_dirs: scan + .linked_dirs + .iter() + .filter(|dir| watched(dir)) + .cloned() + .collect(), + dir_links: scan + .links + .iter() + .filter(|link| link.kind == LinkKind::Dir && watched(&link.relative)) + .map(|link| (link.relative.clone(), link.canonical.clone())) + .collect(), + aliases: scan.file_aliases.clone(), + } + } + + /// Whether an event at `relative` changes which links the index follows: a + /// path that is a symlink now, was a followed link, or was a watched + /// linked directory and no longer is one. + fn is_topology_event(&self, path: &Path, relative: &str) -> bool { + fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) + || self.link_paths.contains(relative) + || (self.watched_dirs.contains(relative) && !path.is_dir()) + } +} + +/// Native-recursive backends' root watch does not traverse a symlinked +/// directory, so each watched directory link gets its own recursive watch, up +/// to this many. +const MAX_SUPPLEMENTAL_WATCHES: usize = 256; + +/// The supplemental recursive watches for `dir_links`: the first +/// [`MAX_SUPPLEMENTAL_WATCHES`] in logical order, and whether the cap cut any. +fn supplemental_watches(dir_links: &[(String, PathBuf)]) -> (Vec<(String, PathBuf)>, bool) { + let mut links = dir_links.to_vec(); + links.sort(); + let truncated = links.len() > MAX_SUPPLEMENTAL_WATCHES; + links.truncate(MAX_SUPPLEMENTAL_WATCHES); + (links, truncated) +} + +/// Map an event a supplemental watch reported under its canonical target +/// (FSEvents reports real paths) back to the logical path the index uses; the +/// longest matching target wins, and any other path is returned unchanged. +fn logical_event_path( + path: &Path, + project_root: &Path, + supplemental: &[(String, PathBuf)], +) -> PathBuf { + supplemental + .iter() + .filter_map(|(relative, canonical)| { + path.strip_prefix(canonical) + .ok() + .map(|rest| (canonical.components().count(), relative, rest)) + }) + .max_by_key(|(depth, _, _)| *depth) + .map_or_else( + || path.to_path_buf(), + |(_, relative, rest)| project_root.join(relative).join(rest), + ) +} + +/// What to unwatch and watch when the followed links change. Everything is +/// re-registered: a retarget keeps a logical path but changes what it resolves +/// to, and a watch keeps the inode it resolved when it was added. +fn relink_plan(old: &BTreeSet, new: &BTreeSet) -> (Vec, Vec) { + (old.iter().cloned().collect(), new.iter().cloned().collect()) +} + +/// The removal hint an event really carries. inotify reports a deleted symlink +/// as a file removal and a directory moved away as a rename, so a known +/// directory that no longer is one is a removed directory whatever the hint. +fn effective_removal( + hint: RemovalHint, + relative: &str, + known_dirs: &BTreeSet, + is_dir_now: bool, +) -> RemovalHint { + if hint == RemovalHint::None && !is_dir_now && known_dirs.contains(relative) { + RemovalHint::Directory + } else { + hint + } } fn known_directory_paths(root: &Path, policy: &WatchPolicy) -> BTreeSet { @@ -459,6 +600,58 @@ fn reconcile_watch_dirs( true } +/// Apply a link topology change to the OS watcher: unwatch every previously +/// link-reached watch and watch the new set (see [`relink_plan`]). Returns the +/// new supplemental watches and whether their cap truncated them, or `None` +/// when a watch failure degraded the watcher. +fn relink( + watcher: &SharedWatcher, + project_root: &Path, + (old, old_supplemental): (&LinkState, &[(String, PathBuf)]), + new: &LinkState, + degraded: &Arc, + on_degraded: &Option, + on_sync_error: &Option, +) -> Option<(Vec<(String, PathBuf)>, bool)> { + let registration = watch_registration(platform_watch_backend()); + let (new_supplemental, truncated) = match registration { + WatchRegistration::SingleRootRecursive => supplemental_watches(&new.dir_links), + WatchRegistration::PerDirNonRecursive => (Vec::new(), false), + }; + let (old_set, new_set, mode) = match registration { + WatchRegistration::PerDirNonRecursive => ( + old.watched_dirs.clone(), + new.watched_dirs.clone(), + RecursiveMode::NonRecursive, + ), + WatchRegistration::SingleRootRecursive => ( + old_supplemental.iter().map(|(dir, _)| dir.clone()).collect(), + new_supplemental.iter().map(|(dir, _)| dir.clone()).collect(), + RecursiveMode::Recursive, + ), + }; + let Ok(mut guard) = watcher.lock() else { + return Some((new_supplemental, truncated)); + }; + let Some(watcher) = guard.as_mut() else { + return Some((new_supplemental, truncated)); + }; + let (unwatch, watch) = relink_plan(&old_set, &new_set); + for dir in unwatch { + // Best effort: a removed link's watch is often already gone. + let _ = watcher.unwatch(&project_root.join(dir)); + } + for dir in watch { + if let Err(err) = watcher.watch(&project_root.join(dir), mode) { + match handle_watch_error(&err, degraded, on_degraded, on_sync_error) { + WatchErrorClass::Degrade => return None, + WatchErrorClass::Warn | WatchErrorClass::Other => {} + } + } + } + Some((new_supplemental, truncated)) +} + /// Double `prev` for the next backoff step, saturating at [`MAX_BACKOFF`]. /// /// A zero/sub-ms `prev` seeds the schedule at 1ms so the doubling progresses; the @@ -815,7 +1008,32 @@ impl ProjectWatcher { // `RemoveKind::Any` after deletion, so this pre-event snapshot is the only // deterministic way to distinguish a known directory from an extensionless // file without inspecting a path that no longer exists. - let known_dirs = known_directory_paths(&project_root, &policy); + let (real_dirs, saw_link) = collect_watch_tree(&project_root, &policy); + // The links the index follows, as the scan chose them (#770). A tree + // without a single symlink never pays for that second walk. + let link_state = if saw_link { + LinkState::load(&project_root, &index_paths, &policy) + } else { + LinkState::default() + }; + let mut known_dirs = real_dirs + .iter() + .filter_map(|dir| policy.normalize_relative(dir)) + .collect::>(); + known_dirs.extend(link_state.watched_dirs.iter().cloned()); + let (supplemental, supplemental_truncated) = + if watch_registration(platform_watch_backend()) == WatchRegistration::SingleRootRecursive + { + supplemental_watches(&link_state.dir_links) + } else { + (Vec::new(), false) + }; + if supplemental_truncated && let Some(callback) = &options.on_sync_error { + callback(format!( + "watching only the first {MAX_SUPPLEMENTAL_WATCHES} symlinked directories; \ + changes below the rest are picked up by the next full sync" + )); + } // Build the OS watcher and register the pruned watch set BEFORE spawning // the loop, so its create-event handler can share the same watcher to add @@ -844,6 +1062,22 @@ impl ProjectWatcher { // inotify exhaustion. let backend = platform_watch_backend(); let mut targets = initial_watch_targets(backend, &project_root, &policy); + // Directories reached through a followed link: one NonRecursive + // watch per logical directory, or one Recursive supplemental watch + // per directory link where the root watch is recursive. + match watch_registration(backend) { + WatchRegistration::PerDirNonRecursive => targets.extend( + link_state + .watched_dirs + .iter() + .map(|dir| (project_root.join(dir), RecursiveMode::NonRecursive)), + ), + WatchRegistration::SingleRootRecursive => targets.extend( + supplemental + .iter() + .map(|(dir, _)| (project_root.join(dir), RecursiveMode::Recursive)), + ), + } // The index root is structurally ignored source, but its two // project-control files must still be observed. Per-directory // backends therefore add one explicit non-recursive control watch. @@ -923,6 +1157,8 @@ impl ProjectWatcher { degraded: loop_degraded, watcher: loop_watcher, known_dirs, + link_state, + supplemental, lock_contention_budget, lock_recovery_interval, }); @@ -1144,6 +1380,8 @@ struct EventLoopCtx { degraded: Arc, watcher: SharedWatcher, known_dirs: BTreeSet, + link_state: LinkState, + supplemental: Vec<(String, PathBuf)>, lock_contention_budget: Duration, lock_recovery_interval: Duration, } @@ -1163,6 +1401,8 @@ fn event_loop(ctx: EventLoopCtx) { degraded, watcher, mut known_dirs, + mut link_state, + mut supplemental, lock_contention_budget, lock_recovery_interval, } = ctx; @@ -1173,6 +1413,7 @@ fn event_loop(ctx: EventLoopCtx) { // per-path list (one full sync instead of N incremental ones) yet still // flushes exactly once, on the same debounce deadline. let mut full_sync_pending = false; + let mut supplemental_cap_warned = false; loop { let message = match deadline { Some(when) => match rx.recv_timeout(when.saturating_duration_since(Instant::now())) { @@ -1191,7 +1432,10 @@ fn event_loop(ctx: EventLoopCtx) { let WatchEventBatch { paths, removal } = batch; let mut control_changes = ControlChanges::default(); let mut normalized = Vec::new(); + // Set when this batch changes which links the index follows. + let mut link_topology_changed = false; for path in paths { + let path = logical_event_path(&path, &project_root, &supplemental); if let Some(relative) = runtime_scope.policy.normalize_relative(&path) { let is_control = control_files.classify(&relative, &mut control_changes); normalized.push((path, relative, is_control)); @@ -1228,6 +1472,9 @@ fn event_loop(ctx: EventLoopCtx) { } runtime_scope = next; full_sync_pending = true; + // The reconcile above dropped every link-reached watch; + // re-read the links under the new scope. + link_topology_changed = true; let now = epoch_millis(); for (_, relative, is_control) in &normalized { if *is_control { @@ -1254,6 +1501,19 @@ fn event_loop(ctx: EventLoopCtx) { // forever. The watch policy still applies (an ignored dir is // still ignored), and the removal escalates the burst to one // full sync — the only pass that can find those descendants. + if link_state.is_topology_event(&path, &relative) { + link_topology_changed = true; + let now = epoch_millis(); + pending + .entry(relative.clone()) + .and_modify(|info| info.last_seen_ms = now) + .or_insert(PendingInfo { + first_seen_ms: now, + last_seen_ms: now, + }); + } + let removal = + effective_removal(removal, &relative, &known_dirs, path.is_dir()); if classify_removed_directory(removal, &relative, &mut known_dirs) { if runtime_scope.policy.should_watch_dir(&relative) { full_sync_pending = true; @@ -1274,23 +1534,81 @@ fn event_loop(ctx: EventLoopCtx) { // descendants created in the same burst, e.g. `mkdir -p`) so // edits inside it are seen without a server restart. if path.is_dir() && runtime_scope.policy.should_watch_dir(&relative) { - register_new_dirs(&watcher, &runtime_scope.policy, &path); - known_dirs.extend(known_directory_paths(&path, &runtime_scope.policy)); + // A new link is the scan's to follow (above); a new real + // directory holding a link is a topology change too. + let is_link = fs::symlink_metadata(&path) + .is_ok_and(|meta| meta.file_type().is_symlink()); + if !is_link { + register_new_dirs(&watcher, &runtime_scope.policy, &path); + let (new_dirs, saw_link) = + collect_watch_tree(&path, &runtime_scope.policy); + known_dirs.extend( + new_dirs + .iter() + .filter_map(|dir| runtime_scope.policy.normalize_relative(dir)), + ); + link_topology_changed |= saw_link; + } } if runtime_scope.policy.should_handle_file(&relative) || (runtime_scope.policy.allows_file_path(&relative) && maybe_deleted_source(&relative)) { let now = epoch_millis(); - pending - .entry(relative) - .and_modify(|info| info.last_seen_ms = now) - .or_insert(PendingInfo { - first_seen_ms: now, - last_seen_ms: now, - }); + // A file link aliasing this path is the same content + // under another indexed path, so it is re-indexed too. + let aliases = link_state + .aliases + .get(&relative) + .into_iter() + .flatten() + .filter(|alias| runtime_scope.policy.should_handle_file(alias)) + .cloned() + .collect::>(); + for path in std::iter::once(relative).chain(aliases) { + pending + .entry(path) + .and_modify(|info| info.last_seen_ms = now) + .or_insert(PendingInfo { + first_seen_ms: now, + last_seen_ms: now, + }); + } } } + if link_topology_changed && runtime_scope.enabled { + // Re-read the links from the scan, re-register every + // link-reached watch, and let one full sync apply them. + let next_links = + LinkState::load(&project_root, &index_paths, &runtime_scope.policy); + let Some((next_supplemental, truncated)) = relink( + &watcher, + &project_root, + (&link_state, &supplemental), + &next_links, + °raded, + &on_degraded, + &on_sync_error, + ) else { + break; + }; + if truncated + && !supplemental_cap_warned + && let Some(callback) = &on_sync_error + { + supplemental_cap_warned = true; + callback(format!( + "watching only the first {MAX_SUPPLEMENTAL_WATCHES} symlinked \ + directories; changes below the rest are picked up by the next \ + full sync" + )); + } + known_dirs.retain(|dir| !link_state.watched_dirs.contains(dir)); + known_dirs.extend(next_links.watched_dirs.iter().cloned()); + link_state = next_links; + supplemental = next_supplemental; + full_sync_pending = true; + } // While lock contention paused auto-sync, the recovery retry keeps // its own cadence; a burst of edits must not poll the other writer. if !pending.is_empty() && !degraded.is_recovering() { @@ -3415,4 +3733,312 @@ mod tests { "editing a file in a dir created after start should trigger a sync" ); } + + // ---- #770: symlinked directories and file links ------------------------- + + /// A plain temp directory for link targets: unlike `TestDir`, it holds no + /// index namespace that a followed link would lead into. + struct LinkTarget(PathBuf); + + impl LinkTarget { + fn new(name: &str) -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let path = std::env::temp_dir().join(format!( + "codegraph-link-target-{name}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + } + + impl Drop for LinkTarget { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + /// Create a symlink. Without the privilege (Windows) a local run skips the + /// test, but CI must exercise it, so there a refusal fails loudly. + fn symlink_at(target: &Path, at: &Path, dir: bool) -> bool { + #[cfg(unix)] + let made = { + let _ = dir; + std::os::unix::fs::symlink(target, at) + }; + #[cfg(windows)] + let made = if dir { + std::os::windows::fs::symlink_dir(target, at) + } else { + std::os::windows::fs::symlink_file(target, at) + }; + match made { + Ok(()) => true, + Err(error) if std::env::var_os("CI").is_some() => { + panic!("CI must be able to create symlinks: {error}") + } + Err(_) => false, + } + } + + fn remove_symlink(at: &Path) { + // A directory symlink is a directory entry on Windows, a file on Unix. + if fs::remove_file(at).is_err() { + fs::remove_dir(at).expect("remove symlink"); + } + } + + fn wait_until(mut condition: impl FnMut() -> bool) -> bool { + for _ in 0..100 { + std::thread::sleep(Duration::from_millis(50)); + if condition() { + return true; + } + } + false + } + + type Recorded = Arc>>>; + + /// A real watcher whose syncs only record their paths and count full syncs. + fn recording_watcher(root: &Path) -> (ProjectWatcher, Recorded, Arc) { + let synced: Recorded = Arc::new(Mutex::new(Vec::new())); + let sink = Arc::clone(&synced); + let sync_fn: SyncFn = Arc::new(move |paths| { + sink.lock().unwrap().push(paths); + Ok(SyncOutcome::default()) + }); + let full = Arc::new(AtomicUsize::new(0)); + let full_counter = Arc::clone(&full); + let full_sync_fn: FullSyncFn = Arc::new(move || { + full_counter.fetch_add(1, AtomicOrdering::SeqCst); + Ok(SyncOutcome::default()) + }); + let watcher = ProjectWatcher::start( + root, + WatchOptions { + debounce: Duration::from_millis(50), + sync_fn: Some(sync_fn), + full_sync_fn: Some(full_sync_fn), + ..WatchOptions::default() + }, + ) + .unwrap() + .unwrap(); + // Native backends (FSEvents) need a moment before streams report. + std::thread::sleep(Duration::from_millis(300)); + (watcher, synced, full) + } + + fn saw_path(synced: &Recorded, wanted: &str) -> bool { + synced + .lock() + .unwrap() + .iter() + .any(|paths| paths.iter().any(|path| path == wanted)) + } + + #[test] + fn symlink_supplemental_watches_keep_the_first_256_in_logical_order() { + let links = (0..300) + .rev() + .map(|n| (format!("l{n:03}"), PathBuf::from(format!("/t/{n}")))) + .collect::>(); + let (selected, truncated) = supplemental_watches(&links); + assert!(truncated); + assert_eq!(selected.len(), MAX_SUPPLEMENTAL_WATCHES); + assert_eq!(selected.first().unwrap().0, "l000"); + assert_eq!(selected.last().unwrap().0, "l255"); + let (few, truncated) = supplemental_watches(&links[..3]); + assert!(!truncated); + assert_eq!( + few.iter().map(|(dir, _)| dir.as_str()).collect::>(), + vec!["l297", "l298", "l299"] + ); + } + + #[test] + fn symlink_event_paths_map_back_through_the_longest_canonical_target() { + let root = Path::new("/proj"); + let supplemental = vec![ + ("ext".to_string(), PathBuf::from("/real/lib")), + ("ext/inner".to_string(), PathBuf::from("/real/lib/deep")), + ("other".to_string(), PathBuf::from("/elsewhere")), + ]; + assert_eq!( + logical_event_path(Path::new("/real/lib/a.ts"), root, &supplemental), + PathBuf::from("/proj/ext/a.ts") + ); + assert_eq!( + logical_event_path(Path::new("/real/lib/deep/b.ts"), root, &supplemental), + PathBuf::from("/proj/ext/inner/b.ts") + ); + assert_eq!( + logical_event_path(Path::new("/proj/src/c.ts"), root, &supplemental), + PathBuf::from("/proj/src/c.ts"), + "a path no supplemental watch covers is unchanged" + ); + assert_eq!( + logical_event_path(Path::new("/real/library/d.ts"), root, &supplemental), + PathBuf::from("/real/library/d.ts"), + "a prefix must end on a path component" + ); + } + + #[test] + fn symlink_relink_plan_rewatches_every_link_reached_directory() { + let old = ["kept", "gone"].map(String::from).into_iter().collect(); + let new = ["kept", "added"].map(String::from).into_iter().collect(); + let (unwatch, watch) = relink_plan(&old, &new); + assert_eq!(unwatch, vec!["gone".to_string(), "kept".to_string()]); + assert_eq!( + watch, + vec!["added".to_string(), "kept".to_string()], + "an unchanged logical path is re-watched: its link may now resolve elsewhere" + ); + } + + #[test] + fn symlink_vanished_known_directory_is_a_removal_whatever_the_hint() { + let known = ["lib".to_string()].into_iter().collect::>(); + assert_eq!( + effective_removal(RemovalHint::None, "lib", &known, false), + RemovalHint::Directory, + "inotify reports a deleted symlink as a file removal" + ); + assert_eq!( + effective_removal(RemovalHint::None, "lib", &known, true), + RemovalHint::None + ); + assert_eq!( + effective_removal(RemovalHint::None, "a.ts", &known, false), + RemovalHint::None + ); + assert_eq!( + effective_removal(RemovalHint::Ambiguous, "lib", &known, true), + RemovalHint::Ambiguous + ); + } + + /// The scan alone decides which links the index follows; the watcher only + /// narrows that set with its stricter policy and never adds to it. + #[test] + fn symlink_link_state_follows_the_scan_and_never_widens_it() { + let dir = crate::sync::tests::TestDir::new("watch-symlink-parity"); + let outside = LinkTarget::new("parity"); + fs::create_dir_all(outside.0.join("sub")).unwrap(); + fs::write(outside.0.join("sub/o.ts"), "export const o = 1;\n").unwrap(); + fs::create_dir_all(dir.path().join(".cache")).unwrap(); + fs::write(dir.path().join(".cache/c.ts"), "export const c = 1;\n").unwrap(); + fs::create_dir_all(dir.path().join(".codegraph-sources")).unwrap(); + fs::write( + dir.path().join(".codegraph-sources/s.ts"), + "export const s = 1;\n", + ) + .unwrap(); + if !symlink_at(&dir.path().join(".cache"), &dir.path().join("l"), true) + || !symlink_at( + &dir.path().join(".codegraph-sources"), + &dir.path().join("cs"), + true, + ) + || !symlink_at(&outside.0, &dir.path().join("ext"), true) + { + return; + } + let paths = crate::sync::index_paths(dir.path()).unwrap(); + let policy = WatchPolicy::new(dir.path()); + let state = LinkState::load(dir.path(), &paths, &policy); + assert_eq!( + state.link_paths, + ["ext".to_string()].into_iter().collect(), + "the real `.cache` and `.codegraph-sources` win, so `l` and `cs` are not followed" + ); + assert_eq!( + state.watched_dirs, + ["ext".to_string(), "ext/sub".to_string()] + .into_iter() + .collect() + ); + assert_eq!(state.dir_links.len(), 1); + } + + #[test] + fn symlink_edit_inside_a_linked_directory_syncs_its_logical_path() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-symlink-edit"); + let outside = LinkTarget::new("edit"); + fs::write(outside.0.join("x.ts"), "export const x = 1;\n").unwrap(); + if !symlink_at(&outside.0, &dir.path().join("ext"), true) { + return; + } + let (watcher, synced, _) = recording_watcher(dir.path()); + fs::write(outside.0.join("x.ts"), "export const x = 2;\n").unwrap(); + let seen = wait_until(|| saw_path(&synced, "ext/x.ts")); + watcher.stop(); + assert!(seen, "an edit behind a followed link syncs its logical path"); + } + + #[test] + fn symlink_alias_is_reindexed_with_its_target() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-symlink-alias"); + fs::create_dir_all(dir.path().join("src/real")).unwrap(); + fs::write(dir.path().join("src/real/a.ts"), "export const a = 1;\n").unwrap(); + if !symlink_at( + &dir.path().join("src/real/a.ts"), + &dir.path().join("src/afile.ts"), + false, + ) { + return; + } + let (watcher, synced, _) = recording_watcher(dir.path()); + fs::write(dir.path().join("src/real/a.ts"), "export const a = 2;\n").unwrap(); + let seen = wait_until(|| saw_path(&synced, "src/real/a.ts") && saw_path(&synced, "src/afile.ts")); + watcher.stop(); + assert!(seen, "a file link is re-indexed with the file it aliases"); + } + + #[test] + fn symlink_creation_retarget_and_removal_each_schedule_a_full_sync() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-symlink-topology"); + let first = LinkTarget::new("topology-first"); + let second = LinkTarget::new("topology-second"); + fs::write(dir.path().join("app.ts"), "export const app = 1;\n").unwrap(); + fs::write(first.0.join("x.ts"), "export const x = 1;\n").unwrap(); + fs::write(second.0.join("y.ts"), "export const y = 1;\n").unwrap(); + let (watcher, synced, full) = recording_watcher(dir.path()); + let link = dir.path().join("lib"); + + if !symlink_at(&first.0, &link, true) { + watcher.stop(); + return; + } + let created = wait_until(|| full.load(AtomicOrdering::SeqCst) >= 1); + std::thread::sleep(Duration::from_millis(300)); + fs::write(first.0.join("x.ts"), "export const x = 2;\n").unwrap(); + let watched_first = wait_until(|| saw_path(&synced, "lib/x.ts")); + + let before_retarget = full.load(AtomicOrdering::SeqCst); + remove_symlink(&link); + assert!(symlink_at(&second.0, &link, true)); + let retargeted = wait_until(|| full.load(AtomicOrdering::SeqCst) > before_retarget); + std::thread::sleep(Duration::from_millis(300)); + fs::write(second.0.join("y.ts"), "export const y = 2;\n").unwrap(); + let watched_second = wait_until(|| saw_path(&synced, "lib/y.ts")); + + let before_removal = full.load(AtomicOrdering::SeqCst); + remove_symlink(&link); + let removed = wait_until(|| full.load(AtomicOrdering::SeqCst) > before_removal); + watcher.stop(); + + assert!(created, "creating a directory link schedules a full sync"); + assert!(watched_first, "the new link's target is watched"); + assert!(retargeted, "retargeting a link schedules a full sync"); + assert!(watched_second, "the retargeted link resolves to its new target"); + assert!(removed, "removing a link schedules a full sync"); + } } From d7a8cde018617b4f3f0b7bf2f350fab4ddad1857 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 17:07:49 +0800 Subject: [PATCH 4/5] ci: run the symlink watcher tests on macOS; docs: record symlink following --- .github/workflows/ci.yml | 40 +++++++++++++- crates/codegraph-cli/tests/symlink_sync.rs | 13 +++-- crates/codegraph-watch/src/watcher.rs | 46 ++++++++++------ docs/cli.md | 19 +++++++ docs/upstream-sync/UPSTREAM.md | 52 +++++++++++++++++-- docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md | 5 +- scripts/tests/ci-gate.test.sh | 2 +- 7 files changed, 150 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd5352a..ba9312d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,6 +191,44 @@ jobs: - name: Run long-lived MCP database replacement acceptance run: cargo test -p codegraph-rs --locked --test batch_m_long_lived_mcp + # FSEvents is the one native-recursive watcher backend no other job runs: + # Linux uses inotify and Windows ReadDirectoryChangesW. This job runs the + # symlink-following scan and watcher tests on macOS (#770). + macos-watcher: + name: macOS Watcher + needs: workspace-version + runs-on: macos-15 + timeout-minutes: 40 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master 2026-08-21 + with: + toolchain: 1.96.0 + + - name: Verify workspace version surfaces before Cargo/cache + run: bash scripts/check-workspace-versions.sh + + - name: Cache Rust inputs + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: cargo-home-${{ runner.os }}-${{ runner.arch }} + cache-targets: false + + - name: Configure compiler cache + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + with: + version: "v0.16.0" + + - name: Run the symlink scan and watcher tests + run: | + cargo test -p codegraph-extract --locked --test scan_symlinks + cargo test -p codegraph-watch --locked symlink + audit: name: Security Audit needs: workspace-version @@ -268,7 +306,7 @@ jobs: ci-success: name: CI Success - needs: [workspace-version, linux, windows-clippy, windows-test, audit] + needs: [workspace-version, linux, windows-clippy, windows-test, macos-watcher, audit] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/crates/codegraph-cli/tests/symlink_sync.rs b/crates/codegraph-cli/tests/symlink_sync.rs index 316607b..2a75a21 100644 --- a/crates/codegraph-cli/tests/symlink_sync.rs +++ b/crates/codegraph-cli/tests/symlink_sync.rs @@ -110,17 +110,22 @@ fn store(project: &Path) -> Store { .expect("open the index for reading") } -/// Files (path, hash) and nodes (file, kind, name, line), sorted. -fn snapshot(project: &Path) -> (Vec<(String, String)>, Vec<(String, String, String, i64)>) { +/// An indexed file: (path, content hash). +type FileRow = (String, String); +/// An indexed node: (file, kind, name, start line). +type NodeRow = (String, String, String, i64); + +/// Files and nodes, sorted. +fn snapshot(project: &Path) -> (Vec, Vec) { let store = store(project); - let mut files: Vec<(String, String)> = store + let mut files: Vec = store .all_files() .unwrap() .into_iter() .map(|file| (file.path, file.content_hash)) .collect(); files.sort(); - let mut nodes: Vec<(String, String, String, i64)> = store + let mut nodes: Vec = store .all_nodes() .unwrap() .into_iter() diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index e3703fc..feab1ac 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -411,7 +411,11 @@ impl LinkState { }) }; Self { - link_paths: scan.links.iter().map(|link| link.relative.clone()).collect(), + link_paths: scan + .links + .iter() + .map(|link| link.relative.clone()) + .collect(), watched_dirs: scan .linked_dirs .iter() @@ -625,8 +629,14 @@ fn relink( RecursiveMode::NonRecursive, ), WatchRegistration::SingleRootRecursive => ( - old_supplemental.iter().map(|(dir, _)| dir.clone()).collect(), - new_supplemental.iter().map(|(dir, _)| dir.clone()).collect(), + old_supplemental + .iter() + .map(|(dir, _)| dir.clone()) + .collect(), + new_supplemental + .iter() + .map(|(dir, _)| dir.clone()) + .collect(), RecursiveMode::Recursive, ), }; @@ -1021,13 +1031,13 @@ impl ProjectWatcher { .filter_map(|dir| policy.normalize_relative(dir)) .collect::>(); known_dirs.extend(link_state.watched_dirs.iter().cloned()); - let (supplemental, supplemental_truncated) = - if watch_registration(platform_watch_backend()) == WatchRegistration::SingleRootRecursive - { - supplemental_watches(&link_state.dir_links) - } else { - (Vec::new(), false) - }; + let (supplemental, supplemental_truncated) = if watch_registration(platform_watch_backend()) + == WatchRegistration::SingleRootRecursive + { + supplemental_watches(&link_state.dir_links) + } else { + (Vec::new(), false) + }; if supplemental_truncated && let Some(callback) = &options.on_sync_error { callback(format!( "watching only the first {MAX_SUPPLEMENTAL_WATCHES} symlinked directories; \ @@ -1512,8 +1522,7 @@ fn event_loop(ctx: EventLoopCtx) { last_seen_ms: now, }); } - let removal = - effective_removal(removal, &relative, &known_dirs, path.is_dir()); + let removal = effective_removal(removal, &relative, &known_dirs, path.is_dir()); if classify_removed_directory(removal, &relative, &mut known_dirs) { if runtime_scope.policy.should_watch_dir(&relative) { full_sync_pending = true; @@ -3978,7 +3987,10 @@ mod tests { fs::write(outside.0.join("x.ts"), "export const x = 2;\n").unwrap(); let seen = wait_until(|| saw_path(&synced, "ext/x.ts")); watcher.stop(); - assert!(seen, "an edit behind a followed link syncs its logical path"); + assert!( + seen, + "an edit behind a followed link syncs its logical path" + ); } #[test] @@ -3996,7 +4008,8 @@ mod tests { } let (watcher, synced, _) = recording_watcher(dir.path()); fs::write(dir.path().join("src/real/a.ts"), "export const a = 2;\n").unwrap(); - let seen = wait_until(|| saw_path(&synced, "src/real/a.ts") && saw_path(&synced, "src/afile.ts")); + let seen = + wait_until(|| saw_path(&synced, "src/real/a.ts") && saw_path(&synced, "src/afile.ts")); watcher.stop(); assert!(seen, "a file link is re-indexed with the file it aliases"); } @@ -4038,7 +4051,10 @@ mod tests { assert!(created, "creating a directory link schedules a full sync"); assert!(watched_first, "the new link's target is watched"); assert!(retargeted, "retargeting a link schedules a full sync"); - assert!(watched_second, "the retargeted link resolves to its new target"); + assert!( + watched_second, + "the retargeted link resolves to its new target" + ); assert!(removed, "removing a link schedules a full sync"); } } diff --git a/docs/cli.md b/docs/cli.md index 393fcea..c3baab6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1045,6 +1045,25 @@ output, so it stays indexed and watched. The root `.gitignore` prunes the index and the watcher alike, with git's own rules: a slash-less rule applies at any depth, a leading or inner `/` anchors it to the project root, `*` and `**` glob, and `!` re-includes a path unless a directory above it is ignored. + +Indexing follows symlinked files and directories, including targets outside +the project, and indexes their files under the link's own path. A directory is +indexed once, under the path that reaches it through the fewest symlinks; a tie +goes to the alphabetically first path. So a real directory always wins over a +link to it, and of two links to one target the first path wins. A link is not +followed to the project root or a directory above it, into the project's `.git` +or index root, or to a target that is missing or unreadable. Ignore rules judge +a link at its own path, so a link named `node_modules` is skipped like the +directory. Each full index records the links it followed. A later `sync` re-reads +every file behind a link that is new or now points elsewhere, even when the size +and modification time look unchanged. An index built before that record +existed is re-read once below every link. The watcher watches the directories +indexing reached through a link. Creating, removing or retargeting a link +schedules one full reconcile. An edit to a file that a file symlink points at +also re-indexes the symlink. One exception is not watched: a file symlink whose +target lies outside every indexed directory. Edits to that target are picked +up by the next full `sync`. + This keeps the total watch count well inside the OS inotify limit on large trees and makes daemon startup fast. A newly-created non-ignored directory is picked up automatically on its create event — no restart required. diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index 3ea12c4..afbc2d6 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -42,9 +42,10 @@ below remain immutable historical evidence. > > - **The UI family is DEFERRED** — the viewer and F1–F12, by owner decision > (2026-10-01); see Current alignment above and the audit's re-triage table. -> - **#770 symlinked-directory watching is DEFERRED** until the scan's symlink -> policy is settled: the scan does not follow symlinked directories, so a -> watcher that did would watch what is never indexed. +> - **#770 symlinked-directory watching was DEFERRED at `v0.51.0`**: the scan +> did not follow symlinks, so a watcher that did would have watched what was +> never indexed. It landed afterwards in #288, together with the #935 scan +> port; see the 2026-10-01 symlink entry below. > - **#1829 / #1878 git-stamped pending status stays KEEP-RUST** (a full > inventory); its fast path is deferred pending a measurement. > - **KEEP-RUST divergences** are recorded row by row in the audit — among them @@ -117,6 +118,51 @@ below remain immutable historical evidence. ## Sync log +### 2026-10-01 — #935 row CORRECTED; #935 and #770 symlink following LANDED on main (#288) + +The 2026-06-24 `v1.0.1 → v1.1.0` entry below records "In-root symlink indexing +(#935)" as ALREADY-HAVE. That was wrong. The official `v0.51.0` binary indexed +only `src/real/a.ts` from +`{src/real/a.ts, src/afile.ts -> real/a.ts, src/linkdir -> real, src/extlink -> ../../outside/lib}`, +because the scan skipped every symlink. That row stays as written; this entry +supersedes it. + +The owner decided on 2026-10-01 to follow symlinks by default, as upstream does, +including targets outside the project. Its design passed the kirocodex +plan-convergence review in round 3. #288 ports both behaviors: the scan follows +symlinked files and directories (#935), and the watcher watches what the scan +reached (#770). A binary built from the PR indexes `src/afile.ts`, +`src/extlink/out.ts` and `src/real/a.ts` from the same tree. `src/linkdir` adds +nothing, because the real `src/real` wins. + +KEEP-RUST divergences: + +- **Precedence:** a directory is scanned once, under the logical path with the + fewest symlink hops, ties going to the smallest path. Upstream's winner + depends on `read_dir` order. +- **Targets that are not followed:** a link to the project root or an ancestor + of it, a link into the canonical `.git` or a reserved index root, and a + missing or unreadable target. Upstream re-walks a parent tree under an + ancestor link, and skips `.git` and data directories by entry name only. +- **Retargets keep `sync` equal to `index --force`:** a full build records its + links under `project_metadata` key `followed_links`. Full sync and `status` + re-read every path behind a new, retargeted or retyped link, and behind every + link when the record is missing. +- **The watcher takes its links from the scan.** It narrows them by its stricter + policy, so it never watches a path the scan did not reach. Native-recursive + backends add up to 256 supplemental watches; that cap matches upstream. An + edit to a file link's target re-indexes the link. +- **Known limitation, as upstream:** a file link whose target lies outside every + scanned directory is not watched; the next full sync picks it up. + +#286 (`f23b0bd`) fixed a pre-existing `v0.51.0` defect found along the way. A +full sync judged scope with the watcher's stricter policy, so it deleted indexed +files under `.cache`, `vcpkg_installed`, the other watch-only defaults, and +top-level `.codegraph-*` directories. + +Goldens: all 19 re-indexable corpora are byte-identical; they contain no +symlinks. The change ships in the next release. + ### 2026-10-01 — CLOSEOUT: `v1.6.0 → v1.6.1` sync COMPLETE in codegraph-rs `v0.51.0` (tracked parity advanced to `v1.6.1`) The `v1.6.1` range (`3ed73bc..f4ddf50`, 113 commits) was audited on 2026-09-30 diff --git a/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md b/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md index abd0127..b878246 100644 --- a/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md +++ b/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md @@ -7,9 +7,8 @@ in the range was classified against that tree by two read-only passes (resolution/extraction; MCP/CLI/installer/UI family), and each PORT was re-verified against the code before it was implemented. -- **Tracked release:** [`UPSTREAM.md`](UPSTREAM.md) advanced to `v1.6.1` on - 2026-10-01, once the ported rows shipped in codegraph-rs `v0.51.0` and the - downloaded binaries were verified. +- **Tracked release:** [`UPSTREAM.md`](UPSTREAM.md) keeps `v1.6.0` until the + ported rows ship in a release and the downloaded binaries are verified. - **Kernel commits:** upstream's `codegraph-kernel` (MIT, `publish = false`) mirrors each TypeScript change for its native backend. They were read as reference only; none of them is a separate port. diff --git a/scripts/tests/ci-gate.test.sh b/scripts/tests/ci-gate.test.sh index 33be5ff..7af419c 100755 --- a/scripts/tests/ci-gate.test.sh +++ b/scripts/tests/ci-gate.test.sh @@ -34,7 +34,7 @@ expect() { run A_repository "$root"; expect A_repository zero 'only all-success exits zero' B=$(fixture B_pristine); run B_pristine "$B"; expect B_pristine zero 'release topology' -C=$(fixture C_needs); mutate "$C/.github/workflows/ci.yml" ' needs: [workspace-version, linux, windows-clippy, windows-test, audit]' ' needs: [workspace-version, linux, windows-clippy, audit]'; run C_needs "$C"; expect C_needs nonzero 'MISMATCH \[gate-needs\]' +C=$(fixture C_needs); mutate "$C/.github/workflows/ci.yml" ' needs: [workspace-version, linux, windows-clippy, windows-test, macos-watcher, audit]' ' needs: [workspace-version, linux, windows-clippy, macos-watcher, audit]'; run C_needs "$C"; expect C_needs nonzero 'MISMATCH \[gate-needs\]' D=$(fixture D_always); mutate "$D/.github/workflows/ci.yml" ' if: always()' ' if: success()'; run D_always "$D"; expect D_always nonzero 'MISMATCH \[gate-always\]' E=$(fixture E_new_job); mutate "$E/.github/workflows/ci.yml" ' ci-success:' $' fuzz:\n name: Fuzz\n runs-on: ubuntu-24.04\n steps:\n - run: echo fuzz\n\n ci-success:'; run E_new_job "$E"; expect E_new_job nonzero 'fuzz' F=$(fixture F_coverage); mutate "$F/codecov.yml" ' informational: true' ' informational: false'; run F_coverage "$F"; expect F_coverage nonzero 'MISMATCH \[coverage\]' From 2d88668afeb10ff1f1e0e200779eb15739f1c01e Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 17:26:24 +0800 Subject: [PATCH 5/5] fix(watch): schedule the full sync a link topology change owes, and reject ambiguous link records --- crates/codegraph-watch/src/link_state.rs | 33 ++++++- crates/codegraph-watch/src/watcher.rs | 120 ++++++++++++++++++++--- 2 files changed, 136 insertions(+), 17 deletions(-) diff --git a/crates/codegraph-watch/src/link_state.rs b/crates/codegraph-watch/src/link_state.rs index 717cd29..f506d77 100644 --- a/crates/codegraph-watch/src/link_state.rs +++ b/crates/codegraph-watch/src/link_state.rs @@ -29,9 +29,13 @@ fn kind_name(kind: LinkKind) -> &'static str { } } +/// The canonical record: one row per link, sorted by logical path. fn encode(links: &[FollowedLink]) -> String { + let mut links = links.iter().collect::>(); + links.sort_by(|left, right| left.relative.cmp(&right.relative)); + links.dedup_by(|left, right| left.relative == right.relative); let rows = links - .iter() + .into_iter() .map(|link| { serde_json::json!([ link.relative, @@ -43,16 +47,26 @@ fn encode(links: &[FollowedLink]) -> String { serde_json::Value::Array(rows).to_string() } -/// `None` for anything that is not exactly the format [`encode`] writes. +/// `None` for anything that is not exactly the format [`encode`] writes, +/// including rows out of order or a logical path recorded twice: an ambiguous +/// record proves no link identity, so it counts as missing. fn decode(value: &str) -> Option> { let rows: Vec<(String, String, String)> = serde_json::from_str(value).ok()?; let mut links = BTreeMap::new(); + let mut previous: Option = None; for (relative, canonical, kind) in rows { + if previous + .as_ref() + .is_some_and(|previous| *previous >= relative) + { + return None; + } let kind = match kind.as_str() { "file" => LinkKind::File, "dir" => LinkKind::Dir, _ => return None, }; + previous = Some(relative.clone()); links.insert(relative, (PathBuf::from(canonical), kind)); } Some(links) @@ -141,6 +155,21 @@ mod tests { for malformed in ["", "{}", "[[\"a\",\"/x\"]]", "[[\"a\",\"/x\",\"socket\"]]"] { assert_eq!(decode(malformed), None, "{malformed:?}"); } + // A duplicated or reordered path is ambiguous, so it proves nothing. + let duplicated = r#"[["lib","/old","dir"],["lib","/new","dir"]]"#; + let reordered = r#"[["z","/z","dir"],["a","/a","dir"]]"#; + assert_eq!(decode(duplicated), None); + assert_eq!(decode(reordered), None); + } + + #[test] + fn an_ambiguous_record_rehashes_every_current_link() { + // The last duplicate matches the current target; a decoder that kept it + // would skip the rehash the earlier, different identity demands. + let duplicated = r#"[["lib","/old","dir"],["lib","/new","dir"]]"#; + let current = [link("lib", "/new", LinkKind::Dir)]; + let rehash = RehashUnder::compare(decode(duplicated).as_ref(), ¤t); + assert!(rehash.covers("lib/a.ts")); } #[test] diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index feab1ac..00f4549 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -439,6 +439,9 @@ impl LinkState { fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) || self.link_paths.contains(relative) || (self.watched_dirs.contains(relative) && !path.is_dir()) + // A file link's target turned into a directory: the link now + // reaches a tree the scan must follow. + || (self.aliases.contains_key(relative) && path.is_dir()) } } @@ -447,6 +450,19 @@ impl LinkState { /// to this many. const MAX_SUPPLEMENTAL_WATCHES: usize = 256; +/// The single warning a watcher gives when its supplemental watches were +/// capped, however often its link set is re-read. +fn supplemental_cap_warning(truncated: bool, warned: &mut bool) -> Option { + if !truncated || *warned { + return None; + } + *warned = true; + Some(format!( + "watching only the first {MAX_SUPPLEMENTAL_WATCHES} symlinked directories; \ + changes below the rest are picked up by the next full sync" + )) +} + /// The supplemental recursive watches for `dir_links`: the first /// [`MAX_SUPPLEMENTAL_WATCHES`] in logical order, and whether the cap cut any. fn supplemental_watches(dir_links: &[(String, PathBuf)]) -> (Vec<(String, PathBuf)>, bool) { @@ -1038,11 +1054,12 @@ impl ProjectWatcher { } else { (Vec::new(), false) }; - if supplemental_truncated && let Some(callback) = &options.on_sync_error { - callback(format!( - "watching only the first {MAX_SUPPLEMENTAL_WATCHES} symlinked directories; \ - changes below the rest are picked up by the next full sync" - )); + let mut supplemental_cap_warned = false; + if let Some(warning) = + supplemental_cap_warning(supplemental_truncated, &mut supplemental_cap_warned) + && let Some(callback) = &options.on_sync_error + { + callback(warning); } // Build the OS watcher and register the pruned watch set BEFORE spawning @@ -1169,6 +1186,7 @@ impl ProjectWatcher { known_dirs, link_state, supplemental, + supplemental_cap_warned, lock_contention_budget, lock_recovery_interval, }); @@ -1392,6 +1410,7 @@ struct EventLoopCtx { known_dirs: BTreeSet, link_state: LinkState, supplemental: Vec<(String, PathBuf)>, + supplemental_cap_warned: bool, lock_contention_budget: Duration, lock_recovery_interval: Duration, } @@ -1413,6 +1432,7 @@ fn event_loop(ctx: EventLoopCtx) { mut known_dirs, mut link_state, mut supplemental, + mut supplemental_cap_warned, lock_contention_budget, lock_recovery_interval, } = ctx; @@ -1423,7 +1443,6 @@ fn event_loop(ctx: EventLoopCtx) { // per-path list (one full sync instead of N incremental ones) yet still // flushes exactly once, on the same debounce deadline. let mut full_sync_pending = false; - let mut supplemental_cap_warned = false; loop { let message = match deadline { Some(when) => match rx.recv_timeout(when.saturating_duration_since(Instant::now())) { @@ -1556,7 +1575,19 @@ fn event_loop(ctx: EventLoopCtx) { .iter() .filter_map(|dir| runtime_scope.policy.normalize_relative(dir)), ); - link_topology_changed |= saw_link; + if saw_link { + // The directory may add no pending file, yet the + // full sync it owes must still run on the deadline. + link_topology_changed = true; + let now = epoch_millis(); + pending + .entry(relative.clone()) + .and_modify(|info| info.last_seen_ms = now) + .or_insert(PendingInfo { + first_seen_ms: now, + last_seen_ms: now, + }); + } } } if runtime_scope.policy.should_handle_file(&relative) @@ -1601,16 +1632,11 @@ fn event_loop(ctx: EventLoopCtx) { ) else { break; }; - if truncated - && !supplemental_cap_warned + if let Some(warning) = + supplemental_cap_warning(truncated, &mut supplemental_cap_warned) && let Some(callback) = &on_sync_error { - supplemental_cap_warned = true; - callback(format!( - "watching only the first {MAX_SUPPLEMENTAL_WATCHES} symlinked \ - directories; changes below the rest are picked up by the next \ - full sync" - )); + callback(warning); } known_dirs.retain(|dir| !link_state.watched_dirs.contains(dir)); known_dirs.extend(next_links.watched_dirs.iter().cloned()); @@ -4057,4 +4083,68 @@ mod tests { ); assert!(removed, "removing a link schedules a full sync"); } + + #[test] + fn symlink_cap_warning_is_given_once_per_watcher() { + let mut warned = false; + assert!(supplemental_cap_warning(false, &mut warned).is_none()); + assert!(supplemental_cap_warning(true, &mut warned).is_some()); + assert!(supplemental_cap_warning(true, &mut warned).is_none()); + // A watcher that warned at startup never warns again in its loop. + let mut warned_at_startup = true; + assert!(supplemental_cap_warning(true, &mut warned_at_startup).is_none()); + } + + /// A real directory moved in with a link (and sources) inside adds no + /// pending file of its own; the full sync it owes must still run. + #[test] + fn symlink_moved_in_directory_holding_a_link_schedules_a_full_sync() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-symlink-moved-in"); + let staging = LinkTarget::new("moved-in-staging"); + let target = LinkTarget::new("moved-in-target"); + fs::write(target.0.join("t.ts"), "export const t = 1;\n").unwrap(); + fs::create_dir_all(staging.0.join("pkg")).unwrap(); + fs::write(staging.0.join("pkg/a.ts"), "export const a = 1;\n").unwrap(); + if !symlink_at(&target.0, &staging.0.join("pkg/linked"), true) { + return; + } + let (watcher, _, full) = recording_watcher(dir.path()); + fs::rename(staging.0.join("pkg"), dir.path().join("pkg")).unwrap(); + let scheduled = wait_until(|| full.load(AtomicOrdering::SeqCst) >= 1); + watcher.stop(); + assert!( + scheduled, + "a moved-in directory holding a link schedules a full sync" + ); + } + + #[test] + fn symlink_file_target_turning_into_a_directory_schedules_a_full_sync() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-symlink-retype"); + fs::create_dir_all(dir.path().join("src/real")).unwrap(); + fs::write(dir.path().join("src/real/a.ts"), "export const a = 1;\n").unwrap(); + if !symlink_at( + &dir.path().join("src/real/a.ts"), + &dir.path().join("src/afile.ts"), + false, + ) { + return; + } + let (watcher, _, full) = recording_watcher(dir.path()); + fs::remove_file(dir.path().join("src/real/a.ts")).unwrap(); + fs::create_dir_all(dir.path().join("src/real/a.ts")).unwrap(); + fs::write( + dir.path().join("src/real/a.ts/inner.ts"), + "export const i = 1;\n", + ) + .unwrap(); + let scheduled = wait_until(|| full.load(AtomicOrdering::SeqCst) >= 1); + watcher.stop(); + assert!( + scheduled, + "a file link whose target became a directory now reaches a tree the scan follows" + ); + } }