//! Offline BM25 symbol index. //! //! Walks a project, extracts symbols with the tree-sitter [`crate::syntax`] outliner, and ranks //! them for `symbol_search` **without a language server** — across every tree-sitter-supported //! language at once. Names are tokenized on camelCase / snake_case / non-alphanumeric boundaries; //! ranking is exact → prefix → BM25 over the name tokens. //! //! The files come from [`luxe_core::walk`], the walk `glob` and `grep` use, so the index honors the //! same ignore files, sees them in sorted order, and says when it stopped short. use crate::protocol::symbol_kind_name; use crate::syntax; use std::collections::HashMap; use std::path::{Path, PathBuf}; const K1: f64 = 1.2; const B: f64 = 0.75; /// Directories never descended into, ignore files or not: vendored and generated trees. Hidden /// ones (`.venv`, `.luxe`, `.yarn`) the walk already skips. const IGNORED_DIRS: &[&str] = &["target", "node_modules", "dist", "build", "vendor"]; /// Safety caps so a pathological repo can't blow up the walk. pub(crate) const MAX_FILES: usize = 5000; const MAX_SYMBOLS: usize = 50_000; /// One indexed symbol (project-relative). #[derive(Debug, Clone)] pub struct IndexedSymbol { /// Symbol name. pub name: String, /// LSP `SymbolKind` integer. pub kind: u32, /// Project-relative file path. pub path: String, /// 1-based line of the symbol's name. pub line: u32, /// Enclosing container/parent name, if any. pub container: Option, } /// A built BM25 index over a project's symbols. pub struct SymbolIndex { symbols: Vec, tokens: Vec>, df: HashMap, avgdl: f64, capped: bool, } impl SymbolIndex { /// Build the index by walking `root` and outlining every supported source file. /// /// Blocking (filesystem + parsing); call from `spawn_blocking`. pub fn build(root: &Path) -> Self { let SourceFiles { files, capped } = source_files(root); let mut symbols = Vec::new(); for file in files { if symbols.len() >= MAX_SYMBOLS { break; } let Ok(source) = std::fs::read_to_string(&file) else { continue }; let Some(outline) = syntax::outline(&file, &source) else { continue }; let rel = luxe_core::paths::display(file.strip_prefix(root).unwrap_or(&file)); for sym in outline { symbols.push(IndexedSymbol { name: sym.name, kind: sym.kind, path: rel.clone(), line: sym.line + 1, container: sym.container, }); } } let tokens: Vec> = symbols.iter().map(|s| tokenize(&s.name)).collect(); let mut df: HashMap = HashMap::new(); for doc in &tokens { for term in doc.iter().collect::>() { *df.entry(term.clone()).or_insert(0) += 1; } } let total: usize = tokens.iter().map(Vec::len).sum(); let avgdl = if tokens.is_empty() { 1.0 } else { total as f64 / tokens.len() as f64 }; Self { symbols, tokens, df, avgdl, capped } } /// The line a search adds when the walk stopped at its file limit, so a symbol past it is /// missing, not absent; or nothing. pub fn capped_note(&self) -> String { if !self.capped { return String::new(); } format!(" (index capped at {MAX_FILES} files; a symbol past it is missing, not absent)\n") } /// Whether the index found no symbols (e.g. no tree-sitter-supported files). pub fn is_empty(&self) -> bool { self.symbols.is_empty() } /// Total indexed symbols. pub fn len(&self) -> usize { self.symbols.len() } /// Ranked search: exact name → prefix → BM25 over name tokens. Returns up to `limit` hits. pub fn search(&self, query: &str, limit: usize) -> Vec<&IndexedSymbol> { self.search_with_prior(query, limit, |_| 1.0) } /// Like [`SymbolIndex::search`] but multiplies each hit's score by `prior(symbol)` (clamped to /// `>= 1.0`) before ranking — so a caller can fold in a graph-centrality prior (rank central /// symbols higher) without the index having to know about the graph. Tiers (exact → prefix → /// token) still dominate; the prior only reorders within a tier. pub fn search_with_prior( &self, query: &str, limit: usize, prior: impl Fn(&IndexedSymbol) -> f64, ) -> Vec<&IndexedSymbol> { let query_lower = query.to_lowercase(); let query_tokens = unique(tokenize(query)); let n = self.symbols.len() as f64; let mut scored: Vec<(u8, f64, usize)> = self .symbols .iter() .enumerate() .filter_map(|(index, sym)| { let name_lower = sym.name.to_lowercase(); let tier = if name_lower == query_lower { 0 } else if name_lower.starts_with(&query_lower) { 1 } else { 2 }; let score = self.bm25(&query_tokens, index, n); // Keep exact/prefix hits regardless; keep token hits only if they scored. The prior // (>= 1.0) can't resurrect a zero BM25 score, so token filtering is unchanged. (tier < 2 || score > 0.0).then_some((tier, score * prior(sym).max(1.0), index)) }) .collect(); scored.sort_by(|a, b| { a.0.cmp(&b.0) .then(b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)) .then_with(|| self.symbols[a.2].name.cmp(&self.symbols[b.2].name)) }); scored.into_iter().take(limit).map(|(_, _, index)| &self.symbols[index]).collect() } fn bm25(&self, query_tokens: &[String], doc: usize, n: f64) -> f64 { let doc_tokens = &self.tokens[doc]; let dl = doc_tokens.len() as f64; let mut score = 0.0; for term in query_tokens { let tf = doc_tokens.iter().filter(|t| *t == term).count() as f64; if tf == 0.0 { continue; } let df = self.df.get(term).copied().unwrap_or(0) as f64; let idf = ((n - df + 0.5) / (df + 0.5) + 1.0).ln(); score += idf * (tf * (K1 + 1.0)) / (tf + K1 * (1.0 - B + B * dl / self.avgdl)); } score } } pub(crate) fn format_hit(sym: &IndexedSymbol) -> String { let container = sym.container.as_deref().map(|c| format!(" in {c}")).unwrap_or_default(); format!("{} ({}) {}:{}{container}", sym.name, symbol_kind_name(sym.kind), sym.path, sym.line) } /// The source files an index or graph reads, and whether there were more than it would take. pub(crate) struct SourceFiles { pub(crate) files: Vec, pub(crate) capped: bool, } /// Tree-sitter-supported source files under `root`, sorted, at most [`MAX_FILES`] of them. Shared /// with the code graph so both see the same file set. pub(crate) fn source_files(root: &Path) -> SourceFiles { let walked = luxe_core::walk::index_files(root, IGNORED_DIRS, syntax::supported, MAX_FILES); let files = walked.files.into_iter().map(|path| root.join(path)).collect(); SourceFiles { files, capped: walked.capped } } /// Tokenize an identifier on camelCase / snake_case / non-alphanumeric boundaries, lowercased. fn tokenize(text: &str) -> Vec { let mut tokens = Vec::new(); let mut current = String::new(); let mut prev_lower_or_digit = false; for ch in text.chars() { if ch.is_alphanumeric() { if ch.is_uppercase() && prev_lower_or_digit && !current.is_empty() { tokens.push(std::mem::take(&mut current).to_lowercase()); } current.push(ch); prev_lower_or_digit = ch.is_lowercase() || ch.is_numeric(); } else { if !current.is_empty() { tokens.push(std::mem::take(&mut current).to_lowercase()); } prev_lower_or_digit = false; } } if !current.is_empty() { tokens.push(current.to_lowercase()); } tokens } fn unique(mut tokens: Vec) -> Vec { tokens.sort(); tokens.dedup(); tokens } /// Why there is no index to search. #[derive(Debug, thiserror::Error)] pub enum IndexError { /// The build panicked or was cancelled. #[error("the symbol index could not be built: {0}")] Build(#[source] tokio::task::JoinError), } /// The warm index for one project root, rebuilt when the root changes or a file does. /// /// Shared between the `symbol_search` tool and the file-changed hook, which is what keeps a symbol /// the agent just wrote findable on the next search. #[derive(Default)] pub struct SymbolIndexCache { built: tokio::sync::Mutex)>>, } impl SymbolIndexCache { /// The index for `root`, built if there is none for it yet or `refresh` asks. pub async fn get( &self, root: &Path, refresh: bool, ) -> Result, IndexError> { let mut built = self.built.lock().await; if let Some((indexed, index)) = built.as_ref().filter(|_| !refresh) { if indexed == root { return Ok(std::sync::Arc::clone(index)); } } let owned = root.to_path_buf(); let index = tokio::task::spawn_blocking(move || SymbolIndex::build(&owned)) .await .map_err(IndexError::Build)?; let index = std::sync::Arc::new(index); *built = Some((root.to_path_buf(), std::sync::Arc::clone(&index))); Ok(index) } /// Forgets the index, so the next search rebuilds it from what is on disk now. pub async fn invalidate(&self) { *self.built.lock().await = None; } } #[cfg(test)] mod tests;