Initial public release

Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
Evan Whittaker
2026-10-17 12:08:30 -05:00
commit 09a7de2520
1730 changed files with 412952 additions and 0 deletions

View File

@@ -0,0 +1,267 @@
//! Offline BM25 symbol index.
//!
//! Walks a project, extracts symbols with the tree-sitter [`crate::syntax`] outliner, and ranks
//! them for `symbol_search` **without a language server** — across every tree-sitter-supported
//! language at once. Names are tokenized on camelCase / snake_case / non-alphanumeric boundaries;
//! ranking is exact → prefix → BM25 over the name tokens.
//!
//! The files come from [`luxe_core::walk`], the walk `glob` and `grep` use, so the index honors the
//! same ignore files, sees them in sorted order, and says when it stopped short.
use crate::protocol::symbol_kind_name;
use crate::syntax;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
const K1: f64 = 1.2;
const B: f64 = 0.75;
/// Directories never descended into, ignore files or not: vendored and generated trees. Hidden
/// ones (`.venv`, `.luxe`, `.yarn`) the walk already skips.
const IGNORED_DIRS: &[&str] = &["target", "node_modules", "dist", "build", "vendor"];
/// Safety caps so a pathological repo can't blow up the walk.
pub(crate) const MAX_FILES: usize = 5000;
const MAX_SYMBOLS: usize = 50_000;
/// One indexed symbol (project-relative).
#[derive(Debug, Clone)]
pub struct IndexedSymbol {
/// Symbol name.
pub name: String,
/// LSP `SymbolKind` integer.
pub kind: u32,
/// Project-relative file path.
pub path: String,
/// 1-based line of the symbol's name.
pub line: u32,
/// Enclosing container/parent name, if any.
pub container: Option<String>,
}
/// A built BM25 index over a project's symbols.
pub struct SymbolIndex {
symbols: Vec<IndexedSymbol>,
tokens: Vec<Vec<String>>,
df: HashMap<String, usize>,
avgdl: f64,
capped: bool,
}
impl SymbolIndex {
/// Build the index by walking `root` and outlining every supported source file.
///
/// Blocking (filesystem + parsing); call from `spawn_blocking`.
pub fn build(root: &Path) -> Self {
let SourceFiles { files, capped } = source_files(root);
let mut symbols = Vec::new();
for file in files {
if symbols.len() >= MAX_SYMBOLS {
break;
}
let Ok(source) = std::fs::read_to_string(&file) else { continue };
let Some(outline) = syntax::outline(&file, &source) else { continue };
let rel = luxe_core::paths::display(file.strip_prefix(root).unwrap_or(&file));
for sym in outline {
symbols.push(IndexedSymbol {
name: sym.name,
kind: sym.kind,
path: rel.clone(),
line: sym.line + 1,
container: sym.container,
});
}
}
let tokens: Vec<Vec<String>> = symbols.iter().map(|s| tokenize(&s.name)).collect();
let mut df: HashMap<String, usize> = HashMap::new();
for doc in &tokens {
for term in doc.iter().collect::<std::collections::HashSet<_>>() {
*df.entry(term.clone()).or_insert(0) += 1;
}
}
let total: usize = tokens.iter().map(Vec::len).sum();
let avgdl = if tokens.is_empty() { 1.0 } else { total as f64 / tokens.len() as f64 };
Self { symbols, tokens, df, avgdl, capped }
}
/// The line a search adds when the walk stopped at its file limit, so a symbol past it is
/// missing, not absent; or nothing.
pub fn capped_note(&self) -> String {
if !self.capped {
return String::new();
}
format!(" (index capped at {MAX_FILES} files; a symbol past it is missing, not absent)\n")
}
/// Whether the index found no symbols (e.g. no tree-sitter-supported files).
pub fn is_empty(&self) -> bool {
self.symbols.is_empty()
}
/// Total indexed symbols.
pub fn len(&self) -> usize {
self.symbols.len()
}
/// Ranked search: exact name → prefix → BM25 over name tokens. Returns up to `limit` hits.
pub fn search(&self, query: &str, limit: usize) -> Vec<&IndexedSymbol> {
self.search_with_prior(query, limit, |_| 1.0)
}
/// Like [`SymbolIndex::search`] but multiplies each hit's score by `prior(symbol)` (clamped to
/// `>= 1.0`) before ranking — so a caller can fold in a graph-centrality prior (rank central
/// symbols higher) without the index having to know about the graph. Tiers (exact → prefix →
/// token) still dominate; the prior only reorders within a tier.
pub fn search_with_prior(
&self,
query: &str,
limit: usize,
prior: impl Fn(&IndexedSymbol) -> f64,
) -> Vec<&IndexedSymbol> {
let query_lower = query.to_lowercase();
let query_tokens = unique(tokenize(query));
let n = self.symbols.len() as f64;
let mut scored: Vec<(u8, f64, usize)> = self
.symbols
.iter()
.enumerate()
.filter_map(|(index, sym)| {
let name_lower = sym.name.to_lowercase();
let tier = if name_lower == query_lower {
0
} else if name_lower.starts_with(&query_lower) {
1
} else {
2
};
let score = self.bm25(&query_tokens, index, n);
// Keep exact/prefix hits regardless; keep token hits only if they scored. The prior
// (>= 1.0) can't resurrect a zero BM25 score, so token filtering is unchanged.
(tier < 2 || score > 0.0).then_some((tier, score * prior(sym).max(1.0), index))
})
.collect();
scored.sort_by(|a, b| {
a.0.cmp(&b.0)
.then(b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal))
.then_with(|| self.symbols[a.2].name.cmp(&self.symbols[b.2].name))
});
scored.into_iter().take(limit).map(|(_, _, index)| &self.symbols[index]).collect()
}
fn bm25(&self, query_tokens: &[String], doc: usize, n: f64) -> f64 {
let doc_tokens = &self.tokens[doc];
let dl = doc_tokens.len() as f64;
let mut score = 0.0;
for term in query_tokens {
let tf = doc_tokens.iter().filter(|t| *t == term).count() as f64;
if tf == 0.0 {
continue;
}
let df = self.df.get(term).copied().unwrap_or(0) as f64;
let idf = ((n - df + 0.5) / (df + 0.5) + 1.0).ln();
score += idf * (tf * (K1 + 1.0)) / (tf + K1 * (1.0 - B + B * dl / self.avgdl));
}
score
}
}
pub(crate) fn format_hit(sym: &IndexedSymbol) -> String {
let container = sym.container.as_deref().map(|c| format!(" in {c}")).unwrap_or_default();
format!("{} ({}) {}:{}{container}", sym.name, symbol_kind_name(sym.kind), sym.path, sym.line)
}
/// The source files an index or graph reads, and whether there were more than it would take.
pub(crate) struct SourceFiles {
pub(crate) files: Vec<PathBuf>,
pub(crate) capped: bool,
}
/// Tree-sitter-supported source files under `root`, sorted, at most [`MAX_FILES`] of them. Shared
/// with the code graph so both see the same file set.
pub(crate) fn source_files(root: &Path) -> SourceFiles {
let walked = luxe_core::walk::index_files(root, IGNORED_DIRS, syntax::supported, MAX_FILES);
let files = walked.files.into_iter().map(|path| root.join(path)).collect();
SourceFiles { files, capped: walked.capped }
}
/// Tokenize an identifier on camelCase / snake_case / non-alphanumeric boundaries, lowercased.
fn tokenize(text: &str) -> Vec<String> {
let mut tokens = Vec::new();
let mut current = String::new();
let mut prev_lower_or_digit = false;
for ch in text.chars() {
if ch.is_alphanumeric() {
if ch.is_uppercase() && prev_lower_or_digit && !current.is_empty() {
tokens.push(std::mem::take(&mut current).to_lowercase());
}
current.push(ch);
prev_lower_or_digit = ch.is_lowercase() || ch.is_numeric();
} else {
if !current.is_empty() {
tokens.push(std::mem::take(&mut current).to_lowercase());
}
prev_lower_or_digit = false;
}
}
if !current.is_empty() {
tokens.push(current.to_lowercase());
}
tokens
}
fn unique(mut tokens: Vec<String>) -> Vec<String> {
tokens.sort();
tokens.dedup();
tokens
}
/// Why there is no index to search.
#[derive(Debug, thiserror::Error)]
pub enum IndexError {
/// The build panicked or was cancelled.
#[error("the symbol index could not be built: {0}")]
Build(#[source] tokio::task::JoinError),
}
/// The warm index for one project root, rebuilt when the root changes or a file does.
///
/// Shared between the `symbol_search` tool and the file-changed hook, which is what keeps a symbol
/// the agent just wrote findable on the next search.
#[derive(Default)]
pub struct SymbolIndexCache {
built: tokio::sync::Mutex<Option<(PathBuf, std::sync::Arc<SymbolIndex>)>>,
}
impl SymbolIndexCache {
/// The index for `root`, built if there is none for it yet or `refresh` asks.
pub async fn get(
&self,
root: &Path,
refresh: bool,
) -> Result<std::sync::Arc<SymbolIndex>, IndexError> {
let mut built = self.built.lock().await;
if let Some((indexed, index)) = built.as_ref().filter(|_| !refresh) {
if indexed == root {
return Ok(std::sync::Arc::clone(index));
}
}
let owned = root.to_path_buf();
let index = tokio::task::spawn_blocking(move || SymbolIndex::build(&owned))
.await
.map_err(IndexError::Build)?;
let index = std::sync::Arc::new(index);
*built = Some((root.to_path_buf(), std::sync::Arc::clone(&index)));
Ok(index)
}
/// Forgets the index, so the next search rebuilds it from what is on disk now.
pub async fn invalidate(&self) {
*self.built.lock().await = None;
}
}
#[cfg(test)]
mod tests;