Files
luxe/crates/luxe-tools/src/grep.rs
Evan Whittaker 933c1195b4 grep names each file once with its lines under it, the changes header keeps its position whole, and the bash teardown tests stop flaking on macOS
- grep groups its lines under each file, the layout of rg --heading, rather than repeating the
  path on every line: 9–28% fewer bytes than rg -n on the searches measured. scopes: true labels
  each run of matches with the function or type around it.
- The changes pane's header shortens the path rather than cut its position, and keeps a gap
  before the FILE / DIFF switch.
- The bash teardown tests count a loop still running as a survivor, not one write already under
  way when the kill landed.
2026-10-18 09:18:05 -05:00

482 lines
19 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! `grep` tool — search file contents by regex.
//!
//! Searches in-process: candidates come from [`crate::walk`], every one passes read policy before
//! it is opened, and the approved files are scanned in parallel with `regex`. An external `rg`
//! could not do the middle step, which is the one that keeps denied content out of the answer.
//!
//! The rows are bounded `file:line:text`, with optional context lines and a glob filter — or,
//! where the question is coarser than the lines, the files that match or a count per file. Lines
//! reach the model grouped under their file: the path once rather than on every line. Asked to
//! (`scopes`), each run of matches is labeled with the function or type around it, where an
//! outline can say, so where a match sits needs no read to find out.
use async_trait::async_trait;
use luxe_core::hooks::file_changed::{OutlineSource, Scope};
use luxe_core::{ContentBlock, Tool, ToolContext, ToolResultData, ToolSchema};
use serde_json::json;
use std::path::{Path, PathBuf};
use tracing::warn;
mod guarded;
mod present;
#[cfg(test)]
mod read_policy_tests;
/// JSON-schema for the grep tool arguments.
fn grep_schema() -> ToolSchema {
ToolSchema {
name: "grep".to_string(),
description: "Search file *contents* by regex. Returns each matching file's path once \
with its `line:text` rows under it; `scopes: true` also names the function \
or type around each match, as `[QuerySet › delete]`. Bounded \
— a broad pattern says it was capped instead of flooding your context. Case \
is sensitive unless the pattern says otherwise: prefix `(?i)`. Skips ignored \
paths (`.gitignore`, `.git/`, build output); set `all: true` to include \
them. Answer the coarse question coarsely: `outputMode: \"files\"` for \
which files match and `\"count\"` for how many lines each, both a fraction \
of the tokens of the lines themselves. Finding files by *name* is `glob`; \
finding a symbol's definition is `symbol_search`; finding everything that \
*uses* one \u{2014} callers, dependents, covering tests \u{2014} is `impact`, \
which reads a code graph rather than guessing at a pattern."
.to_string(),
parameters: json!({
"type": "object",
"properties": {
"pattern": {
"type": "string",
"description": "Regex pattern to search for. Prefix '(?i)' for a case-insensitive search."
},
"path": {
"type": "string",
"description": "File or directory to search (defaults to cwd)"
},
"glob": {
"type": "string",
"description": "Glob pattern to filter files (e.g., '*.rs')"
},
"outputMode": {
"type": "string",
"enum": ["content", "files", "count"],
"description": "What to return: 'content' (matching lines under each file, the default), \
'files' (one path per matching file), or 'count' (file:count \
of matching lines). Prefer 'files' when you only need to know \
where to look."
},
"limit": {
"type": "integer",
"description": "Cap the rows returned (default and maximum 200). Lower it when \
a sample will do — the equivalent of piping to 'head'."
},
"context": {
"type": "integer",
"description": "Context lines on both sides of each match (optional)"
},
"before": {
"type": "integer",
"description": "Context lines before each match; overrides 'context' on that side."
},
"after": {
"type": "integer",
"description": "Context lines after each match; overrides 'context' on that side."
},
"invert": {
"type": "boolean",
"description": "Return the lines that do NOT match the pattern. Default false."
},
"scopes": {
"type": "boolean",
"description": "Name the function or type each match sits in, for code a \
parser reads. Costs a row per change of scope; worth it \
when *where* a match is matters, as for call sites. \
Default false."
},
"all": {
"type": "boolean",
"description": "Include ignored files (build output, vendored deps). Default false."
}
},
"required": ["pattern"]
}),
}
}
/// Max rows returned to the model before the result is capped, and the ceiling on `limit`.
const GREP_MAX_LINES: usize = 200;
/// Character budget for one search, the second half of the bound the schema promises.
///
/// Sized against Luxe's own tree, where a broad pattern's 200 rows measure ~21 KB and the worst
/// measured — one reaching generated assets — came to 72 KB. So this leaves an ordinary search
/// untouched and cuts the pathological one to a third.
const GREP_MAX_CHARS: usize = 24_000;
/// Bytes of a file read before deciding it is binary and skipping it.
///
/// A NUL in the first few kilobytes is what every grep-alike uses, and it is right often enough:
/// the alternative is dumping a shared object's bytes into the context window.
const BINARY_SNIFF_BYTES: usize = 8192;
/// What a search answers with.
///
/// The lines are the expensive answer and often not the question: "which files mention this" and
/// "how many are there" each cost a fraction of the tokens, and a model that has to read 200 rows
/// to learn six filenames pays for the other 194.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
enum OutputMode {
/// `file:line:text`
#[default]
Content,
/// One path per file with at least one match
Files,
/// `file:count` of matching lines
Count,
}
impl OutputMode {
/// Parse the schema's spelling, falling back to the default for anything unrecognized.
fn parse(value: Option<&str>) -> Self {
match value {
Some("files") => Self::Files,
Some("count") => Self::Count,
_ => Self::Content,
}
}
/// What the cap counts, for the note it appends.
fn unit(self) -> &'static str {
match self {
Self::Content => "matching lines",
Self::Files | Self::Count => "files",
}
}
/// How to see the rest, which for the expensive mode includes the cheaper ones.
fn hint(self) -> &'static str {
match self {
Self::Content => {
"Narrow the pattern, path, or glob, or use outputMode 'files' to see which files \
match."
}
Self::Files | Self::Count => "Narrow the pattern, path, or glob to see the rest.",
}
}
}
/// One row a search found, kept whole until the answer is laid out.
#[derive(Debug, Clone)]
struct Hit {
/// The file, relative to where the search ran
file: std::sync::Arc<str>,
kind: HitKind,
/// 1-based line, for a match or a context row
line: usize,
/// The line itself, or the number of matching lines for [`HitKind::Count`]
text: String,
/// The scopes around a match, outermost first, `QuerySet › delete`, where an outline can say
scope: Option<String>,
}
/// What a [`Hit`] is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HitKind {
Match,
Context,
/// A file with a match, in the files mode
File,
/// A file's number of matching lines, in the count mode
Count,
}
impl Hit {
/// The row as ripgrep prints it, which the caps measure: `file:line:text`, `file-line-text`
/// for context, a bare path, or `file:count`.
fn flat(&self) -> String {
match self.kind {
HitKind::Match => format!("{}:{}:{}", self.file, self.line, self.text),
HitKind::Context => format!("{}-{}-{}", self.file, self.line, self.text),
HitKind::File => self.file.to_string(),
HitKind::Count => format!("{}:{}", self.file, self.text),
}
}
}
/// One parsed search.
///
/// `limit` is deliberately *not* consulted while searching. Stopping early would make the total the
/// result reports a count of what was collected rather than what exists — "5 more matching lines
/// not shown (6 total)" for a pattern with three hundred hits.
#[derive(Debug, Clone)]
struct Search<'a> {
pattern: &'a str,
path: &'a Path,
glob: Option<&'a str>,
before: usize,
after: usize,
all: bool,
invert: bool,
/// Label each match with the scopes around it, where an outline can read the file
scopes: bool,
mode: OutputMode,
limit: usize,
}
/// What `search` finds under `cwd`, unguarded and as ripgrep would print it, as the tests ask it.
#[cfg(test)]
fn native_search(cwd: &Path, search: &Search<'_>) -> Result<Vec<String>, String> {
let paths = crate::walk::files(cwd, search.path, search.all);
let hits = native_search_files(cwd, search, paths, None, || false)?;
Ok(hits.iter().map(Hit::flat).collect())
}
/// Scans `paths` (relative to `cwd`) for `search`, in parallel, rows in path order, each match
/// labeled with its scopes where `outline` can parse the file.
///
/// Each worker takes a contiguous run of the sorted paths and the runs are joined in order, so the
/// answer reads the same however many threads produced it. `stopped` is checked between files.
fn native_search_files(
cwd: &Path,
search: &Search<'_>,
paths: Vec<PathBuf>,
outline: Option<&dyn OutlineSource>,
stopped: impl Fn() -> bool + Sync,
) -> Result<Vec<Hit>, String> {
let regex = regex::Regex::new(search.pattern)
.map_err(|error| format!("'{}' is not a valid regex: {error}", search.pattern))?;
let prefilter = prefilter(search);
let filter = search
.glob
.map(|glob| {
crate::walk::matcher(glob)
.map_err(|error| format!("'{glob}' is not a valid glob: {error}"))
})
.transpose()?;
let paths: Vec<PathBuf> = paths
.into_iter()
.filter(|path| filter.as_ref().is_none_or(|filter| filter.is_match(path)))
.collect();
let workers = std::thread::available_parallelism().map_or(1, usize::from).min(16);
let run = paths.len().div_ceil(workers).max(1);
// a worker is `Err` when it panicked, `Ok(None)` when it was stopped
let scanned: Vec<std::thread::Result<Option<Vec<Hit>>>> = std::thread::scope(|scope| {
let handles: Vec<_> = paths
.chunks(run)
.map(|chunk| {
let (regex, prefilter, stopped) = (&regex, prefilter.as_ref(), &stopped);
scope.spawn(move || {
let mut rows = Vec::new();
for relative in chunk {
if stopped() {
return None;
}
let matcher = (regex, prefilter);
rows.extend(search_file(cwd, relative, matcher, search, outline));
}
Some(rows)
})
})
.collect();
handles.into_iter().map(|handle| handle.join()).collect()
});
combine(scanned, stopped())
}
/// The workers' rows joined in order; a crashed worker is a failure, a stopped one a cancel.
fn combine(
scanned: Vec<std::thread::Result<Option<Vec<Hit>>>>,
stopped: bool,
) -> Result<Vec<Hit>, String> {
if scanned.iter().any(Result::is_err) {
return Err("a search worker failed; nothing it scanned is reported".into());
}
if stopped || scanned.iter().any(|worker| matches!(worker, Ok(None))) {
return Err("grep cancelled".into());
}
Ok(scanned.into_iter().flatten().flatten().flatten().collect())
}
/// A whole-text test that can only rule a file out, never in: the pattern with `^`/`$` at every
/// line end, CRLF included, as matching line by line reads them.
///
/// `None` where it could rule out a file that matches: an inverted search; an anchor to the start
/// or end of the text, which a line has and a whole file has only once; and an inline flag that
/// turns a mode off, which can undo the line semantics. A file with a lone `\r` skips it too, in
/// [`search_file`], since CRLF mode keeps `.` off a `\r` that a line still holds.
fn prefilter(search: &Search<'_>) -> Option<regex::Regex> {
let pattern = search.pattern;
if search.invert
|| pattern.contains("\\A")
|| pattern.contains("\\z")
|| turns_off_a_flag(pattern)
{
return None;
}
regex::RegexBuilder::new(pattern).multi_line(true).crlf(true).build().ok()
}
/// Whether `pattern` has an inline group like `(?-m)` or `(?i-s:…)`.
fn turns_off_a_flag(pattern: &str) -> bool {
pattern.match_indices("(?").any(|(at, _)| {
let flags = &pattern[at + 2..];
let end = flags.find([')', ':']).unwrap_or(flags.len());
flags[..end].contains('-')
})
}
/// Whether `text` holds a `\r` that does not end a line, which a line keeps inside it.
fn has_lone_carriage_return(text: &str) -> bool {
let bytes = text.as_bytes();
bytes.iter().enumerate().any(|(at, byte)| *byte == b'\r' && bytes.get(at + 1) != Some(&b'\n'))
}
/// The rows one file contributes: its matching and context lines, or its name or count for the
/// coarser modes. A match is labeled with the scopes around it where `outline` parses the file.
fn search_file(
cwd: &Path,
relative: &Path,
(regex, prefilter): (&regex::Regex, Option<&regex::Regex>),
search: &Search<'_>,
outline: Option<&dyn OutlineSource>,
) -> Vec<Hit> {
let Ok(bytes) = std::fs::read(cwd.join(relative)) else { return Vec::new() };
if bytes.iter().take(BINARY_SNIFF_BYTES).any(|byte| *byte == 0) {
return Vec::new();
}
let Ok(text) = String::from_utf8(bytes) else { return Vec::new() };
// most files hold no match, and one pass over the whole text is far cheaper than per line
let filtered = prefilter.filter(|_| !has_lone_carriage_return(&text));
if filtered.is_some_and(|prefilter| !prefilter.is_match(&text)) {
return Vec::new();
}
let file: std::sync::Arc<str> = luxe_core::paths::display(relative).into();
let lines: Vec<&str> = text.lines().collect();
let matched: Vec<usize> = lines
.iter()
.enumerate()
.filter(|(_, line)| regex.is_match(line) != search.invert)
.map(|(index, _)| index)
.collect();
if matched.is_empty() {
return Vec::new();
}
let row = |kind, line, text: String| Hit { file: file.clone(), kind, line, text, scope: None };
match search.mode {
OutputMode::Files => vec![row(HitKind::File, 0, String::new())],
OutputMode::Count => vec![row(HitKind::Count, 0, matched.len().to_string())],
OutputMode::Content => {
let outline = outline.filter(|_| search.scopes);
let scopes = outline.and_then(|outline| outline.scopes(relative, &text));
line_hits(&lines, &matched, search)
.map(|(index, is_match)| {
let kind = if is_match { HitKind::Match } else { HitKind::Context };
let mut hit = row(kind, index + 1, lines[index].to_string());
hit.scope =
scopes.as_deref().filter(|_| is_match).and_then(|s| scope_label(s, index));
hit
})
.collect()
}
}
}
/// The lines to show and whether each is a match, in order.
///
/// Decides *which* lines to print and *what each one is* separately, then yields each once. Doing
/// it per match instead conflates the two: a match inside an earlier match's trailing context gets
/// emitted as context and never as the answer.
fn line_hits<'a>(
lines: &'a [&str],
matched: &'a [usize],
search: &Search<'_>,
) -> impl Iterator<Item = (usize, bool)> + 'a {
let mut shown = vec![false; lines.len()];
for index in matched {
let first = index.saturating_sub(search.before);
let last = (index + search.after + 1).min(lines.len());
shown[first..last].iter_mut().for_each(|slot| *slot = true);
}
(0..lines.len())
.filter(move |index| shown[*index])
.map(move |index| (index, matched.binary_search(&index).is_ok()))
}
/// The scopes around 0-based line `line`, outermost first, as `QuerySet › delete`.
///
/// A scope whose declaration starts on the line is left out: the line itself names it, so
/// `def delete(self):` is labeled `QuerySet`, not `QuerySet › delete`.
fn scope_label(scopes: &[Scope], line: usize) -> Option<String> {
let mut around: Vec<&Scope> =
scopes.iter().filter(|scope| scope.start_line < line && line <= scope.end_line).collect();
around.sort_by_key(|scope| (scope.start_line, std::cmp::Reverse(scope.end_line)));
let names: Vec<&str> = around.iter().map(|scope| scope.name.as_str()).collect();
(!names.is_empty()).then(|| names.join(" › "))
}
/// The `grep` tool implementation.
pub struct GrepTool {
/// Shared read tracker; a file whose lines a search printed is recorded as read
tracker: Option<std::sync::Arc<luxe_core::read_tracker::ReadTracker>>,
/// What names the scopes a match sits in; without it lines come unlabeled
outline: Option<std::sync::Arc<dyn OutlineSource>>,
}
impl GrepTool {
/// Create a new grep tool instance.
pub fn new() -> Self {
Self { tracker: None, outline: None }
}
/// Attach the outline that labels each match with the function or type around it, a parser
/// that lives outside this crate.
pub fn with_outline(mut self, outline: std::sync::Arc<dyn OutlineSource>) -> Self {
self.outline = Some(outline);
self
}
/// Attach the shared read tracker, so the lines a search showed satisfy the edit guards the
/// way a `read` of them would.
pub fn with_tracker(
mut self,
tracker: std::sync::Arc<luxe_core::read_tracker::ReadTracker>,
) -> Self {
self.tracker = Some(tracker);
self
}
}
impl Default for GrepTool {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Tool for GrepTool {
fn name(&self) -> &str {
"grep"
}
fn schema(&self) -> ToolSchema {
grep_schema()
}
async fn run_with_read_hook(
&self,
args: serde_json::Value,
cx: &ToolContext,
read_hook: &dyn luxe_core::tool::FileReadHook,
) -> ToolResultData {
guarded::run(self, args, cx, read_hook).await
}
async fn run(&self, args: serde_json::Value, cx: &ToolContext) -> ToolResultData {
guarded::run(self, args, cx, &crate::NoReadPolicy).await
}
}
#[cfg(test)]
mod scope_tests;
#[cfg(test)]
mod search_tests;
#[cfg(test)]
mod tests;