grep names each file once with its lines under it, the changes header keeps its position whole, and the bash teardown tests stop flaking on macOS

- grep groups its lines under each file, the layout of rg --heading, rather than repeating the
  path on every line: 9–28% fewer bytes than rg -n on the searches measured. scopes: true labels
  each run of matches with the function or type around it.
- The changes pane's header shortens the path rather than cut its position, and keeps a gap
  before the FILE / DIFF switch.
- The bash teardown tests count a loop still running as a survivor, not one write already under
  way when the kill landed.
This commit is contained in:
Evan Whittaker
2026-10-18 09:18:05 -05:00
parent 7b460ab9c7
commit 933c1195b4
23 changed files with 673 additions and 163 deletions

View File

@@ -442,10 +442,18 @@ mod tree_teardown {
format!("(for _ in $(seq 1 40); do touch {}; sleep 0.25; done) & wait", marker.display())
}
/// Whether anything is still touching `marker` after it is removed.
/// Whether anything is still touching `marker`, the loop, once a touch that was already under
/// way has had its moment.
///
/// The kill reaches the whole group at once, but a `touch` forked as it lands can finish one
/// write after it, and on macOS's slower teardown that write can follow the first clear. One
/// straggling write is not a survivor; a loop still running touches again within a quarter
/// second of the second clear.
async fn survivor_still_running(marker: &std::path::Path) -> bool {
let _ = std::fs::remove_file(marker);
tokio::time::sleep(Duration::from_millis(1_200)).await;
tokio::time::sleep(Duration::from_millis(300)).await;
let _ = std::fs::remove_file(marker);
tokio::time::sleep(Duration::from_millis(1_000)).await;
marker.exists()
}

View File

@@ -4,16 +4,21 @@
//! it is opened, and the approved files are scanned in parallel with `regex`. An external `rg`
//! could not do the middle step, which is the one that keeps denied content out of the answer.
//!
//! The result is bounded `file:line:text`, with optional context lines and a glob filter — or,
//! where the question is coarser than the lines, the files that match or a count per file.
//! The rows are bounded `file:line:text`, with optional context lines and a glob filter — or,
//! where the question is coarser than the lines, the files that match or a count per file. Lines
//! reach the model grouped under their file: the path once rather than on every line. Asked to
//! (`scopes`), each run of matches is labeled with the function or type around it, where an
//! outline can say, so where a match sits needs no read to find out.
use async_trait::async_trait;
use luxe_core::hooks::file_changed::{OutlineSource, Scope};
use luxe_core::{ContentBlock, Tool, ToolContext, ToolResultData, ToolSchema};
use serde_json::json;
use std::path::{Path, PathBuf};
use tracing::warn;
mod guarded;
mod present;
#[cfg(test)]
mod read_policy_tests;
@@ -21,7 +26,9 @@ mod read_policy_tests;
fn grep_schema() -> ToolSchema {
ToolSchema {
name: "grep".to_string(),
description: "Search file *contents* by regex. Returns `file:line:text`, bounded \
description: "Search file *contents* by regex. Returns each matching file's path once \
with its `line:text` rows under it; `scopes: true` also names the function \
or type around each match, as `[QuerySet › delete]`. Bounded \
— a broad pattern says it was capped instead of flooding your context. Case \
is sensitive unless the pattern says otherwise: prefix `(?i)`. Skips ignored \
paths (`.gitignore`, `.git/`, build output); set `all: true` to include \
@@ -50,7 +57,7 @@ fn grep_schema() -> ToolSchema {
"outputMode": {
"type": "string",
"enum": ["content", "files", "count"],
"description": "What to return: 'content' (file:line:text, the default), \
"description": "What to return: 'content' (matching lines under each file, the default), \
'files' (one path per matching file), or 'count' (file:count \
of matching lines). Prefer 'files' when you only need to know \
where to look."
@@ -76,6 +83,13 @@ fn grep_schema() -> ToolSchema {
"type": "boolean",
"description": "Return the lines that do NOT match the pattern. Default false."
},
"scopes": {
"type": "boolean",
"description": "Name the function or type each match sits in, for code a \
parser reads. Costs a row per change of scope; worth it \
when *where* a match is matters, as for call sites. \
Default false."
},
"all": {
"type": "boolean",
"description": "Include ignored files (build output, vendored deps). Default false."
@@ -148,6 +162,44 @@ impl OutputMode {
}
}
/// One row a search found, kept whole until the answer is laid out.
#[derive(Debug, Clone)]
struct Hit {
/// The file, relative to where the search ran
file: std::sync::Arc<str>,
kind: HitKind,
/// 1-based line, for a match or a context row
line: usize,
/// The line itself, or the number of matching lines for [`HitKind::Count`]
text: String,
/// The scopes around a match, outermost first, `QuerySet › delete`, where an outline can say
scope: Option<String>,
}
/// What a [`Hit`] is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HitKind {
Match,
Context,
/// A file with a match, in the files mode
File,
/// A file's number of matching lines, in the count mode
Count,
}
impl Hit {
/// The row as ripgrep prints it, which the caps measure: `file:line:text`, `file-line-text`
/// for context, a bare path, or `file:count`.
fn flat(&self) -> String {
match self.kind {
HitKind::Match => format!("{}:{}:{}", self.file, self.line, self.text),
HitKind::Context => format!("{}-{}-{}", self.file, self.line, self.text),
HitKind::File => self.file.to_string(),
HitKind::Count => format!("{}:{}", self.file, self.text),
}
}
}
/// One parsed search.
///
/// `limit` is deliberately *not* consulted while searching. Stopping early would make the total the
@@ -162,17 +214,22 @@ struct Search<'a> {
after: usize,
all: bool,
invert: bool,
/// Label each match with the scopes around it, where an outline can read the file
scopes: bool,
mode: OutputMode,
limit: usize,
}
/// What `search` finds under `cwd`, unguarded, as the tests ask it.
/// What `search` finds under `cwd`, unguarded and as ripgrep would print it, as the tests ask it.
#[cfg(test)]
fn native_search(cwd: &Path, search: &Search<'_>) -> Result<Vec<String>, String> {
native_search_files(cwd, search, crate::walk::files(cwd, search.path, search.all), || false)
let paths = crate::walk::files(cwd, search.path, search.all);
let hits = native_search_files(cwd, search, paths, None, || false)?;
Ok(hits.iter().map(Hit::flat).collect())
}
/// Scans `paths` (relative to `cwd`) for `search`, in parallel, rows in path order.
/// Scans `paths` (relative to `cwd`) for `search`, in parallel, rows in path order, each match
/// labeled with its scopes where `outline` can parse the file.
///
/// Each worker takes a contiguous run of the sorted paths and the runs are joined in order, so the
/// answer reads the same however many threads produced it. `stopped` is checked between files.
@@ -180,8 +237,9 @@ fn native_search_files(
cwd: &Path,
search: &Search<'_>,
paths: Vec<PathBuf>,
outline: Option<&dyn OutlineSource>,
stopped: impl Fn() -> bool + Sync,
) -> Result<Vec<String>, String> {
) -> Result<Vec<Hit>, String> {
let regex = regex::Regex::new(search.pattern)
.map_err(|error| format!("'{}' is not a valid regex: {error}", search.pattern))?;
let prefilter = prefilter(search);
@@ -199,7 +257,7 @@ fn native_search_files(
let workers = std::thread::available_parallelism().map_or(1, usize::from).min(16);
let run = paths.len().div_ceil(workers).max(1);
// a worker is `Err` when it panicked, `Ok(None)` when it was stopped
let scanned: Vec<std::thread::Result<Option<Vec<String>>>> = std::thread::scope(|scope| {
let scanned: Vec<std::thread::Result<Option<Vec<Hit>>>> = std::thread::scope(|scope| {
let handles: Vec<_> = paths
.chunks(run)
.map(|chunk| {
@@ -210,7 +268,8 @@ fn native_search_files(
if stopped() {
return None;
}
rows.extend(search_file(cwd, relative, (regex, prefilter), search));
let matcher = (regex, prefilter);
rows.extend(search_file(cwd, relative, matcher, search, outline));
}
Some(rows)
})
@@ -223,9 +282,9 @@ fn native_search_files(
/// The workers' rows joined in order; a crashed worker is a failure, a stopped one a cancel.
fn combine(
scanned: Vec<std::thread::Result<Option<Vec<String>>>>,
scanned: Vec<std::thread::Result<Option<Vec<Hit>>>>,
stopped: bool,
) -> Result<Vec<String>, String> {
) -> Result<Vec<Hit>, String> {
if scanned.iter().any(Result::is_err) {
return Err("a search worker failed; nothing it scanned is reported".into());
}
@@ -269,14 +328,15 @@ fn has_lone_carriage_return(text: &str) -> bool {
bytes.iter().enumerate().any(|(at, byte)| *byte == b'\r' && bytes.get(at + 1) != Some(&b'\n'))
}
/// The rows one file contributes, in the shape ripgrep prints them: `file:line:text` for a match,
/// `file-line-text` for context, or the file's name or count for the coarser modes.
/// The rows one file contributes: its matching and context lines, or its name or count for the
/// coarser modes. A match is labeled with the scopes around it where `outline` parses the file.
fn search_file(
cwd: &Path,
relative: &Path,
(regex, prefilter): (&regex::Regex, Option<&regex::Regex>),
search: &Search<'_>,
) -> Vec<String> {
outline: Option<&dyn OutlineSource>,
) -> Vec<Hit> {
let Ok(bytes) = std::fs::read(cwd.join(relative)) else { return Vec::new() };
if bytes.iter().take(BINARY_SNIFF_BYTES).any(|byte| *byte == 0) {
return Vec::new();
@@ -287,7 +347,7 @@ fn search_file(
if filtered.is_some_and(|prefilter| !prefilter.is_match(&text)) {
return Vec::new();
}
let name = luxe_core::paths::display(relative);
let file: std::sync::Arc<str> = luxe_core::paths::display(relative).into();
let lines: Vec<&str> = text.lines().collect();
let matched: Vec<usize> = lines
.iter()
@@ -298,45 +358,78 @@ fn search_file(
if matched.is_empty() {
return Vec::new();
}
let row = |kind, line, text: String| Hit { file: file.clone(), kind, line, text, scope: None };
match search.mode {
OutputMode::Files => vec![name],
OutputMode::Count => vec![format!("{name}:{}", matched.len())],
OutputMode::Files => vec![row(HitKind::File, 0, String::new())],
OutputMode::Count => vec![row(HitKind::Count, 0, matched.len().to_string())],
OutputMode::Content => {
// Decide *which* lines to print and *what each one is* separately, then print each
// once. Doing it per match instead conflates the two: a match inside an earlier
// match's trailing context gets emitted as context and never as the answer.
let mut shown = vec![false; lines.len()];
for index in &matched {
let first = index.saturating_sub(search.before);
let last = (index + search.after + 1).min(lines.len());
shown[first..last].iter_mut().for_each(|slot| *slot = true);
}
let is_match = |index: usize| matched.binary_search(&index).is_ok();
lines
.iter()
.enumerate()
.filter(|(index, _)| shown[*index])
.map(|(index, line)| {
// `-` marks a context line; the caller's parsers and the model both read the
// `:` rows as the answer
let separator = if is_match(index) { ':' } else { '-' };
format!("{name}{separator}{}{separator}{line}", index + 1)
let outline = outline.filter(|_| search.scopes);
let scopes = outline.and_then(|outline| outline.scopes(relative, &text));
line_hits(&lines, &matched, search)
.map(|(index, is_match)| {
let kind = if is_match { HitKind::Match } else { HitKind::Context };
let mut hit = row(kind, index + 1, lines[index].to_string());
hit.scope =
scopes.as_deref().filter(|_| is_match).and_then(|s| scope_label(s, index));
hit
})
.collect()
}
}
}
/// The lines to show and whether each is a match, in order.
///
/// Decides *which* lines to print and *what each one is* separately, then yields each once. Doing
/// it per match instead conflates the two: a match inside an earlier match's trailing context gets
/// emitted as context and never as the answer.
fn line_hits<'a>(
lines: &'a [&str],
matched: &'a [usize],
search: &Search<'_>,
) -> impl Iterator<Item = (usize, bool)> + 'a {
let mut shown = vec![false; lines.len()];
for index in matched {
let first = index.saturating_sub(search.before);
let last = (index + search.after + 1).min(lines.len());
shown[first..last].iter_mut().for_each(|slot| *slot = true);
}
(0..lines.len())
.filter(move |index| shown[*index])
.map(move |index| (index, matched.binary_search(&index).is_ok()))
}
/// The scopes around 0-based line `line`, outermost first, as `QuerySet › delete`.
///
/// A scope whose declaration starts on the line is left out: the line itself names it, so
/// `def delete(self):` is labeled `QuerySet`, not `QuerySet › delete`.
fn scope_label(scopes: &[Scope], line: usize) -> Option<String> {
let mut around: Vec<&Scope> =
scopes.iter().filter(|scope| scope.start_line < line && line <= scope.end_line).collect();
around.sort_by_key(|scope| (scope.start_line, std::cmp::Reverse(scope.end_line)));
let names: Vec<&str> = around.iter().map(|scope| scope.name.as_str()).collect();
(!names.is_empty()).then(|| names.join(" › "))
}
/// The `grep` tool implementation.
pub struct GrepTool {
/// Shared read tracker; a file whose lines a search printed is recorded as read
tracker: Option<std::sync::Arc<luxe_core::read_tracker::ReadTracker>>,
/// What names the scopes a match sits in; without it lines come unlabeled
outline: Option<std::sync::Arc<dyn OutlineSource>>,
}
impl GrepTool {
/// Create a new grep tool instance.
pub fn new() -> Self {
Self { tracker: None }
Self { tracker: None, outline: None }
}
/// Attach the outline that labels each match with the function or type around it, a parser
/// that lives outside this crate.
pub fn with_outline(mut self, outline: std::sync::Arc<dyn OutlineSource>) -> Self {
self.outline = Some(outline);
self
}
/// Attach the shared read tracker, so the lines a search showed satisfy the edit guards the
@@ -380,6 +473,8 @@ impl Tool for GrepTool {
}
}
#[cfg(test)]
mod scope_tests;
#[cfg(test)]
mod search_tests;
#[cfg(test)]

View File

@@ -48,6 +48,7 @@ pub(super) fn parse_search(args: &serde_json::Value) -> Result<Search<'_>, Strin
after: side("after"),
all: args.get("all").and_then(|value| value.as_bool()).unwrap_or(false),
invert: args.get("invert").and_then(|value| value.as_bool()).unwrap_or(false),
scopes: args.get("scopes").and_then(|value| value.as_bool()).unwrap_or(false),
// camelCase is advertised; snake_case is what models reach for, and a silently ignored
// mode returns the expensive answer
mode: OutputMode::parse(
@@ -106,16 +107,15 @@ async fn scan(
args: serde_json::Value,
cx: &ToolContext,
paths: Vec<PathBuf>,
stopped: Arc<AtomicBool>,
) -> Result<String, String> {
(outline, stopped): (Option<Arc<dyn OutlineSource>>, Arc<AtomicBool>),
) -> Result<Vec<Hit>, String> {
let cwd = cx.cwd.clone();
let cancelled = cx.cancelled.clone();
tokio::task::spawn_blocking(move || {
let search = parse_search(&args)?;
super::native_search_files(&cwd, &search, paths, || {
super::native_search_files(&cwd, &search, paths, outline.as_deref(), || {
stopped.load(Ordering::Relaxed) || *cancelled.borrow()
})
.map(|rows| rows.join("\n"))
})
.await
.map_err(|error| format!("grep scan worker failed: {error}"))?
@@ -141,7 +141,7 @@ pub(super) async fn run(
Ok(plan) => plan,
Err(error) => return ToolResultData::error(error),
};
let found = scan(args.clone(), cx, plan.allowed, stopped).await;
let found = scan(args.clone(), cx, plan.allowed, (tool.outline.clone(), stopped)).await;
let mut result = finish(tool, cx, &search, found);
if plan.denied + plan.asks > 0 {
result.content.push(ContentBlock::Text(format!(
@@ -159,20 +159,24 @@ pub(super) fn finish(
tool: &GrepTool,
cx: &ToolContext,
search: &Search<'_>,
found: Result<String, String>,
found: Result<Vec<Hit>, String>,
) -> ToolResultData {
match found {
Ok(matches) if matches.trim().is_empty() => ToolResultData::text("No matches found"),
Ok(matches) => {
let bounded = crate::output_limit::cap_lines_within(
&matches,
Ok(hits) if hits.is_empty() => ToolResultData::text("No matches found"),
Ok(hits) if search.mode == OutputMode::Content => {
let (text, shown) = super::present::content(&hits, search);
record_matches(tool, cx, search, &hits[..shown]);
ToolResultData::text(text)
}
Ok(hits) => {
let rows: Vec<String> = hits.iter().map(Hit::flat).collect();
ToolResultData::text(crate::output_limit::cap_lines_within(
&rows.join("\n"),
search.limit,
GREP_MAX_CHARS,
search.mode.unit(),
search.mode.hint(),
);
record_matches(tool, cx, search, &bounded);
ToolResultData::text(bounded)
))
}
Err(problem) => {
warn!(pattern = search.pattern, error = %problem, "grep failed");
@@ -181,15 +185,18 @@ pub(super) fn finish(
}
}
fn record_matches(tool: &GrepTool, cx: &ToolContext, search: &Search<'_>, bounded: &str) {
let (Some(tracker), OutputMode::Content) = (&tool.tracker, search.mode) else { return };
/// Records as read every file a line of which the answer showed, the way a `read` of it would be.
fn record_matches(tool: &GrepTool, cx: &ToolContext, search: &Search<'_>, shown: &[Hit]) {
let Some(tracker) = &tool.tracker else { return };
let searched = cx.cwd.join(search.path);
let shown = if searched.is_file() {
vec![searched]
} else {
crate::bash::seen::files_in_rows(bounded, &cx.cwd)
let files: Vec<PathBuf> = match searched.is_file() {
true => vec![searched],
false => shown.iter().map(|hit| cx.cwd.join(&*hit.file)).collect(),
};
for path in shown {
tracker.record(&cx.session_id, &path);
let mut recorded = std::collections::HashSet::new();
for path in files {
if recorded.insert(path.clone()) {
tracker.record(&cx.session_id, &path);
}
}
}

View File

@@ -0,0 +1,106 @@
//! Present — a content search laid out for reading: each file named once, its lines under it, and
//! the scope around a run of matches where an outline can say.
//!
//! The rows are counted and capped as ripgrep would print them, `file:line:text`, so the bounds and
//! the totals are the ones the other modes keep. Only the layout changes. A path repeated on every
//! line is a large share of a broad search, and the function a match sits in is what the next
//! `read` would have gone to find out.
use super::{Hit, HitKind, Search, GREP_MAX_CHARS};
/// The capped, grouped answer, and how many of `hits` it shows.
pub(super) fn content(hits: &[Hit], search: &Search<'_>) -> (String, usize) {
let lengths = hits.iter().map(|hit| hit.flat().chars().count());
let kept = crate::output_limit::lines_within(lengths, search.limit, GREP_MAX_CHARS);
let shown = grouped(&hits[..kept], search.before + search.after > 0);
let text = match kept < hits.len() {
true => crate::output_limit::with_note(
&shown,
hits.len() - kept,
hits.len(),
search.mode.unit(),
search.mode.hint(),
),
false => crate::output_limit::clamp(&shown),
};
(text, kept)
}
/// `hits` under their files: the path, then `line:text` for a match and `line-text` for context,
/// ` [Scope › name]` before a match whose scope differs from the last one's, and `--` where
/// context leaves a gap.
fn grouped(hits: &[Hit], context: bool) -> String {
let mut out: Vec<String> = Vec::new();
let mut previous: Option<&Hit> = None;
let mut scope: Option<&str> = None;
for hit in hits {
match previous {
Some(last) if last.file == hit.file => {
if context && hit.line > last.line + 1 {
out.push("--".into());
}
}
_ => {
out.push(hit.file.to_string());
scope = None;
}
}
if hit.kind == HitKind::Match && hit.scope.as_deref() != scope {
out.push(scope_row(hit.scope.as_deref()));
scope = hit.scope.as_deref();
}
let separator = if hit.kind == HitKind::Match { ':' } else { '-' };
out.push(format!("{}{separator}{}", hit.line, hit.text));
previous = Some(hit);
}
out.join("\n")
}
/// ` [QuerySet › delete]`, or ` [top level]` for a match that has left the scopes before it.
fn scope_row(scope: Option<&str>) -> String {
format!(" [{}]", scope.unwrap_or("top level"))
}
#[cfg(test)]
mod tests {
use super::*;
fn hit(file: &str, kind: HitKind, line: usize, text: &str, scope: Option<&str>) -> Hit {
let (file, text, scope) = (file.into(), text.to_string(), scope.map(String::from));
Hit { file, kind, line, text, scope }
}
/// Each file is named once, its lines under it, and a scope row comes only where it changes.
#[test]
fn matches_group_under_their_file_with_a_row_per_change_of_scope() {
let hits = [
hit("a.py", HitKind::Match, 3, " def delete(self):", Some("QuerySet")),
hit("a.py", HitKind::Match, 9, " def delete_all(self):", Some("QuerySet")),
hit("a.py", HitKind::Match, 20, "def delete(): pass", None),
hit("b.py", HitKind::Match, 5, " delete()", Some("Model › save")),
];
let expected = "a.py\n [QuerySet]\n3: def delete(self):\n9: def delete_all(self):\n \
[top level]\n20:def delete(): pass\nb.py\n [Model › save]\n5: delete()";
assert_eq!(grouped(&hits, false), expected);
}
/// A file no outline can read gets no scope rows at all, rather than a `[top level]` that
/// would claim to know.
#[test]
fn an_unlabeled_file_gets_no_scope_rows() {
let hits = [hit("notes.md", HitKind::Match, 1, "delete it", None)];
assert_eq!(grouped(&hits, false), "notes.md\n1:delete it");
}
/// Context lines take `-`, and a gap between runs of context is marked `--`.
#[test]
fn context_rows_take_a_dash_and_a_gap_is_marked() {
let hits = [
hit("a.rs", HitKind::Match, 2, "needle", None),
hit("a.rs", HitKind::Context, 3, "after", None),
hit("a.rs", HitKind::Match, 9, "needle", None),
];
assert_eq!(grouped(&hits, true), "a.rs\n2:needle\n3-after\n--\n9:needle");
assert_eq!(grouped(&hits, false), "a.rs\n2:needle\n3-after\n9:needle", "no context asked");
}
}

View File

@@ -0,0 +1,77 @@
use super::*;
use luxe_core::hooks::file_changed::FunctionSize;
use std::sync::Arc;
use tokio::sync::{mpsc, watch};
/// A stand-in for the tree-sitter outline: in a `.py` file, a class over lines 1–9 (0-based 0–8)
/// with a method over lines 3–6, and nothing it can read in any other file.
struct OneClass;
impl OutlineSource for OneClass {
fn functions(&self, _: &Path, _: &str) -> Option<Vec<FunctionSize>> {
None
}
fn scopes(&self, path: &Path, _: &str) -> Option<Vec<Scope>> {
let scope =
|name: &str, start_line, end_line| Scope { name: name.into(), start_line, end_line };
(path.extension()? == "py").then(|| vec![scope("QuerySet", 0, 8), scope("delete", 2, 5)])
}
}
fn context(cwd: &Path) -> ToolContext {
let (event_tx, _event_rx) = mpsc::channel(64);
let (_cancel_tx, cancel_rx) = watch::channel(false);
ToolContext {
cwd: cwd.to_path_buf(),
session_id: "test-session".to_string(),
cancelled: Arc::new(cancel_rx),
event_tx,
hook_ctx: None,
question_broker: None,
}
}
/// Each match is labeled with the scopes around it, outermost first, and only where the label
/// changes. The definition line takes its parent's label, since it names itself; a file no outline
/// can read gets no label at all; and the path comes once, above its lines.
#[tokio::test]
async fn a_match_is_labeled_with_the_scopes_around_it() {
let dir = tempfile::tempdir().unwrap();
let python =
"class QuerySet:\n \n def delete(self):\n self.delete_batch()\n \
return self\n \n \nOTHER = 1\n \ndef delete():\n pass\n";
std::fs::write(dir.path().join("a.py"), python).unwrap();
std::fs::write(dir.path().join("notes.md"), "delete it\n").unwrap();
let tool = GrepTool::new().with_outline(Arc::new(OneClass));
let args = json!({"pattern": "delete", "scopes": true});
let result = tool.run(args, &context(dir.path())).await;
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
let expected = "a.py\n [QuerySet]\n3: def delete(self):\n [QuerySet › delete]\n4: \
self.delete_batch()\n [top level]\n10:def delete():\nnotes.md\n1:delete it";
assert_eq!(text, expected);
}
/// Labels cost a row each, so a search without `scopes` gets the grouped lines alone, outline or
/// not.
#[tokio::test]
async fn a_search_is_unlabeled_unless_it_asks() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.py"), "class A:\n \n def delete(self):\n").unwrap();
let tool = GrepTool::new().with_outline(Arc::new(OneClass));
let result = tool.run(json!({"pattern": "delete"}), &context(dir.path())).await;
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
assert_eq!(text, "a.py\n3: def delete(self):");
}
/// The coarse modes have no lines to label, so they ask nothing of the outline.
#[tokio::test]
async fn the_files_mode_carries_no_scopes() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.py"), "class A:\n def delete(self): pass\n").unwrap();
let tool = GrepTool::new().with_outline(Arc::new(OneClass));
let args = json!({"pattern": "delete", "outputMode": "files"});
let result = tool.run(args, &context(dir.path())).await;
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
assert_eq!(text.trim(), "a.py");
}

View File

@@ -17,6 +17,7 @@ fn base(pattern: &str) -> Search<'_> {
after: 0,
all: false,
invert: false,
scopes: false,
mode: OutputMode::Content,
limit: 500,
}
@@ -147,10 +148,20 @@ fn the_prefilter_never_hides_a_matching_line() {
/// A worker that crashed is a failure the caller hears about; one that was stopped is a cancel.
#[test]
fn a_crashed_worker_is_a_failure_not_a_cancel() {
let rows = |row: &str| Ok(Some(vec![row.to_string()]));
let crashed: std::thread::Result<Option<Vec<String>>> = Err(Box::new("boom"));
let rows = |file: &str| {
let hit = Hit {
file: file.into(),
kind: HitKind::File,
line: 0,
text: String::new(),
scope: None,
};
Ok(Some(vec![hit]))
};
let crashed: std::thread::Result<Option<Vec<Hit>>> = Err(Box::new("boom"));
let failed = combine(vec![rows("a"), crashed], false).unwrap_err();
assert!(failed.contains("worker failed"), "{failed}");
assert_eq!(combine(vec![rows("a"), Ok(None)], false).unwrap_err(), "grep cancelled");
assert_eq!(combine(vec![rows("a"), rows("b")], false).unwrap(), ["a", "b"]);
let joined = combine(vec![rows("a"), rows("b")], false).unwrap();
assert_eq!(joined.iter().map(Hit::flat).collect::<Vec<_>>(), ["a", "b"]);
}

View File

@@ -17,6 +17,13 @@ fn test_context(tmp_dir: &std::path::Path) -> ToolContext {
}
}
/// The matching-line rows of a content answer, `12:text`, not the paths or scope rows around them.
fn match_rows(text: &str) -> usize {
text.lines()
.filter(|line| line.split_once(':').is_some_and(|(n, _)| n.parse::<usize>().is_ok()))
.count()
}
/// A plain content search of `.`, for a test to vary one field of.
fn search<'a>(pattern: &'a str) -> Search<'a> {
Search {
@@ -27,6 +34,7 @@ fn search<'a>(pattern: &'a str) -> Search<'a> {
after: 0,
all: false,
invert: false,
scopes: false,
mode: OutputMode::Content,
limit: GREP_MAX_LINES,
}
@@ -246,7 +254,7 @@ async fn a_zero_limit_is_an_absent_limit() {
.run(json!({"pattern": "needle", "limit": 0}), &test_context(dir.path()))
.await;
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
assert_eq!(text.lines().count(), 20, "every match, not none: {text}");
assert_eq!(match_rows(text), 20, "every match, not none: {text}");
assert!(!text.contains("not shown"), "and nothing was cut: {text}");
}
@@ -282,16 +290,12 @@ async fn a_limit_can_only_tighten_the_cap() {
let sampled = GrepTool::new().run(json!({"pattern": "needle", "limit": 5}), &cx).await;
let ContentBlock::Text(text) = &sampled.content[0] else { panic!("expected text") };
assert_eq!(text.lines().filter(|line| line.contains("many.txt")).count(), 5, "{text}");
assert_eq!(match_rows(text), 5, "{text}");
assert!(text.contains("295 more matching lines not shown (300 total)"), "{text}");
let greedy = GrepTool::new().run(json!({"pattern": "needle", "limit": 5000}), &cx).await;
let ContentBlock::Text(text) = &greedy.content[0] else { panic!("expected text") };
assert_eq!(
text.lines().filter(|line| line.contains("many.txt")).count(),
GREP_MAX_LINES,
"a limit past the ceiling is the ceiling: {text}"
);
assert_eq!(match_rows(text), GREP_MAX_LINES, "a limit past the ceiling is the ceiling: {text}");
}
/// The count a capped result reports is the count that exists, not the count that was kept.

View File

@@ -49,6 +49,9 @@ pub mod write;
/// The `todo` tool requires a store root (e.g. `.luxe/todos`). Pass `None` to skip registering it.
/// `web` configures `web_search` and `web_fetch`; [`web::WebConfig::from_env`] is the usual start.
///
/// `outline` labels each `grep` match with the function or type around it; it is a parser this
/// crate cannot depend on (tree-sitter lives in `luxe-lsp`), and `None` leaves matches unlabeled.
///
/// Returns the shared [`luxe_core::read_tracker::ReadTracker`] backing the read-before-edit /
/// staleness / blind-write guards, so the caller can also wire it into the LSP funnel reads
/// (`read_symbol`, `module_report`) that see file content but live in a crate that can't depend
@@ -57,6 +60,7 @@ pub fn register_builtins(
registry: &mut luxe_core::ToolRegistry,
todo_store_root: Option<std::path::PathBuf>,
web: web::WebConfig,
outline: Option<std::sync::Arc<dyn luxe_core::hooks::file_changed::OutlineSource>>,
) -> std::sync::Arc<luxe_core::read_tracker::ReadTracker> {
use std::sync::Arc;
// One shared read tracker backs the read-before-edit + staleness + blind-write guards across
@@ -76,7 +80,11 @@ pub fn register_builtins(
registry.register(Arc::new(write::WriteTool::new().with_tracker(read_tracker.clone())));
registry.register(Arc::new(edit::EditTool::new().with_tracker(read_tracker.clone())));
registry.register(Arc::new(bash::BashTool::new().with_tracker(read_tracker.clone())));
registry.register(Arc::new(grep::GrepTool::new().with_tracker(read_tracker.clone())));
let grep = grep::GrepTool::new().with_tracker(read_tracker.clone());
registry.register(Arc::new(match outline {
Some(outline) => grep.with_outline(outline),
None => grep,
}));
registry.register(Arc::new(glob::GlobTool::new()));
registry.register(Arc::new(question::QuestionTool::new(question::QuestionConfig::default())));
registry.register(Arc::new(skill::SkillTool::new(skill::SkillConfig)));

View File

@@ -37,24 +37,54 @@ pub(crate) fn cap_lines_within(
hint: &str,
) -> String {
let total = text.lines().count();
let mut kept_lines = 0usize;
let mut spent = 0usize;
for line in text.lines().take(max_lines) {
let cost = line.chars().count().saturating_add(1);
if kept_lines > 0 && spent.saturating_add(cost) > max_chars {
break;
}
spent = spent.saturating_add(cost);
kept_lines += 1;
}
let kept_lines =
lines_within(text.lines().map(|line| line.chars().count()), max_lines, max_chars);
if kept_lines >= total {
return clamp_bytes(text);
}
let kept = text.lines().take(kept_lines).collect::<Vec<_>>().join("\n");
let remaining = total - kept_lines;
with_note(&kept, total - kept_lines, total, unit, hint)
}
/// How many lines, given each one's length in characters, fit under both `max_lines` and
/// `max_chars`; at least one when there is any, however long it is.
///
/// The cut [`cap_lines_within`] makes, for a tool that lays its lines out after choosing them.
pub(crate) fn lines_within(
lengths: impl IntoIterator<Item = usize>,
max_lines: usize,
max_chars: usize,
) -> usize {
let mut kept = 0usize;
let mut spent = 0usize;
for length in lengths.into_iter().take(max_lines) {
let cost = length.saturating_add(1);
if kept > 0 && spent.saturating_add(cost) > max_chars {
break;
}
spent = spent.saturating_add(cost);
kept += 1;
}
kept
}
/// `kept`, then the note that `remaining` more of `total` were cut and how to see them, clamped to
/// the byte backstop.
pub(crate) fn with_note(
kept: &str,
remaining: usize,
total: usize,
unit: &str,
hint: &str,
) -> String {
clamp_bytes(&format!("{kept}\n\n… {remaining} more {unit} not shown ({total} total). {hint}"))
}
/// [`clamp_bytes`] for a tool that caps its own lines first.
pub(crate) fn clamp(text: &str) -> String {
clamp_bytes(text)
}
/// Head+tail byte clamp for the rare single-huge-line case the line cap can't shrink.
fn clamp_bytes(text: &str) -> String {
if text.len() <= MAX_OUTPUT_BYTES {