grep names each file once with its lines under it, the changes header keeps its position whole, and the bash teardown tests stop flaking on macOS
- grep groups its lines under each file, the layout of rg --heading, rather than repeating the path on every line: 9–28% fewer bytes than rg -n on the searches measured. scopes: true labels each run of matches with the function or type around it. - The changes pane's header shortens the path rather than cut its position, and keeps a gap before the FILE / DIFF switch. - The bash teardown tests count a loop still running as a survivor, not one write already under way when the kill landed.
This commit is contained in:
@@ -442,10 +442,18 @@ mod tree_teardown {
|
||||
format!("(for _ in $(seq 1 40); do touch {}; sleep 0.25; done) & wait", marker.display())
|
||||
}
|
||||
|
||||
/// Whether anything is still touching `marker` after it is removed.
|
||||
/// Whether anything is still touching `marker`, the loop, once a touch that was already under
|
||||
/// way has had its moment.
|
||||
///
|
||||
/// The kill reaches the whole group at once, but a `touch` forked as it lands can finish one
|
||||
/// write after it, and on macOS's slower teardown that write can follow the first clear. One
|
||||
/// straggling write is not a survivor; a loop still running touches again within a quarter
|
||||
/// second of the second clear.
|
||||
async fn survivor_still_running(marker: &std::path::Path) -> bool {
|
||||
let _ = std::fs::remove_file(marker);
|
||||
tokio::time::sleep(Duration::from_millis(1_200)).await;
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
let _ = std::fs::remove_file(marker);
|
||||
tokio::time::sleep(Duration::from_millis(1_000)).await;
|
||||
marker.exists()
|
||||
}
|
||||
|
||||
|
||||
@@ -4,16 +4,21 @@
|
||||
//! it is opened, and the approved files are scanned in parallel with `regex`. An external `rg`
|
||||
//! could not do the middle step, which is the one that keeps denied content out of the answer.
|
||||
//!
|
||||
//! The result is bounded `file:line:text`, with optional context lines and a glob filter — or,
|
||||
//! where the question is coarser than the lines, the files that match or a count per file.
|
||||
//! The rows are bounded `file:line:text`, with optional context lines and a glob filter — or,
|
||||
//! where the question is coarser than the lines, the files that match or a count per file. Lines
|
||||
//! reach the model grouped under their file: the path once rather than on every line. Asked to
|
||||
//! (`scopes`), each run of matches is labeled with the function or type around it, where an
|
||||
//! outline can say, so where a match sits needs no read to find out.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use luxe_core::hooks::file_changed::{OutlineSource, Scope};
|
||||
use luxe_core::{ContentBlock, Tool, ToolContext, ToolResultData, ToolSchema};
|
||||
use serde_json::json;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::warn;
|
||||
|
||||
mod guarded;
|
||||
mod present;
|
||||
#[cfg(test)]
|
||||
mod read_policy_tests;
|
||||
|
||||
@@ -21,7 +26,9 @@ mod read_policy_tests;
|
||||
fn grep_schema() -> ToolSchema {
|
||||
ToolSchema {
|
||||
name: "grep".to_string(),
|
||||
description: "Search file *contents* by regex. Returns `file:line:text`, bounded \
|
||||
description: "Search file *contents* by regex. Returns each matching file's path once \
|
||||
with its `line:text` rows under it; `scopes: true` also names the function \
|
||||
or type around each match, as `[QuerySet › delete]`. Bounded \
|
||||
— a broad pattern says it was capped instead of flooding your context. Case \
|
||||
is sensitive unless the pattern says otherwise: prefix `(?i)`. Skips ignored \
|
||||
paths (`.gitignore`, `.git/`, build output); set `all: true` to include \
|
||||
@@ -50,7 +57,7 @@ fn grep_schema() -> ToolSchema {
|
||||
"outputMode": {
|
||||
"type": "string",
|
||||
"enum": ["content", "files", "count"],
|
||||
"description": "What to return: 'content' (file:line:text, the default), \
|
||||
"description": "What to return: 'content' (matching lines under each file, the default), \
|
||||
'files' (one path per matching file), or 'count' (file:count \
|
||||
of matching lines). Prefer 'files' when you only need to know \
|
||||
where to look."
|
||||
@@ -76,6 +83,13 @@ fn grep_schema() -> ToolSchema {
|
||||
"type": "boolean",
|
||||
"description": "Return the lines that do NOT match the pattern. Default false."
|
||||
},
|
||||
"scopes": {
|
||||
"type": "boolean",
|
||||
"description": "Name the function or type each match sits in, for code a \
|
||||
parser reads. Costs a row per change of scope; worth it \
|
||||
when *where* a match is matters, as for call sites. \
|
||||
Default false."
|
||||
},
|
||||
"all": {
|
||||
"type": "boolean",
|
||||
"description": "Include ignored files (build output, vendored deps). Default false."
|
||||
@@ -148,6 +162,44 @@ impl OutputMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// One row a search found, kept whole until the answer is laid out.
|
||||
#[derive(Debug, Clone)]
|
||||
struct Hit {
|
||||
/// The file, relative to where the search ran
|
||||
file: std::sync::Arc<str>,
|
||||
kind: HitKind,
|
||||
/// 1-based line, for a match or a context row
|
||||
line: usize,
|
||||
/// The line itself, or the number of matching lines for [`HitKind::Count`]
|
||||
text: String,
|
||||
/// The scopes around a match, outermost first, `QuerySet › delete`, where an outline can say
|
||||
scope: Option<String>,
|
||||
}
|
||||
|
||||
/// What a [`Hit`] is.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum HitKind {
|
||||
Match,
|
||||
Context,
|
||||
/// A file with a match, in the files mode
|
||||
File,
|
||||
/// A file's number of matching lines, in the count mode
|
||||
Count,
|
||||
}
|
||||
|
||||
impl Hit {
|
||||
/// The row as ripgrep prints it, which the caps measure: `file:line:text`, `file-line-text`
|
||||
/// for context, a bare path, or `file:count`.
|
||||
fn flat(&self) -> String {
|
||||
match self.kind {
|
||||
HitKind::Match => format!("{}:{}:{}", self.file, self.line, self.text),
|
||||
HitKind::Context => format!("{}-{}-{}", self.file, self.line, self.text),
|
||||
HitKind::File => self.file.to_string(),
|
||||
HitKind::Count => format!("{}:{}", self.file, self.text),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One parsed search.
|
||||
///
|
||||
/// `limit` is deliberately *not* consulted while searching. Stopping early would make the total the
|
||||
@@ -162,17 +214,22 @@ struct Search<'a> {
|
||||
after: usize,
|
||||
all: bool,
|
||||
invert: bool,
|
||||
/// Label each match with the scopes around it, where an outline can read the file
|
||||
scopes: bool,
|
||||
mode: OutputMode,
|
||||
limit: usize,
|
||||
}
|
||||
|
||||
/// What `search` finds under `cwd`, unguarded, as the tests ask it.
|
||||
/// What `search` finds under `cwd`, unguarded and as ripgrep would print it, as the tests ask it.
|
||||
#[cfg(test)]
|
||||
fn native_search(cwd: &Path, search: &Search<'_>) -> Result<Vec<String>, String> {
|
||||
native_search_files(cwd, search, crate::walk::files(cwd, search.path, search.all), || false)
|
||||
let paths = crate::walk::files(cwd, search.path, search.all);
|
||||
let hits = native_search_files(cwd, search, paths, None, || false)?;
|
||||
Ok(hits.iter().map(Hit::flat).collect())
|
||||
}
|
||||
|
||||
/// Scans `paths` (relative to `cwd`) for `search`, in parallel, rows in path order.
|
||||
/// Scans `paths` (relative to `cwd`) for `search`, in parallel, rows in path order, each match
|
||||
/// labeled with its scopes where `outline` can parse the file.
|
||||
///
|
||||
/// Each worker takes a contiguous run of the sorted paths and the runs are joined in order, so the
|
||||
/// answer reads the same however many threads produced it. `stopped` is checked between files.
|
||||
@@ -180,8 +237,9 @@ fn native_search_files(
|
||||
cwd: &Path,
|
||||
search: &Search<'_>,
|
||||
paths: Vec<PathBuf>,
|
||||
outline: Option<&dyn OutlineSource>,
|
||||
stopped: impl Fn() -> bool + Sync,
|
||||
) -> Result<Vec<String>, String> {
|
||||
) -> Result<Vec<Hit>, String> {
|
||||
let regex = regex::Regex::new(search.pattern)
|
||||
.map_err(|error| format!("'{}' is not a valid regex: {error}", search.pattern))?;
|
||||
let prefilter = prefilter(search);
|
||||
@@ -199,7 +257,7 @@ fn native_search_files(
|
||||
let workers = std::thread::available_parallelism().map_or(1, usize::from).min(16);
|
||||
let run = paths.len().div_ceil(workers).max(1);
|
||||
// a worker is `Err` when it panicked, `Ok(None)` when it was stopped
|
||||
let scanned: Vec<std::thread::Result<Option<Vec<String>>>> = std::thread::scope(|scope| {
|
||||
let scanned: Vec<std::thread::Result<Option<Vec<Hit>>>> = std::thread::scope(|scope| {
|
||||
let handles: Vec<_> = paths
|
||||
.chunks(run)
|
||||
.map(|chunk| {
|
||||
@@ -210,7 +268,8 @@ fn native_search_files(
|
||||
if stopped() {
|
||||
return None;
|
||||
}
|
||||
rows.extend(search_file(cwd, relative, (regex, prefilter), search));
|
||||
let matcher = (regex, prefilter);
|
||||
rows.extend(search_file(cwd, relative, matcher, search, outline));
|
||||
}
|
||||
Some(rows)
|
||||
})
|
||||
@@ -223,9 +282,9 @@ fn native_search_files(
|
||||
|
||||
/// The workers' rows joined in order; a crashed worker is a failure, a stopped one a cancel.
|
||||
fn combine(
|
||||
scanned: Vec<std::thread::Result<Option<Vec<String>>>>,
|
||||
scanned: Vec<std::thread::Result<Option<Vec<Hit>>>>,
|
||||
stopped: bool,
|
||||
) -> Result<Vec<String>, String> {
|
||||
) -> Result<Vec<Hit>, String> {
|
||||
if scanned.iter().any(Result::is_err) {
|
||||
return Err("a search worker failed; nothing it scanned is reported".into());
|
||||
}
|
||||
@@ -269,14 +328,15 @@ fn has_lone_carriage_return(text: &str) -> bool {
|
||||
bytes.iter().enumerate().any(|(at, byte)| *byte == b'\r' && bytes.get(at + 1) != Some(&b'\n'))
|
||||
}
|
||||
|
||||
/// The rows one file contributes, in the shape ripgrep prints them: `file:line:text` for a match,
|
||||
/// `file-line-text` for context, or the file's name or count for the coarser modes.
|
||||
/// The rows one file contributes: its matching and context lines, or its name or count for the
|
||||
/// coarser modes. A match is labeled with the scopes around it where `outline` parses the file.
|
||||
fn search_file(
|
||||
cwd: &Path,
|
||||
relative: &Path,
|
||||
(regex, prefilter): (®ex::Regex, Option<®ex::Regex>),
|
||||
search: &Search<'_>,
|
||||
) -> Vec<String> {
|
||||
outline: Option<&dyn OutlineSource>,
|
||||
) -> Vec<Hit> {
|
||||
let Ok(bytes) = std::fs::read(cwd.join(relative)) else { return Vec::new() };
|
||||
if bytes.iter().take(BINARY_SNIFF_BYTES).any(|byte| *byte == 0) {
|
||||
return Vec::new();
|
||||
@@ -287,7 +347,7 @@ fn search_file(
|
||||
if filtered.is_some_and(|prefilter| !prefilter.is_match(&text)) {
|
||||
return Vec::new();
|
||||
}
|
||||
let name = luxe_core::paths::display(relative);
|
||||
let file: std::sync::Arc<str> = luxe_core::paths::display(relative).into();
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let matched: Vec<usize> = lines
|
||||
.iter()
|
||||
@@ -298,45 +358,78 @@ fn search_file(
|
||||
if matched.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let row = |kind, line, text: String| Hit { file: file.clone(), kind, line, text, scope: None };
|
||||
match search.mode {
|
||||
OutputMode::Files => vec![name],
|
||||
OutputMode::Count => vec![format!("{name}:{}", matched.len())],
|
||||
OutputMode::Files => vec![row(HitKind::File, 0, String::new())],
|
||||
OutputMode::Count => vec![row(HitKind::Count, 0, matched.len().to_string())],
|
||||
OutputMode::Content => {
|
||||
// Decide *which* lines to print and *what each one is* separately, then print each
|
||||
// once. Doing it per match instead conflates the two: a match inside an earlier
|
||||
// match's trailing context gets emitted as context and never as the answer.
|
||||
let mut shown = vec![false; lines.len()];
|
||||
for index in &matched {
|
||||
let first = index.saturating_sub(search.before);
|
||||
let last = (index + search.after + 1).min(lines.len());
|
||||
shown[first..last].iter_mut().for_each(|slot| *slot = true);
|
||||
}
|
||||
let is_match = |index: usize| matched.binary_search(&index).is_ok();
|
||||
lines
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(index, _)| shown[*index])
|
||||
.map(|(index, line)| {
|
||||
// `-` marks a context line; the caller's parsers and the model both read the
|
||||
// `:` rows as the answer
|
||||
let separator = if is_match(index) { ':' } else { '-' };
|
||||
format!("{name}{separator}{}{separator}{line}", index + 1)
|
||||
let outline = outline.filter(|_| search.scopes);
|
||||
let scopes = outline.and_then(|outline| outline.scopes(relative, &text));
|
||||
line_hits(&lines, &matched, search)
|
||||
.map(|(index, is_match)| {
|
||||
let kind = if is_match { HitKind::Match } else { HitKind::Context };
|
||||
let mut hit = row(kind, index + 1, lines[index].to_string());
|
||||
hit.scope =
|
||||
scopes.as_deref().filter(|_| is_match).and_then(|s| scope_label(s, index));
|
||||
hit
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The lines to show and whether each is a match, in order.
|
||||
///
|
||||
/// Decides *which* lines to print and *what each one is* separately, then yields each once. Doing
|
||||
/// it per match instead conflates the two: a match inside an earlier match's trailing context gets
|
||||
/// emitted as context and never as the answer.
|
||||
fn line_hits<'a>(
|
||||
lines: &'a [&str],
|
||||
matched: &'a [usize],
|
||||
search: &Search<'_>,
|
||||
) -> impl Iterator<Item = (usize, bool)> + 'a {
|
||||
let mut shown = vec![false; lines.len()];
|
||||
for index in matched {
|
||||
let first = index.saturating_sub(search.before);
|
||||
let last = (index + search.after + 1).min(lines.len());
|
||||
shown[first..last].iter_mut().for_each(|slot| *slot = true);
|
||||
}
|
||||
(0..lines.len())
|
||||
.filter(move |index| shown[*index])
|
||||
.map(move |index| (index, matched.binary_search(&index).is_ok()))
|
||||
}
|
||||
|
||||
/// The scopes around 0-based line `line`, outermost first, as `QuerySet › delete`.
|
||||
///
|
||||
/// A scope whose declaration starts on the line is left out: the line itself names it, so
|
||||
/// `def delete(self):` is labeled `QuerySet`, not `QuerySet › delete`.
|
||||
fn scope_label(scopes: &[Scope], line: usize) -> Option<String> {
|
||||
let mut around: Vec<&Scope> =
|
||||
scopes.iter().filter(|scope| scope.start_line < line && line <= scope.end_line).collect();
|
||||
around.sort_by_key(|scope| (scope.start_line, std::cmp::Reverse(scope.end_line)));
|
||||
let names: Vec<&str> = around.iter().map(|scope| scope.name.as_str()).collect();
|
||||
(!names.is_empty()).then(|| names.join(" › "))
|
||||
}
|
||||
|
||||
/// The `grep` tool implementation.
|
||||
pub struct GrepTool {
|
||||
/// Shared read tracker; a file whose lines a search printed is recorded as read
|
||||
tracker: Option<std::sync::Arc<luxe_core::read_tracker::ReadTracker>>,
|
||||
/// What names the scopes a match sits in; without it lines come unlabeled
|
||||
outline: Option<std::sync::Arc<dyn OutlineSource>>,
|
||||
}
|
||||
|
||||
impl GrepTool {
|
||||
/// Create a new grep tool instance.
|
||||
pub fn new() -> Self {
|
||||
Self { tracker: None }
|
||||
Self { tracker: None, outline: None }
|
||||
}
|
||||
|
||||
/// Attach the outline that labels each match with the function or type around it, a parser
|
||||
/// that lives outside this crate.
|
||||
pub fn with_outline(mut self, outline: std::sync::Arc<dyn OutlineSource>) -> Self {
|
||||
self.outline = Some(outline);
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the shared read tracker, so the lines a search showed satisfy the edit guards the
|
||||
@@ -380,6 +473,8 @@ impl Tool for GrepTool {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scope_tests;
|
||||
#[cfg(test)]
|
||||
mod search_tests;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -48,6 +48,7 @@ pub(super) fn parse_search(args: &serde_json::Value) -> Result<Search<'_>, Strin
|
||||
after: side("after"),
|
||||
all: args.get("all").and_then(|value| value.as_bool()).unwrap_or(false),
|
||||
invert: args.get("invert").and_then(|value| value.as_bool()).unwrap_or(false),
|
||||
scopes: args.get("scopes").and_then(|value| value.as_bool()).unwrap_or(false),
|
||||
// camelCase is advertised; snake_case is what models reach for, and a silently ignored
|
||||
// mode returns the expensive answer
|
||||
mode: OutputMode::parse(
|
||||
@@ -106,16 +107,15 @@ async fn scan(
|
||||
args: serde_json::Value,
|
||||
cx: &ToolContext,
|
||||
paths: Vec<PathBuf>,
|
||||
stopped: Arc<AtomicBool>,
|
||||
) -> Result<String, String> {
|
||||
(outline, stopped): (Option<Arc<dyn OutlineSource>>, Arc<AtomicBool>),
|
||||
) -> Result<Vec<Hit>, String> {
|
||||
let cwd = cx.cwd.clone();
|
||||
let cancelled = cx.cancelled.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let search = parse_search(&args)?;
|
||||
super::native_search_files(&cwd, &search, paths, || {
|
||||
super::native_search_files(&cwd, &search, paths, outline.as_deref(), || {
|
||||
stopped.load(Ordering::Relaxed) || *cancelled.borrow()
|
||||
})
|
||||
.map(|rows| rows.join("\n"))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| format!("grep scan worker failed: {error}"))?
|
||||
@@ -141,7 +141,7 @@ pub(super) async fn run(
|
||||
Ok(plan) => plan,
|
||||
Err(error) => return ToolResultData::error(error),
|
||||
};
|
||||
let found = scan(args.clone(), cx, plan.allowed, stopped).await;
|
||||
let found = scan(args.clone(), cx, plan.allowed, (tool.outline.clone(), stopped)).await;
|
||||
let mut result = finish(tool, cx, &search, found);
|
||||
if plan.denied + plan.asks > 0 {
|
||||
result.content.push(ContentBlock::Text(format!(
|
||||
@@ -159,20 +159,24 @@ pub(super) fn finish(
|
||||
tool: &GrepTool,
|
||||
cx: &ToolContext,
|
||||
search: &Search<'_>,
|
||||
found: Result<String, String>,
|
||||
found: Result<Vec<Hit>, String>,
|
||||
) -> ToolResultData {
|
||||
match found {
|
||||
Ok(matches) if matches.trim().is_empty() => ToolResultData::text("No matches found"),
|
||||
Ok(matches) => {
|
||||
let bounded = crate::output_limit::cap_lines_within(
|
||||
&matches,
|
||||
Ok(hits) if hits.is_empty() => ToolResultData::text("No matches found"),
|
||||
Ok(hits) if search.mode == OutputMode::Content => {
|
||||
let (text, shown) = super::present::content(&hits, search);
|
||||
record_matches(tool, cx, search, &hits[..shown]);
|
||||
ToolResultData::text(text)
|
||||
}
|
||||
Ok(hits) => {
|
||||
let rows: Vec<String> = hits.iter().map(Hit::flat).collect();
|
||||
ToolResultData::text(crate::output_limit::cap_lines_within(
|
||||
&rows.join("\n"),
|
||||
search.limit,
|
||||
GREP_MAX_CHARS,
|
||||
search.mode.unit(),
|
||||
search.mode.hint(),
|
||||
);
|
||||
record_matches(tool, cx, search, &bounded);
|
||||
ToolResultData::text(bounded)
|
||||
))
|
||||
}
|
||||
Err(problem) => {
|
||||
warn!(pattern = search.pattern, error = %problem, "grep failed");
|
||||
@@ -181,15 +185,18 @@ pub(super) fn finish(
|
||||
}
|
||||
}
|
||||
|
||||
fn record_matches(tool: &GrepTool, cx: &ToolContext, search: &Search<'_>, bounded: &str) {
|
||||
let (Some(tracker), OutputMode::Content) = (&tool.tracker, search.mode) else { return };
|
||||
/// Records as read every file a line of which the answer showed, the way a `read` of it would be.
|
||||
fn record_matches(tool: &GrepTool, cx: &ToolContext, search: &Search<'_>, shown: &[Hit]) {
|
||||
let Some(tracker) = &tool.tracker else { return };
|
||||
let searched = cx.cwd.join(search.path);
|
||||
let shown = if searched.is_file() {
|
||||
vec![searched]
|
||||
} else {
|
||||
crate::bash::seen::files_in_rows(bounded, &cx.cwd)
|
||||
let files: Vec<PathBuf> = match searched.is_file() {
|
||||
true => vec![searched],
|
||||
false => shown.iter().map(|hit| cx.cwd.join(&*hit.file)).collect(),
|
||||
};
|
||||
for path in shown {
|
||||
tracker.record(&cx.session_id, &path);
|
||||
let mut recorded = std::collections::HashSet::new();
|
||||
for path in files {
|
||||
if recorded.insert(path.clone()) {
|
||||
tracker.record(&cx.session_id, &path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
106
crates/luxe-tools/src/grep/present.rs
Normal file
106
crates/luxe-tools/src/grep/present.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! Present — a content search laid out for reading: each file named once, its lines under it, and
|
||||
//! the scope around a run of matches where an outline can say.
|
||||
//!
|
||||
//! The rows are counted and capped as ripgrep would print them, `file:line:text`, so the bounds and
|
||||
//! the totals are the ones the other modes keep. Only the layout changes. A path repeated on every
|
||||
//! line is a large share of a broad search, and the function a match sits in is what the next
|
||||
//! `read` would have gone to find out.
|
||||
|
||||
use super::{Hit, HitKind, Search, GREP_MAX_CHARS};
|
||||
|
||||
/// The capped, grouped answer, and how many of `hits` it shows.
|
||||
pub(super) fn content(hits: &[Hit], search: &Search<'_>) -> (String, usize) {
|
||||
let lengths = hits.iter().map(|hit| hit.flat().chars().count());
|
||||
let kept = crate::output_limit::lines_within(lengths, search.limit, GREP_MAX_CHARS);
|
||||
let shown = grouped(&hits[..kept], search.before + search.after > 0);
|
||||
let text = match kept < hits.len() {
|
||||
true => crate::output_limit::with_note(
|
||||
&shown,
|
||||
hits.len() - kept,
|
||||
hits.len(),
|
||||
search.mode.unit(),
|
||||
search.mode.hint(),
|
||||
),
|
||||
false => crate::output_limit::clamp(&shown),
|
||||
};
|
||||
(text, kept)
|
||||
}
|
||||
|
||||
/// `hits` under their files: the path, then `line:text` for a match and `line-text` for context,
|
||||
/// ` [Scope › name]` before a match whose scope differs from the last one's, and `--` where
|
||||
/// context leaves a gap.
|
||||
fn grouped(hits: &[Hit], context: bool) -> String {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let mut previous: Option<&Hit> = None;
|
||||
let mut scope: Option<&str> = None;
|
||||
for hit in hits {
|
||||
match previous {
|
||||
Some(last) if last.file == hit.file => {
|
||||
if context && hit.line > last.line + 1 {
|
||||
out.push("--".into());
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
out.push(hit.file.to_string());
|
||||
scope = None;
|
||||
}
|
||||
}
|
||||
if hit.kind == HitKind::Match && hit.scope.as_deref() != scope {
|
||||
out.push(scope_row(hit.scope.as_deref()));
|
||||
scope = hit.scope.as_deref();
|
||||
}
|
||||
let separator = if hit.kind == HitKind::Match { ':' } else { '-' };
|
||||
out.push(format!("{}{separator}{}", hit.line, hit.text));
|
||||
previous = Some(hit);
|
||||
}
|
||||
out.join("\n")
|
||||
}
|
||||
|
||||
/// ` [QuerySet › delete]`, or ` [top level]` for a match that has left the scopes before it.
|
||||
fn scope_row(scope: Option<&str>) -> String {
|
||||
format!(" [{}]", scope.unwrap_or("top level"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn hit(file: &str, kind: HitKind, line: usize, text: &str, scope: Option<&str>) -> Hit {
|
||||
let (file, text, scope) = (file.into(), text.to_string(), scope.map(String::from));
|
||||
Hit { file, kind, line, text, scope }
|
||||
}
|
||||
|
||||
/// Each file is named once, its lines under it, and a scope row comes only where it changes.
|
||||
#[test]
|
||||
fn matches_group_under_their_file_with_a_row_per_change_of_scope() {
|
||||
let hits = [
|
||||
hit("a.py", HitKind::Match, 3, " def delete(self):", Some("QuerySet")),
|
||||
hit("a.py", HitKind::Match, 9, " def delete_all(self):", Some("QuerySet")),
|
||||
hit("a.py", HitKind::Match, 20, "def delete(): pass", None),
|
||||
hit("b.py", HitKind::Match, 5, " delete()", Some("Model › save")),
|
||||
];
|
||||
let expected = "a.py\n [QuerySet]\n3: def delete(self):\n9: def delete_all(self):\n \
|
||||
[top level]\n20:def delete(): pass\nb.py\n [Model › save]\n5: delete()";
|
||||
assert_eq!(grouped(&hits, false), expected);
|
||||
}
|
||||
|
||||
/// A file no outline can read gets no scope rows at all, rather than a `[top level]` that
|
||||
/// would claim to know.
|
||||
#[test]
|
||||
fn an_unlabeled_file_gets_no_scope_rows() {
|
||||
let hits = [hit("notes.md", HitKind::Match, 1, "delete it", None)];
|
||||
assert_eq!(grouped(&hits, false), "notes.md\n1:delete it");
|
||||
}
|
||||
|
||||
/// Context lines take `-`, and a gap between runs of context is marked `--`.
|
||||
#[test]
|
||||
fn context_rows_take_a_dash_and_a_gap_is_marked() {
|
||||
let hits = [
|
||||
hit("a.rs", HitKind::Match, 2, "needle", None),
|
||||
hit("a.rs", HitKind::Context, 3, "after", None),
|
||||
hit("a.rs", HitKind::Match, 9, "needle", None),
|
||||
];
|
||||
assert_eq!(grouped(&hits, true), "a.rs\n2:needle\n3-after\n--\n9:needle");
|
||||
assert_eq!(grouped(&hits, false), "a.rs\n2:needle\n3-after\n9:needle", "no context asked");
|
||||
}
|
||||
}
|
||||
77
crates/luxe-tools/src/grep/scope_tests.rs
Normal file
77
crates/luxe-tools/src/grep/scope_tests.rs
Normal file
@@ -0,0 +1,77 @@
|
||||
use super::*;
|
||||
use luxe_core::hooks::file_changed::FunctionSize;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{mpsc, watch};
|
||||
|
||||
/// A stand-in for the tree-sitter outline: in a `.py` file, a class over lines 1–9 (0-based 0–8)
|
||||
/// with a method over lines 3–6, and nothing it can read in any other file.
|
||||
struct OneClass;
|
||||
|
||||
impl OutlineSource for OneClass {
|
||||
fn functions(&self, _: &Path, _: &str) -> Option<Vec<FunctionSize>> {
|
||||
None
|
||||
}
|
||||
|
||||
fn scopes(&self, path: &Path, _: &str) -> Option<Vec<Scope>> {
|
||||
let scope =
|
||||
|name: &str, start_line, end_line| Scope { name: name.into(), start_line, end_line };
|
||||
(path.extension()? == "py").then(|| vec![scope("QuerySet", 0, 8), scope("delete", 2, 5)])
|
||||
}
|
||||
}
|
||||
|
||||
fn context(cwd: &Path) -> ToolContext {
|
||||
let (event_tx, _event_rx) = mpsc::channel(64);
|
||||
let (_cancel_tx, cancel_rx) = watch::channel(false);
|
||||
ToolContext {
|
||||
cwd: cwd.to_path_buf(),
|
||||
session_id: "test-session".to_string(),
|
||||
cancelled: Arc::new(cancel_rx),
|
||||
event_tx,
|
||||
hook_ctx: None,
|
||||
question_broker: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Each match is labeled with the scopes around it, outermost first, and only where the label
|
||||
/// changes. The definition line takes its parent's label, since it names itself; a file no outline
|
||||
/// can read gets no label at all; and the path comes once, above its lines.
|
||||
#[tokio::test]
|
||||
async fn a_match_is_labeled_with_the_scopes_around_it() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let python =
|
||||
"class QuerySet:\n \n def delete(self):\n self.delete_batch()\n \
|
||||
return self\n \n \nOTHER = 1\n \ndef delete():\n pass\n";
|
||||
std::fs::write(dir.path().join("a.py"), python).unwrap();
|
||||
std::fs::write(dir.path().join("notes.md"), "delete it\n").unwrap();
|
||||
let tool = GrepTool::new().with_outline(Arc::new(OneClass));
|
||||
let args = json!({"pattern": "delete", "scopes": true});
|
||||
let result = tool.run(args, &context(dir.path())).await;
|
||||
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
|
||||
let expected = "a.py\n [QuerySet]\n3: def delete(self):\n [QuerySet › delete]\n4: \
|
||||
self.delete_batch()\n [top level]\n10:def delete():\nnotes.md\n1:delete it";
|
||||
assert_eq!(text, expected);
|
||||
}
|
||||
|
||||
/// Labels cost a row each, so a search without `scopes` gets the grouped lines alone, outline or
|
||||
/// not.
|
||||
#[tokio::test]
|
||||
async fn a_search_is_unlabeled_unless_it_asks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("a.py"), "class A:\n \n def delete(self):\n").unwrap();
|
||||
let tool = GrepTool::new().with_outline(Arc::new(OneClass));
|
||||
let result = tool.run(json!({"pattern": "delete"}), &context(dir.path())).await;
|
||||
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
|
||||
assert_eq!(text, "a.py\n3: def delete(self):");
|
||||
}
|
||||
|
||||
/// The coarse modes have no lines to label, so they ask nothing of the outline.
|
||||
#[tokio::test]
|
||||
async fn the_files_mode_carries_no_scopes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("a.py"), "class A:\n def delete(self): pass\n").unwrap();
|
||||
let tool = GrepTool::new().with_outline(Arc::new(OneClass));
|
||||
let args = json!({"pattern": "delete", "outputMode": "files"});
|
||||
let result = tool.run(args, &context(dir.path())).await;
|
||||
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
|
||||
assert_eq!(text.trim(), "a.py");
|
||||
}
|
||||
@@ -17,6 +17,7 @@ fn base(pattern: &str) -> Search<'_> {
|
||||
after: 0,
|
||||
all: false,
|
||||
invert: false,
|
||||
scopes: false,
|
||||
mode: OutputMode::Content,
|
||||
limit: 500,
|
||||
}
|
||||
@@ -147,10 +148,20 @@ fn the_prefilter_never_hides_a_matching_line() {
|
||||
/// A worker that crashed is a failure the caller hears about; one that was stopped is a cancel.
|
||||
#[test]
|
||||
fn a_crashed_worker_is_a_failure_not_a_cancel() {
|
||||
let rows = |row: &str| Ok(Some(vec![row.to_string()]));
|
||||
let crashed: std::thread::Result<Option<Vec<String>>> = Err(Box::new("boom"));
|
||||
let rows = |file: &str| {
|
||||
let hit = Hit {
|
||||
file: file.into(),
|
||||
kind: HitKind::File,
|
||||
line: 0,
|
||||
text: String::new(),
|
||||
scope: None,
|
||||
};
|
||||
Ok(Some(vec![hit]))
|
||||
};
|
||||
let crashed: std::thread::Result<Option<Vec<Hit>>> = Err(Box::new("boom"));
|
||||
let failed = combine(vec![rows("a"), crashed], false).unwrap_err();
|
||||
assert!(failed.contains("worker failed"), "{failed}");
|
||||
assert_eq!(combine(vec![rows("a"), Ok(None)], false).unwrap_err(), "grep cancelled");
|
||||
assert_eq!(combine(vec![rows("a"), rows("b")], false).unwrap(), ["a", "b"]);
|
||||
let joined = combine(vec![rows("a"), rows("b")], false).unwrap();
|
||||
assert_eq!(joined.iter().map(Hit::flat).collect::<Vec<_>>(), ["a", "b"]);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,13 @@ fn test_context(tmp_dir: &std::path::Path) -> ToolContext {
|
||||
}
|
||||
}
|
||||
|
||||
/// The matching-line rows of a content answer, `12:text`, not the paths or scope rows around them.
|
||||
fn match_rows(text: &str) -> usize {
|
||||
text.lines()
|
||||
.filter(|line| line.split_once(':').is_some_and(|(n, _)| n.parse::<usize>().is_ok()))
|
||||
.count()
|
||||
}
|
||||
|
||||
/// A plain content search of `.`, for a test to vary one field of.
|
||||
fn search<'a>(pattern: &'a str) -> Search<'a> {
|
||||
Search {
|
||||
@@ -27,6 +34,7 @@ fn search<'a>(pattern: &'a str) -> Search<'a> {
|
||||
after: 0,
|
||||
all: false,
|
||||
invert: false,
|
||||
scopes: false,
|
||||
mode: OutputMode::Content,
|
||||
limit: GREP_MAX_LINES,
|
||||
}
|
||||
@@ -246,7 +254,7 @@ async fn a_zero_limit_is_an_absent_limit() {
|
||||
.run(json!({"pattern": "needle", "limit": 0}), &test_context(dir.path()))
|
||||
.await;
|
||||
let ContentBlock::Text(text) = &result.content[0] else { panic!("expected text") };
|
||||
assert_eq!(text.lines().count(), 20, "every match, not none: {text}");
|
||||
assert_eq!(match_rows(text), 20, "every match, not none: {text}");
|
||||
assert!(!text.contains("not shown"), "and nothing was cut: {text}");
|
||||
}
|
||||
|
||||
@@ -282,16 +290,12 @@ async fn a_limit_can_only_tighten_the_cap() {
|
||||
|
||||
let sampled = GrepTool::new().run(json!({"pattern": "needle", "limit": 5}), &cx).await;
|
||||
let ContentBlock::Text(text) = &sampled.content[0] else { panic!("expected text") };
|
||||
assert_eq!(text.lines().filter(|line| line.contains("many.txt")).count(), 5, "{text}");
|
||||
assert_eq!(match_rows(text), 5, "{text}");
|
||||
assert!(text.contains("295 more matching lines not shown (300 total)"), "{text}");
|
||||
|
||||
let greedy = GrepTool::new().run(json!({"pattern": "needle", "limit": 5000}), &cx).await;
|
||||
let ContentBlock::Text(text) = &greedy.content[0] else { panic!("expected text") };
|
||||
assert_eq!(
|
||||
text.lines().filter(|line| line.contains("many.txt")).count(),
|
||||
GREP_MAX_LINES,
|
||||
"a limit past the ceiling is the ceiling: {text}"
|
||||
);
|
||||
assert_eq!(match_rows(text), GREP_MAX_LINES, "a limit past the ceiling is the ceiling: {text}");
|
||||
}
|
||||
|
||||
/// The count a capped result reports is the count that exists, not the count that was kept.
|
||||
|
||||
@@ -49,6 +49,9 @@ pub mod write;
|
||||
/// The `todo` tool requires a store root (e.g. `.luxe/todos`). Pass `None` to skip registering it.
|
||||
/// `web` configures `web_search` and `web_fetch`; [`web::WebConfig::from_env`] is the usual start.
|
||||
///
|
||||
/// `outline` labels each `grep` match with the function or type around it; it is a parser this
|
||||
/// crate cannot depend on (tree-sitter lives in `luxe-lsp`), and `None` leaves matches unlabeled.
|
||||
///
|
||||
/// Returns the shared [`luxe_core::read_tracker::ReadTracker`] backing the read-before-edit /
|
||||
/// staleness / blind-write guards, so the caller can also wire it into the LSP funnel reads
|
||||
/// (`read_symbol`, `module_report`) that see file content but live in a crate that can't depend
|
||||
@@ -57,6 +60,7 @@ pub fn register_builtins(
|
||||
registry: &mut luxe_core::ToolRegistry,
|
||||
todo_store_root: Option<std::path::PathBuf>,
|
||||
web: web::WebConfig,
|
||||
outline: Option<std::sync::Arc<dyn luxe_core::hooks::file_changed::OutlineSource>>,
|
||||
) -> std::sync::Arc<luxe_core::read_tracker::ReadTracker> {
|
||||
use std::sync::Arc;
|
||||
// One shared read tracker backs the read-before-edit + staleness + blind-write guards across
|
||||
@@ -76,7 +80,11 @@ pub fn register_builtins(
|
||||
registry.register(Arc::new(write::WriteTool::new().with_tracker(read_tracker.clone())));
|
||||
registry.register(Arc::new(edit::EditTool::new().with_tracker(read_tracker.clone())));
|
||||
registry.register(Arc::new(bash::BashTool::new().with_tracker(read_tracker.clone())));
|
||||
registry.register(Arc::new(grep::GrepTool::new().with_tracker(read_tracker.clone())));
|
||||
let grep = grep::GrepTool::new().with_tracker(read_tracker.clone());
|
||||
registry.register(Arc::new(match outline {
|
||||
Some(outline) => grep.with_outline(outline),
|
||||
None => grep,
|
||||
}));
|
||||
registry.register(Arc::new(glob::GlobTool::new()));
|
||||
registry.register(Arc::new(question::QuestionTool::new(question::QuestionConfig::default())));
|
||||
registry.register(Arc::new(skill::SkillTool::new(skill::SkillConfig)));
|
||||
|
||||
@@ -37,24 +37,54 @@ pub(crate) fn cap_lines_within(
|
||||
hint: &str,
|
||||
) -> String {
|
||||
let total = text.lines().count();
|
||||
let mut kept_lines = 0usize;
|
||||
let mut spent = 0usize;
|
||||
for line in text.lines().take(max_lines) {
|
||||
let cost = line.chars().count().saturating_add(1);
|
||||
if kept_lines > 0 && spent.saturating_add(cost) > max_chars {
|
||||
break;
|
||||
}
|
||||
spent = spent.saturating_add(cost);
|
||||
kept_lines += 1;
|
||||
}
|
||||
let kept_lines =
|
||||
lines_within(text.lines().map(|line| line.chars().count()), max_lines, max_chars);
|
||||
if kept_lines >= total {
|
||||
return clamp_bytes(text);
|
||||
}
|
||||
let kept = text.lines().take(kept_lines).collect::<Vec<_>>().join("\n");
|
||||
let remaining = total - kept_lines;
|
||||
with_note(&kept, total - kept_lines, total, unit, hint)
|
||||
}
|
||||
|
||||
/// How many lines, given each one's length in characters, fit under both `max_lines` and
|
||||
/// `max_chars`; at least one when there is any, however long it is.
|
||||
///
|
||||
/// The cut [`cap_lines_within`] makes, for a tool that lays its lines out after choosing them.
|
||||
pub(crate) fn lines_within(
|
||||
lengths: impl IntoIterator<Item = usize>,
|
||||
max_lines: usize,
|
||||
max_chars: usize,
|
||||
) -> usize {
|
||||
let mut kept = 0usize;
|
||||
let mut spent = 0usize;
|
||||
for length in lengths.into_iter().take(max_lines) {
|
||||
let cost = length.saturating_add(1);
|
||||
if kept > 0 && spent.saturating_add(cost) > max_chars {
|
||||
break;
|
||||
}
|
||||
spent = spent.saturating_add(cost);
|
||||
kept += 1;
|
||||
}
|
||||
kept
|
||||
}
|
||||
|
||||
/// `kept`, then the note that `remaining` more of `total` were cut and how to see them, clamped to
|
||||
/// the byte backstop.
|
||||
pub(crate) fn with_note(
|
||||
kept: &str,
|
||||
remaining: usize,
|
||||
total: usize,
|
||||
unit: &str,
|
||||
hint: &str,
|
||||
) -> String {
|
||||
clamp_bytes(&format!("{kept}\n\n… {remaining} more {unit} not shown ({total} total). {hint}"))
|
||||
}
|
||||
|
||||
/// [`clamp_bytes`] for a tool that caps its own lines first.
|
||||
pub(crate) fn clamp(text: &str) -> String {
|
||||
clamp_bytes(text)
|
||||
}
|
||||
|
||||
/// Head+tail byte clamp for the rare single-huge-line case the line cap can't shrink.
|
||||
fn clamp_bytes(text: &str) -> String {
|
||||
if text.len() <= MAX_OUTPUT_BYTES {
|
||||
|
||||
Reference in New Issue
Block a user