Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
193
crates/luxe-cli/examples/tool_bench.rs
Normal file
193
crates/luxe-cli/examples/tool_bench.rs
Normal file
@@ -0,0 +1,193 @@
|
||||
//! Tool bench — each tool against the shell command it replaces, measured.
|
||||
//!
|
||||
//! Runs both in the working tree it is started from and writes what each returned, in bytes and
|
||||
//! lines, to the JSON the site's Tools page draws its bars from. A recording, like the captures:
|
||||
//! the numbers move with the repository, nothing checks them, and the file names the commit they
|
||||
//! came from. `scripts/tool-bench.sh` runs it.
|
||||
//!
|
||||
//! The tools run the way a session registers them, so the sizes are what a model would be sent.
|
||||
//! The shell side captures stdout and stderr together, since an agent's shell sees both — and is
|
||||
//! then put through each other harness's own shell-output limit ([`forwarded`]), because nobody's
|
||||
//! model is sent a raw 900 KB `git log`: the honest comparison is with what a harness forwards.
|
||||
//!
|
||||
//! `TOOL_BENCH_CASES` names a JSON file of cases (`id`, `shell`, `tool`, `args`) for measuring in a
|
||||
//! repository other than this one; without it, the cases below measure Luxe itself.
|
||||
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// One comparison: a shell command, and the tool call that answers the same question.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Case {
|
||||
id: String,
|
||||
shell: String,
|
||||
tool: String,
|
||||
args: serde_json::Value,
|
||||
}
|
||||
|
||||
/// What another harness sends its model of a shell command's output, in bytes.
|
||||
///
|
||||
/// Each limit is the harness's own, read from its source and checked against what crossed the
|
||||
/// wire in the harness evaluation: Pi and OpenCode keep the last 2,000 lines
|
||||
/// or 50 KB, Codex keeps the head and tail of about 10 KB, and Claude Code keeps 30,000
|
||||
/// characters. A limit is all this models; a harness that also saves the rest to a file says so
|
||||
/// in a line or two this leaves out.
|
||||
fn forwarded(text: &str) -> serde_json::Value {
|
||||
let tail = |max_lines: usize, max_bytes: usize| {
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let kept = &lines[lines.len().saturating_sub(max_lines)..];
|
||||
kept.join("\n").len().min(max_bytes)
|
||||
};
|
||||
serde_json::json!({
|
||||
"pi": tail(2000, 50 * 1024),
|
||||
"opencode": tail(2000, 50 * 1024),
|
||||
"codex": text.len().min(10 * 1024),
|
||||
"claude": text.chars().take(30_000).map(char::len_utf8).sum::<usize>(),
|
||||
})
|
||||
}
|
||||
|
||||
fn cases() -> Vec<Case> {
|
||||
if let Ok(path) = std::env::var("TOOL_BENCH_CASES") {
|
||||
let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"));
|
||||
return serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path}: {e}"));
|
||||
}
|
||||
let engine = "crates/luxe-core/src/engine.rs";
|
||||
let case = |id: &str, shell: &str, tool: &str, args: serde_json::Value| Case {
|
||||
id: id.to_string(),
|
||||
shell: shell.to_string(),
|
||||
tool: tool.to_string(),
|
||||
args,
|
||||
};
|
||||
vec![
|
||||
case(
|
||||
"glob",
|
||||
"find . -name '*.json'",
|
||||
"glob",
|
||||
serde_json::json!({ "pattern": "**/*.json" }),
|
||||
),
|
||||
case("grep", "rg -n render", "grep", serde_json::json!({ "pattern": "render" })),
|
||||
case(
|
||||
"bash",
|
||||
"cargo tree -p luxe-cli",
|
||||
"bash",
|
||||
serde_json::json!({ "command": "cargo tree -p luxe-cli" }),
|
||||
),
|
||||
case(
|
||||
"test",
|
||||
"cargo test -p luxe-tools",
|
||||
"test",
|
||||
serde_json::json!({ "command": "cargo test -p luxe-tools" }),
|
||||
),
|
||||
case("git", "git log", "git", serde_json::json!({ "action": "log" })),
|
||||
case(
|
||||
"read_symbol",
|
||||
"cat crates/luxe-core/src/engine.rs",
|
||||
"read_symbol",
|
||||
serde_json::json!({ "path": engine, "symbol": "set_cancelled" }),
|
||||
),
|
||||
case(
|
||||
"module_report",
|
||||
"cat crates/luxe-core/src/engine.rs",
|
||||
"module_report",
|
||||
serde_json::json!({ "path": engine }),
|
||||
),
|
||||
case(
|
||||
"web_fetch",
|
||||
"curl -sL https://docs.rs/tokio/latest/tokio/",
|
||||
"web_fetch",
|
||||
serde_json::json!({ "url": "https://docs.rs/tokio/latest/tokio/" }),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let out = std::env::args().nth(1).unwrap_or_else(|| "web/src/data/tool-bench.json".into());
|
||||
let cwd = std::env::current_dir()?;
|
||||
let registry = registry(&cwd);
|
||||
// held for the whole run: a dropped sender reads as a cancel to a tool waiting on it
|
||||
let (_cancel, cx) = context(&cwd);
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for case in cases() {
|
||||
let shell = Command::new("sh").arg("-c").arg(&case.shell).current_dir(&cwd).output()?;
|
||||
let shell_text = [shell.stdout, shell.stderr].concat();
|
||||
let shell_text = String::from_utf8_lossy(&shell_text);
|
||||
|
||||
let tool = registry
|
||||
.get(&case.tool)
|
||||
.ok_or_else(|| anyhow::anyhow!("no tool named {}", case.tool))?;
|
||||
let result = tool.run(case.args.clone(), &cx).await;
|
||||
// A failing test run is the case worth measuring; any other error is a broken case
|
||||
anyhow::ensure!(
|
||||
!result.is_error || case.tool == "test",
|
||||
"{} failed: {:?}",
|
||||
case.id,
|
||||
result.content
|
||||
);
|
||||
let tool_text: String = result
|
||||
.content
|
||||
.iter()
|
||||
.filter_map(|block| match block {
|
||||
luxe_core::ContentBlock::Text(text) => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
eprintln!("{:<14} shell {:>9} B tool {:>7} B", case.id, shell_text.len(), tool_text.len());
|
||||
rows.push(serde_json::json!({
|
||||
"id": case.id,
|
||||
"shell": case.shell,
|
||||
"tool": case.tool,
|
||||
"args": case.args,
|
||||
"shellBytes": shell_text.len(),
|
||||
"shellLines": shell_text.lines().count(),
|
||||
"toolBytes": tool_text.len(),
|
||||
"toolLines": tool_text.lines().count(),
|
||||
"forwarded": forwarded(&shell_text),
|
||||
}));
|
||||
}
|
||||
|
||||
let commit = Command::new("git").args(["rev-parse", "--short", "HEAD"]).output()?;
|
||||
let repository = cwd.file_name().map(|name| name.to_string_lossy().into_owned());
|
||||
let report = serde_json::json!({
|
||||
"repository": repository,
|
||||
"commit": String::from_utf8_lossy(&commit.stdout).trim(),
|
||||
"rows": rows,
|
||||
});
|
||||
std::fs::write(&out, serde_json::to_string_pretty(&report)? + "\n")?;
|
||||
eprintln!("wrote {out}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The tools a session registers, minus the ones that wait on a person or a debugger.
|
||||
fn registry(cwd: &std::path::Path) -> luxe_core::ToolRegistry {
|
||||
let mut registry = luxe_core::ToolRegistry::new();
|
||||
let tracker =
|
||||
luxe_tools::register_builtins(&mut registry, None, luxe_tools::web::WebConfig::from_env());
|
||||
let lsp = Arc::new(luxe_lsp::WorkspaceLspClient::with_servers(Default::default()));
|
||||
let graph = Arc::new(luxe_lsp::graph::GraphService::new(cwd));
|
||||
registry.register(Arc::new(
|
||||
luxe_lsp::ModuleReportTool::new(lsp.clone())
|
||||
.with_tracker(tracker.clone())
|
||||
.with_graph(graph),
|
||||
));
|
||||
registry.register(Arc::new(luxe_lsp::ReadSymbolTool::new(lsp).with_tracker(tracker)));
|
||||
registry
|
||||
}
|
||||
|
||||
fn context(cwd: &std::path::Path) -> (tokio::sync::watch::Sender<bool>, luxe_core::ToolContext) {
|
||||
let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
|
||||
let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(1024);
|
||||
tokio::spawn(async move { while event_rx.recv().await.is_some() {} });
|
||||
let cx = luxe_core::ToolContext {
|
||||
cwd: cwd.to_path_buf(),
|
||||
session_id: "tool-bench".into(),
|
||||
cancelled: Arc::new(cancel_rx),
|
||||
event_tx,
|
||||
hook_ctx: None,
|
||||
question_broker: None,
|
||||
};
|
||||
(cancel_tx, cx)
|
||||
}
|
||||
Reference in New Issue
Block a user