Initial public release

Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
Evan Whittaker
2026-10-17 12:08:30 -05:00
commit 09a7de2520
1730 changed files with 412952 additions and 0 deletions

View File

@@ -0,0 +1,390 @@
//! `screenshot` tool — cross-platform screen/window/URL capture for visual verification.
//!
//! Captures the screen, a region, a window, or a rendered URL to a PNG, then returns it inline
//! (so vision-capable models can see it) alongside the saved path. Uses the platform's native
//! capture tool (see `capture`) — no heavyweight capture crate — so the dependency surface and
//! the supply-chain gate stay clean.
mod browser;
mod capture;
use async_trait::async_trait;
use base64::Engine;
use luxe_core::provider::ToolSchema;
use luxe_core::{ContentBlock, Tool, ToolContext, ToolResultData};
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use capture::CaptureTarget;
/// The `screenshot` tool.
#[derive(Default)]
pub struct ScreenshotTool;
impl ScreenshotTool {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl Tool for ScreenshotTool {
fn name(&self) -> &str {
"screenshot"
}
/// `mode=url` hands a URL to a headless browser, and a browser told to load a host that never
/// answers waits forever — `--virtual-time-budget` bounds the page's *clock*, not its network.
/// The native capture backends are sub-second, so one network-shaped bound covers every mode.
fn timeout(&self) -> Option<std::time::Duration> {
Some(luxe_core::tool::NETWORK_TOOL_TIMEOUT)
}
fn schema(&self) -> ToolSchema {
ToolSchema {
name: "screenshot".to_string(),
description: "Capture a screenshot and return it inline (for visual UI verification) \
plus save it to .luxe/screenshots/. Cross-platform via the OS's native \
tool (Wayland/grim, X11/maim|scrot, macOS/screencapture, Windows/\
PowerShell); mode=url renders a page in headless Chrome/Chromium/Edge — \
the best way to check a web UI or dev-server output. Params by mode: \
fullscreen{output?}; region{region:\"X,Y WxH\"}; window{window?:title \
substring}; url{url, width?, height?, waitMs?}. `note` is echoed back.\n\
mode=url reaches public sites and this machine — localhost, 127.0.0.1 and \
::1 all work with no setup, so capturing your dev server just works. It \
refuses private-network hosts and cloud-metadata endpoints (reaching \
those is not this tool's job), and refuses file:// (use `read` for file \
contents). Only the requested host resolves: redirects, frames and \
assets from any other host fail to load.\n\
Prefer mode=url for anything web: it needs no desktop, no approval, and \
shows only what the page drew. The other three photograph the reader's \
actual screen, so they prompt for approval and are refused outright when \
no one is at the keyboard (headless, the loop, a review agent)."
.to_string(),
parameters: serde_json::json!({
"type": "object",
"properties": {
"mode": {
"type": "string",
"enum": ["fullscreen", "region", "window", "url"],
"description": "What to capture (default: url when a `url` is given, else fullscreen)"
},
"region": {
"type": "string",
"description": "region mode: capture rectangle as 'X,Y WxH', e.g. '0,0 1280x800'"
},
"window": {
"type": "string",
"description": "window mode: case-sensitive title substring; omit for the active window (Hyprland/X11 only)"
},
"url": {
"type": "string",
"description": "url mode: the http(s):// page to render and capture. Public or local (localhost/127.0.0.1/::1); private-network and cloud-metadata hosts are refused, and so is file://"
},
"output": {
"type": "string",
"description": "fullscreen mode: a specific monitor/output name (Wayland); omit for all"
},
"width": {
"type": "integer",
"description": "url mode: viewport width in px (default 1280)"
},
"height": {
"type": "integer",
"description": "url mode: viewport height in px (default 800)"
},
"waitMs": {
"type": "integer",
"description": "url mode: ms to let the page render before capture (default 1500)"
},
"note": {
"type": "string",
"description": "optional label echoed back with the result (what you're inspecting)"
}
}
}),
}
}
async fn run(&self, args: serde_json::Value, cx: &ToolContext) -> ToolResultData {
match self.capture(&args, cx).await {
Ok(result) => result,
Err(err) => ToolResultData::error(format!("Screenshot failed: {err:#}")),
}
}
}
impl ScreenshotTool {
async fn capture(
&self,
args: &serde_json::Value,
cx: &ToolContext,
) -> anyhow::Result<ToolResultData> {
let mode = luxe_core::permission::screenshot_mode(args);
let note = args.get("note").and_then(|v| v.as_str());
// Photographing the desktop needs somebody at the keyboard. The permission engine makes it
// an `ask`, and an `ask` auto-allows where nothing can prompt — which is precisely the run
// that must not have it: a headless session or a loop iteration would capture a screen no
// one is watching and send it to a provider. Same rule, and the same reasoning, as a
// language server out of the project's own `node_modules`.
if captures_the_desktop(mode) && cx.question_broker.is_none() {
anyhow::bail!(
"mode={mode} photographs the whole screen, so it runs only with someone at the \
keyboard to approve it — this session has no way to ask. Use mode=url to capture \
a page (a dev server, a local build), which needs no desktop and no approval."
);
}
let out_path = new_output_path(&cx.cwd)?;
let backend_label = if mode == "url" {
"headless browser".to_string()
} else {
capture::backend_label(capture::detect_backend()).to_string()
};
match mode {
"url" => {
let url = args
.get("url")
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow::anyhow!("mode=url requires a `url`"))?;
let width = args.get("width").and_then(|v| v.as_u64()).unwrap_or(1280) as u32;
let height = args.get("height").and_then(|v| v.as_u64()).unwrap_or(800) as u32;
let wait_ms = luxe_core::tool::arg(args, &["waitMs", "wait_ms"])
.and_then(|v| v.as_u64())
.unwrap_or(1500) as u32;
browser::capture_url(url, (width, height), wait_ms, &out_path, &cx.cwd).await?;
}
"fullscreen" | "region" | "window" => {
let target = build_target(mode, args)?;
let backend = capture::detect_backend();
capture::capture(backend, &target, &out_path).await?;
}
other => {
anyhow::bail!("unknown mode '{other}' (use fullscreen | region | window | url)")
}
}
encode_result(&out_path, mode, &backend_label, note)
}
}
/// Builds the capture target for a screen-capture mode from the tool arguments.
fn build_target(mode: &str, args: &serde_json::Value) -> anyhow::Result<CaptureTarget> {
match mode {
"fullscreen" => Ok(CaptureTarget::Fullscreen {
output: args.get("output").and_then(|v| v.as_str()).map(str::to_string),
}),
"region" => {
let geometry = args
.get("region")
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow::anyhow!("mode=region requires a `region` (\"X,Y WxH\")"))?;
Ok(CaptureTarget::Region { geometry: geometry.to_string() })
}
"window" => Ok(CaptureTarget::Window {
title: args.get("window").and_then(|v| v.as_str()).map(str::to_string),
}),
other => anyhow::bail!("unsupported capture mode '{other}'"),
}
}
/// Reads the captured PNG and packages it as an inline image plus a text summary.
fn encode_result(
out_path: &std::path::Path,
mode: &str,
backend_label: &str,
note: Option<&str>,
) -> anyhow::Result<ToolResultData> {
let bytes = std::fs::read(out_path)
.map_err(|err| anyhow::anyhow!("captured file could not be read: {err}"))?;
if bytes.is_empty() {
anyhow::bail!("capture produced an empty file");
}
// Keep the saved file full-resolution, but downscale the INLINE copy so a big multi-monitor
// capture doesn't blow up token cost.
let (inline_bytes, dimensions) = downscale_for_inline(&bytes);
let data = base64::engine::general_purpose::STANDARD.encode(&inline_bytes);
let mut summary = format!(
"Captured {mode} via {backend_label} ({dimensions}, {size} KB on disk). Saved: {path}",
size = bytes.len() / 1024,
path = luxe_core::paths::display(out_path)
);
if let Some(note) = note {
summary.push_str(&format!("\nNote: {note}"));
}
Ok(ToolResultData {
content: vec![
ContentBlock::Text(summary),
// The saved path travels with the pixels: a terminal that cannot draw the capture
// offers to open it instead, and a model looking at three of them can tell them apart.
ContentBlock::Image {
data,
mime: "image/png".to_string(),
name: Some(luxe_core::paths::display(out_path)),
},
],
is_error: false,
})
}
/// Longest edge (px) for the inline image sent to the model — keeps token cost sane while staying
/// legible (Anthropic's recommended max long edge).
const MAX_INLINE_DIM: u32 = 1568;
/// Returns the inline PNG bytes (downscaled if either side exceeds [`MAX_INLINE_DIM`]) plus a
/// human-readable dimensions string. On any decode/encode failure, falls back to the original
/// bytes so a capture is never lost to a resize hiccup.
fn downscale_for_inline(bytes: &[u8]) -> (Vec<u8>, String) {
let Ok(img) = image::load_from_memory(bytes) else {
return (bytes.to_vec(), "unknown size".to_string());
};
let (w, h) = (img.width(), img.height());
if w <= MAX_INLINE_DIM && h <= MAX_INLINE_DIM {
return (bytes.to_vec(), format!("{w}x{h}"));
}
let resized = img.resize(MAX_INLINE_DIM, MAX_INLINE_DIM, image::imageops::FilterType::Triangle);
let (nw, nh) = (resized.width(), resized.height());
let mut out = std::io::Cursor::new(Vec::new());
match resized.write_to(&mut out, image::ImageFormat::Png) {
Ok(()) => (out.into_inner(), format!("{w}x{h} → {nw}x{nh} inline")),
Err(_) => (bytes.to_vec(), format!("{w}x{h}")),
}
}
/// Allocates a fresh `<cwd>/.luxe/screenshots/shot-<millis>.png`, creating the directory.
fn new_output_path(cwd: &std::path::Path) -> anyhow::Result<PathBuf> {
let dir = cwd.join(".luxe").join("screenshots");
std::fs::create_dir_all(&dir)
.map_err(|err| anyhow::anyhow!("could not create {}: {err}", dir.display()))?;
let millis = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis()).unwrap_or(0);
Ok(dir.join(format!("shot-{millis}.png")))
}
/// Whether a screenshot could actually be taken here: a program that can photograph the desktop,
/// or a headless browser for URL captures.
///
/// On a CI runner or an SSH session with neither, every call this tool could make ends in an
/// install hint — so it is not worth a schema in every request. The desktop half asks for the
/// *program*, not merely a display: see `capture::desktop_capture_available`.
pub fn is_available() -> bool {
capture::desktop_capture_available(capture::detect_backend()) || browser::is_available()
}
/// Modes that photograph whatever is on the reader's screen, rather than a page Luxe rendered.
///
/// The distinction the permission engine and the unattended guard are both drawn on: `url` shows a
/// model a document it asked Luxe to load, and these show it the desktop — a password manager, a
/// private message, a colleague on a shared display.
fn captures_the_desktop(mode: &str) -> bool {
matches!(mode, "fullscreen" | "region" | "window")
}
#[cfg(test)]
mod tests {
use super::*;
fn tiny_png(w: u32, h: u32) -> Vec<u8> {
let img = image::RgbaImage::from_pixel(w, h, image::Rgba([10, 20, 30, 255]));
let mut out = std::io::Cursor::new(Vec::new());
image::DynamicImage::ImageRgba8(img).write_to(&mut out, image::ImageFormat::Png).unwrap();
out.into_inner()
}
#[test]
fn schema_exposes_four_modes() {
let tool = ScreenshotTool::new();
assert_eq!(tool.name(), "screenshot");
let schema = tool.schema();
let modes = schema.parameters["properties"]["mode"]["enum"].as_array().unwrap();
assert_eq!(modes.len(), 4, "fullscreen/region/window/url");
}
#[test]
fn region_target_requires_geometry() {
assert!(build_target("region", &serde_json::json!({})).is_err());
assert!(build_target("region", &serde_json::json!({ "region": "0,0 8x6" })).is_ok());
}
#[test]
fn small_images_pass_through_undownscaled() {
let png = tiny_png(64, 48);
let (bytes, dims) = downscale_for_inline(&png);
assert_eq!(bytes, png);
assert_eq!(dims, "64x48");
}
#[test]
fn large_images_are_downscaled_for_inline() {
let png = tiny_png(3000, 1000);
let (bytes, dims) = downscale_for_inline(&png);
assert!(dims.contains('→'), "dims note should show the resize: {dims}");
let img = image::load_from_memory(&bytes).unwrap();
assert!(img.width() <= MAX_INLINE_DIM && img.height() <= MAX_INLINE_DIM);
}
#[test]
fn output_path_creates_the_screenshots_dir() {
let tmp = tempfile::tempdir().unwrap();
let path = new_output_path(tmp.path()).unwrap();
assert!(path.parent().unwrap().is_dir());
assert!(path.to_string_lossy().ends_with(".png"));
}
/// Availability is a real capability question, and the answer is a *program*: a display is not
/// a screenshot tool. Wayland and X11 ship neither, so a desktop without `grim` must not
/// advertise this tool and then fail every capture with an install hint.
#[test]
fn availability_tracks_a_capture_program_or_a_browser() {
let can_capture = capture::desktop_capture_available(capture::detect_backend());
assert_eq!(is_available(), can_capture || browser::is_available());
}
/// A display with nothing installed to photograph it is not availability.
#[test]
fn a_bare_display_server_provides_no_capture() {
use capture::{desktop_capture_available, DisplayBackend};
// The two that carry their capture tool in the OS, and the one that carries nothing.
assert!(desktop_capture_available(DisplayBackend::MacOS));
assert!(desktop_capture_available(DisplayBackend::Windows));
assert!(!desktop_capture_available(DisplayBackend::Unknown));
}
/// The desktop modes are the ones that need a human; `url` renders a page and needs nobody.
#[test]
fn only_the_desktop_modes_photograph_the_reader() {
for mode in ["fullscreen", "region", "window"] {
assert!(captures_the_desktop(mode), "{mode}");
}
assert!(!captures_the_desktop("url"));
}
/// An unattended run refuses rather than capturing: an `ask` auto-allows where nothing can
/// prompt, which is exactly the session that must not photograph a screen nobody is watching.
/// The refusal names `mode=url`, because that is the one that still works there.
#[tokio::test]
async fn an_unattended_run_refuses_to_photograph_the_screen() {
let tmp = tempfile::tempdir().unwrap();
let (event_tx, _rx) = tokio::sync::mpsc::channel(8);
let (_cancel, cancel_rx) = tokio::sync::watch::channel(false);
let cx = ToolContext {
cwd: tmp.path().to_path_buf(),
session_id: "s".into(),
cancelled: std::sync::Arc::new(cancel_rx),
event_tx,
hook_ctx: None,
// The signal itself: `None` is "no front-end can ask a human anything".
question_broker: None,
};
let result =
ScreenshotTool::new().run(serde_json::json!({"mode": "fullscreen"}), &cx).await;
assert!(result.is_error, "an unattended desktop capture must not proceed");
let ContentBlock::Text(message) = &result.content[0] else { panic!("a text refusal") };
assert!(message.contains("someone at the keyboard"), "says why: {message}");
assert!(message.contains("mode=url"), "and what does work instead: {message}");
}
}