Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
390
crates/luxe-tools/src/screenshot.rs
Normal file
390
crates/luxe-tools/src/screenshot.rs
Normal file
@@ -0,0 +1,390 @@
|
||||
//! `screenshot` tool — cross-platform screen/window/URL capture for visual verification.
|
||||
//!
|
||||
//! Captures the screen, a region, a window, or a rendered URL to a PNG, then returns it inline
|
||||
//! (so vision-capable models can see it) alongside the saved path. Uses the platform's native
|
||||
//! capture tool (see `capture`) — no heavyweight capture crate — so the dependency surface and
|
||||
//! the supply-chain gate stay clean.
|
||||
|
||||
mod browser;
|
||||
mod capture;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine;
|
||||
use luxe_core::provider::ToolSchema;
|
||||
use luxe_core::{ContentBlock, Tool, ToolContext, ToolResultData};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use capture::CaptureTarget;
|
||||
|
||||
/// The `screenshot` tool.
|
||||
#[derive(Default)]
|
||||
pub struct ScreenshotTool;
|
||||
|
||||
impl ScreenshotTool {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for ScreenshotTool {
|
||||
fn name(&self) -> &str {
|
||||
"screenshot"
|
||||
}
|
||||
|
||||
/// `mode=url` hands a URL to a headless browser, and a browser told to load a host that never
|
||||
/// answers waits forever — `--virtual-time-budget` bounds the page's *clock*, not its network.
|
||||
/// The native capture backends are sub-second, so one network-shaped bound covers every mode.
|
||||
fn timeout(&self) -> Option<std::time::Duration> {
|
||||
Some(luxe_core::tool::NETWORK_TOOL_TIMEOUT)
|
||||
}
|
||||
|
||||
fn schema(&self) -> ToolSchema {
|
||||
ToolSchema {
|
||||
name: "screenshot".to_string(),
|
||||
description: "Capture a screenshot and return it inline (for visual UI verification) \
|
||||
plus save it to .luxe/screenshots/. Cross-platform via the OS's native \
|
||||
tool (Wayland/grim, X11/maim|scrot, macOS/screencapture, Windows/\
|
||||
PowerShell); mode=url renders a page in headless Chrome/Chromium/Edge — \
|
||||
the best way to check a web UI or dev-server output. Params by mode: \
|
||||
fullscreen{output?}; region{region:\"X,Y WxH\"}; window{window?:title \
|
||||
substring}; url{url, width?, height?, waitMs?}. `note` is echoed back.\n\
|
||||
mode=url reaches public sites and this machine — localhost, 127.0.0.1 and \
|
||||
::1 all work with no setup, so capturing your dev server just works. It \
|
||||
refuses private-network hosts and cloud-metadata endpoints (reaching \
|
||||
those is not this tool's job), and refuses file:// (use `read` for file \
|
||||
contents). Only the requested host resolves: redirects, frames and \
|
||||
assets from any other host fail to load.\n\
|
||||
Prefer mode=url for anything web: it needs no desktop, no approval, and \
|
||||
shows only what the page drew. The other three photograph the reader's \
|
||||
actual screen, so they prompt for approval and are refused outright when \
|
||||
no one is at the keyboard (headless, the loop, a review agent)."
|
||||
.to_string(),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["fullscreen", "region", "window", "url"],
|
||||
"description": "What to capture (default: url when a `url` is given, else fullscreen)"
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
"description": "region mode: capture rectangle as 'X,Y WxH', e.g. '0,0 1280x800'"
|
||||
},
|
||||
"window": {
|
||||
"type": "string",
|
||||
"description": "window mode: case-sensitive title substring; omit for the active window (Hyprland/X11 only)"
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "url mode: the http(s):// page to render and capture. Public or local (localhost/127.0.0.1/::1); private-network and cloud-metadata hosts are refused, and so is file://"
|
||||
},
|
||||
"output": {
|
||||
"type": "string",
|
||||
"description": "fullscreen mode: a specific monitor/output name (Wayland); omit for all"
|
||||
},
|
||||
"width": {
|
||||
"type": "integer",
|
||||
"description": "url mode: viewport width in px (default 1280)"
|
||||
},
|
||||
"height": {
|
||||
"type": "integer",
|
||||
"description": "url mode: viewport height in px (default 800)"
|
||||
},
|
||||
"waitMs": {
|
||||
"type": "integer",
|
||||
"description": "url mode: ms to let the page render before capture (default 1500)"
|
||||
},
|
||||
"note": {
|
||||
"type": "string",
|
||||
"description": "optional label echoed back with the result (what you're inspecting)"
|
||||
}
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(&self, args: serde_json::Value, cx: &ToolContext) -> ToolResultData {
|
||||
match self.capture(&args, cx).await {
|
||||
Ok(result) => result,
|
||||
Err(err) => ToolResultData::error(format!("Screenshot failed: {err:#}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ScreenshotTool {
|
||||
async fn capture(
|
||||
&self,
|
||||
args: &serde_json::Value,
|
||||
cx: &ToolContext,
|
||||
) -> anyhow::Result<ToolResultData> {
|
||||
let mode = luxe_core::permission::screenshot_mode(args);
|
||||
let note = args.get("note").and_then(|v| v.as_str());
|
||||
|
||||
// Photographing the desktop needs somebody at the keyboard. The permission engine makes it
|
||||
// an `ask`, and an `ask` auto-allows where nothing can prompt — which is precisely the run
|
||||
// that must not have it: a headless session or a loop iteration would capture a screen no
|
||||
// one is watching and send it to a provider. Same rule, and the same reasoning, as a
|
||||
// language server out of the project's own `node_modules`.
|
||||
if captures_the_desktop(mode) && cx.question_broker.is_none() {
|
||||
anyhow::bail!(
|
||||
"mode={mode} photographs the whole screen, so it runs only with someone at the \
|
||||
keyboard to approve it — this session has no way to ask. Use mode=url to capture \
|
||||
a page (a dev server, a local build), which needs no desktop and no approval."
|
||||
);
|
||||
}
|
||||
|
||||
let out_path = new_output_path(&cx.cwd)?;
|
||||
|
||||
let backend_label = if mode == "url" {
|
||||
"headless browser".to_string()
|
||||
} else {
|
||||
capture::backend_label(capture::detect_backend()).to_string()
|
||||
};
|
||||
|
||||
match mode {
|
||||
"url" => {
|
||||
let url = args
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("mode=url requires a `url`"))?;
|
||||
let width = args.get("width").and_then(|v| v.as_u64()).unwrap_or(1280) as u32;
|
||||
let height = args.get("height").and_then(|v| v.as_u64()).unwrap_or(800) as u32;
|
||||
let wait_ms = luxe_core::tool::arg(args, &["waitMs", "wait_ms"])
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(1500) as u32;
|
||||
browser::capture_url(url, (width, height), wait_ms, &out_path, &cx.cwd).await?;
|
||||
}
|
||||
"fullscreen" | "region" | "window" => {
|
||||
let target = build_target(mode, args)?;
|
||||
let backend = capture::detect_backend();
|
||||
capture::capture(backend, &target, &out_path).await?;
|
||||
}
|
||||
other => {
|
||||
anyhow::bail!("unknown mode '{other}' (use fullscreen | region | window | url)")
|
||||
}
|
||||
}
|
||||
|
||||
encode_result(&out_path, mode, &backend_label, note)
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the capture target for a screen-capture mode from the tool arguments.
|
||||
fn build_target(mode: &str, args: &serde_json::Value) -> anyhow::Result<CaptureTarget> {
|
||||
match mode {
|
||||
"fullscreen" => Ok(CaptureTarget::Fullscreen {
|
||||
output: args.get("output").and_then(|v| v.as_str()).map(str::to_string),
|
||||
}),
|
||||
"region" => {
|
||||
let geometry = args
|
||||
.get("region")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("mode=region requires a `region` (\"X,Y WxH\")"))?;
|
||||
Ok(CaptureTarget::Region { geometry: geometry.to_string() })
|
||||
}
|
||||
"window" => Ok(CaptureTarget::Window {
|
||||
title: args.get("window").and_then(|v| v.as_str()).map(str::to_string),
|
||||
}),
|
||||
other => anyhow::bail!("unsupported capture mode '{other}'"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the captured PNG and packages it as an inline image plus a text summary.
|
||||
fn encode_result(
|
||||
out_path: &std::path::Path,
|
||||
mode: &str,
|
||||
backend_label: &str,
|
||||
note: Option<&str>,
|
||||
) -> anyhow::Result<ToolResultData> {
|
||||
let bytes = std::fs::read(out_path)
|
||||
.map_err(|err| anyhow::anyhow!("captured file could not be read: {err}"))?;
|
||||
if bytes.is_empty() {
|
||||
anyhow::bail!("capture produced an empty file");
|
||||
}
|
||||
// Keep the saved file full-resolution, but downscale the INLINE copy so a big multi-monitor
|
||||
// capture doesn't blow up token cost.
|
||||
let (inline_bytes, dimensions) = downscale_for_inline(&bytes);
|
||||
let data = base64::engine::general_purpose::STANDARD.encode(&inline_bytes);
|
||||
let mut summary = format!(
|
||||
"Captured {mode} via {backend_label} ({dimensions}, {size} KB on disk). Saved: {path}",
|
||||
size = bytes.len() / 1024,
|
||||
path = luxe_core::paths::display(out_path)
|
||||
);
|
||||
if let Some(note) = note {
|
||||
summary.push_str(&format!("\nNote: {note}"));
|
||||
}
|
||||
Ok(ToolResultData {
|
||||
content: vec![
|
||||
ContentBlock::Text(summary),
|
||||
// The saved path travels with the pixels: a terminal that cannot draw the capture
|
||||
// offers to open it instead, and a model looking at three of them can tell them apart.
|
||||
ContentBlock::Image {
|
||||
data,
|
||||
mime: "image/png".to_string(),
|
||||
name: Some(luxe_core::paths::display(out_path)),
|
||||
},
|
||||
],
|
||||
is_error: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Longest edge (px) for the inline image sent to the model — keeps token cost sane while staying
|
||||
/// legible (Anthropic's recommended max long edge).
|
||||
const MAX_INLINE_DIM: u32 = 1568;
|
||||
|
||||
/// Returns the inline PNG bytes (downscaled if either side exceeds [`MAX_INLINE_DIM`]) plus a
|
||||
/// human-readable dimensions string. On any decode/encode failure, falls back to the original
|
||||
/// bytes so a capture is never lost to a resize hiccup.
|
||||
fn downscale_for_inline(bytes: &[u8]) -> (Vec<u8>, String) {
|
||||
let Ok(img) = image::load_from_memory(bytes) else {
|
||||
return (bytes.to_vec(), "unknown size".to_string());
|
||||
};
|
||||
let (w, h) = (img.width(), img.height());
|
||||
if w <= MAX_INLINE_DIM && h <= MAX_INLINE_DIM {
|
||||
return (bytes.to_vec(), format!("{w}x{h}"));
|
||||
}
|
||||
let resized = img.resize(MAX_INLINE_DIM, MAX_INLINE_DIM, image::imageops::FilterType::Triangle);
|
||||
let (nw, nh) = (resized.width(), resized.height());
|
||||
let mut out = std::io::Cursor::new(Vec::new());
|
||||
match resized.write_to(&mut out, image::ImageFormat::Png) {
|
||||
Ok(()) => (out.into_inner(), format!("{w}x{h} → {nw}x{nh} inline")),
|
||||
Err(_) => (bytes.to_vec(), format!("{w}x{h}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Allocates a fresh `<cwd>/.luxe/screenshots/shot-<millis>.png`, creating the directory.
|
||||
fn new_output_path(cwd: &std::path::Path) -> anyhow::Result<PathBuf> {
|
||||
let dir = cwd.join(".luxe").join("screenshots");
|
||||
std::fs::create_dir_all(&dir)
|
||||
.map_err(|err| anyhow::anyhow!("could not create {}: {err}", dir.display()))?;
|
||||
let millis = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis()).unwrap_or(0);
|
||||
Ok(dir.join(format!("shot-{millis}.png")))
|
||||
}
|
||||
|
||||
/// Whether a screenshot could actually be taken here: a program that can photograph the desktop,
|
||||
/// or a headless browser for URL captures.
|
||||
///
|
||||
/// On a CI runner or an SSH session with neither, every call this tool could make ends in an
|
||||
/// install hint — so it is not worth a schema in every request. The desktop half asks for the
|
||||
/// *program*, not merely a display: see `capture::desktop_capture_available`.
|
||||
pub fn is_available() -> bool {
|
||||
capture::desktop_capture_available(capture::detect_backend()) || browser::is_available()
|
||||
}
|
||||
|
||||
/// Modes that photograph whatever is on the reader's screen, rather than a page Luxe rendered.
|
||||
///
|
||||
/// The distinction the permission engine and the unattended guard are both drawn on: `url` shows a
|
||||
/// model a document it asked Luxe to load, and these show it the desktop — a password manager, a
|
||||
/// private message, a colleague on a shared display.
|
||||
fn captures_the_desktop(mode: &str) -> bool {
|
||||
matches!(mode, "fullscreen" | "region" | "window")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tiny_png(w: u32, h: u32) -> Vec<u8> {
|
||||
let img = image::RgbaImage::from_pixel(w, h, image::Rgba([10, 20, 30, 255]));
|
||||
let mut out = std::io::Cursor::new(Vec::new());
|
||||
image::DynamicImage::ImageRgba8(img).write_to(&mut out, image::ImageFormat::Png).unwrap();
|
||||
out.into_inner()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_exposes_four_modes() {
|
||||
let tool = ScreenshotTool::new();
|
||||
assert_eq!(tool.name(), "screenshot");
|
||||
let schema = tool.schema();
|
||||
let modes = schema.parameters["properties"]["mode"]["enum"].as_array().unwrap();
|
||||
assert_eq!(modes.len(), 4, "fullscreen/region/window/url");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn region_target_requires_geometry() {
|
||||
assert!(build_target("region", &serde_json::json!({})).is_err());
|
||||
assert!(build_target("region", &serde_json::json!({ "region": "0,0 8x6" })).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_images_pass_through_undownscaled() {
|
||||
let png = tiny_png(64, 48);
|
||||
let (bytes, dims) = downscale_for_inline(&png);
|
||||
assert_eq!(bytes, png);
|
||||
assert_eq!(dims, "64x48");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_images_are_downscaled_for_inline() {
|
||||
let png = tiny_png(3000, 1000);
|
||||
let (bytes, dims) = downscale_for_inline(&png);
|
||||
assert!(dims.contains('→'), "dims note should show the resize: {dims}");
|
||||
let img = image::load_from_memory(&bytes).unwrap();
|
||||
assert!(img.width() <= MAX_INLINE_DIM && img.height() <= MAX_INLINE_DIM);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_path_creates_the_screenshots_dir() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let path = new_output_path(tmp.path()).unwrap();
|
||||
assert!(path.parent().unwrap().is_dir());
|
||||
assert!(path.to_string_lossy().ends_with(".png"));
|
||||
}
|
||||
|
||||
/// Availability is a real capability question, and the answer is a *program*: a display is not
|
||||
/// a screenshot tool. Wayland and X11 ship neither, so a desktop without `grim` must not
|
||||
/// advertise this tool and then fail every capture with an install hint.
|
||||
#[test]
|
||||
fn availability_tracks_a_capture_program_or_a_browser() {
|
||||
let can_capture = capture::desktop_capture_available(capture::detect_backend());
|
||||
assert_eq!(is_available(), can_capture || browser::is_available());
|
||||
}
|
||||
|
||||
/// A display with nothing installed to photograph it is not availability.
|
||||
#[test]
|
||||
fn a_bare_display_server_provides_no_capture() {
|
||||
use capture::{desktop_capture_available, DisplayBackend};
|
||||
// The two that carry their capture tool in the OS, and the one that carries nothing.
|
||||
assert!(desktop_capture_available(DisplayBackend::MacOS));
|
||||
assert!(desktop_capture_available(DisplayBackend::Windows));
|
||||
assert!(!desktop_capture_available(DisplayBackend::Unknown));
|
||||
}
|
||||
|
||||
/// The desktop modes are the ones that need a human; `url` renders a page and needs nobody.
|
||||
#[test]
|
||||
fn only_the_desktop_modes_photograph_the_reader() {
|
||||
for mode in ["fullscreen", "region", "window"] {
|
||||
assert!(captures_the_desktop(mode), "{mode}");
|
||||
}
|
||||
assert!(!captures_the_desktop("url"));
|
||||
}
|
||||
|
||||
/// An unattended run refuses rather than capturing: an `ask` auto-allows where nothing can
|
||||
/// prompt, which is exactly the session that must not photograph a screen nobody is watching.
|
||||
/// The refusal names `mode=url`, because that is the one that still works there.
|
||||
#[tokio::test]
|
||||
async fn an_unattended_run_refuses_to_photograph_the_screen() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (event_tx, _rx) = tokio::sync::mpsc::channel(8);
|
||||
let (_cancel, cancel_rx) = tokio::sync::watch::channel(false);
|
||||
let cx = ToolContext {
|
||||
cwd: tmp.path().to_path_buf(),
|
||||
session_id: "s".into(),
|
||||
cancelled: std::sync::Arc::new(cancel_rx),
|
||||
event_tx,
|
||||
hook_ctx: None,
|
||||
// The signal itself: `None` is "no front-end can ask a human anything".
|
||||
question_broker: None,
|
||||
};
|
||||
|
||||
let result =
|
||||
ScreenshotTool::new().run(serde_json::json!({"mode": "fullscreen"}), &cx).await;
|
||||
|
||||
assert!(result.is_error, "an unattended desktop capture must not proceed");
|
||||
let ContentBlock::Text(message) = &result.content[0] else { panic!("a text refusal") };
|
||||
assert!(message.contains("someone at the keyboard"), "says why: {message}");
|
||||
assert!(message.contains("mode=url"), "and what does work instead: {message}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user