Files
luxe/crates/luxe-tools/src/web.rs
Evan Whittaker 09a7de2520 Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
2026-10-17 12:08:30 -05:00

155 lines
5.4 KiB
Rust

// Web search and fetch tools.
//
// web_search — pluggable search backend (Brave/Tavily/Exa/Firecrawl) returning normalized hits.
// web_fetch — GET URL, extract readable content → markdown, with optional stealth fallback.
mod clone;
mod fetch;
mod render;
mod search;
pub use fetch::WebFetchTool;
pub use search::{create_backend, Recency, SearchBackend, SearchHit, SearchOptions, WebSearchTool};
use anyhow::Result;
use async_trait::async_trait;
use luxe_core::provider::ToolSchema;
use luxe_core::tool::{Tool, ToolContext, ToolResultData};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::RwLock;
use url::Url;
/// Web tool error.
#[derive(Debug, thiserror::Error)]
pub enum WebError {
#[error("HTTP error: {0}")]
Http(String),
#[error("network error: {0}")]
Network(String),
/// The far end never finished answering: a name that would not resolve, or a socket that
/// connected and then went quiet. Its own variant because it is the one failure a retry cannot
/// help — see `WebError::worth_a_second_try`.
#[error("timeout: {0}")]
Timeout(String),
#[error("parse error: {0}")]
Parse(String),
#[error("backend error: {0}")]
Backend(String),
}
impl WebError {
/// Whether refetching with a browser-like user agent could plausibly do better.
///
/// The stealth retry exists to get past bot protection — a 403, a challenge page, a body that is
/// all JavaScript — and it works by looking like a browser. A host that never answered does not
/// care what we call ourselves, so retrying a timeout only spends the budget twice: two 30s
/// attempts against a silent host is a minute of nothing, which is exactly how a dead domain came
/// to look like a hung agent.
fn worth_a_second_try(&self) -> bool {
!matches!(self, Self::Timeout(_))
}
}
/// Configuration for web tools.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebConfig {
/// Search backend to use (brave, tavily, exa, firecrawl).
pub search_backend: Option<String>,
/// API key for the search backend.
pub search_api_key: Option<String>,
/// Enable stealth fetch fallback.
pub enable_stealth: bool,
/// Firecrawl key, held apart from the search key because it also backs `web_fetch`'s last
/// resort — so it is worth having even when another vendor won the search slot.
#[serde(default)]
pub firecrawl_api_key: Option<String>,
/// Whether a GitHub repository URL is cloned into `.luxe/gh-cache/` rather than read as a page.
#[serde(default = "clone_by_default")]
pub clone_repositories: bool,
}
impl Default for WebConfig {
fn default() -> Self {
Self {
search_backend: None,
search_api_key: None,
enable_stealth: false,
firecrawl_api_key: None,
clone_repositories: clone_by_default(),
}
}
}
fn clone_by_default() -> bool {
true
}
/// The keyed search vendors, in the order a configured key claims the slot.
const SEARCH_KEYS: &[(&str, &str)] = &[
("brave", "BRAVE_API_KEY"),
("tavily", "TAVILY_API_KEY"),
("exa", "EXA_API_KEY"),
("firecrawl", "FIRECRAWL_API_KEY"),
];
impl WebConfig {
/// Build config from the environment: a configured key picks its backend, in `SEARCH_KEYS`
/// order, and otherwise search falls back to Exa's keyless MCP endpoint.
///
/// An explicit key outranks the fallback — someone who configured one chose it. But search is
/// a *capability*, not a setup step: without this fallback, `web_search` was registered and
/// then failed on first use for every user who had not signed up for a search vendor.
pub fn from_env() -> Self {
let (backend, key) = SEARCH_KEYS
.iter()
.find_map(|(name, var)| env_key(var).map(|key| ((*name).to_string(), key)))
.map_or((None, None), |(name, key)| (Some(name), Some(key)));
Self {
search_backend: backend,
search_api_key: key,
enable_stealth: true,
firecrawl_api_key: env_key("FIRECRAWL_API_KEY"),
clone_repositories: clone_by_default(),
}
}
/// Whether a search backend key is configured. Search works either way — this only says
/// whether it will go through a keyed vendor or the keyless fallback.
pub fn has_search_key(&self) -> bool {
self.search_api_key.is_some()
}
}
/// A non-blank environment value, since an exported-but-empty key is not a key.
fn env_key(var: &str) -> Option<String> {
std::env::var(var).ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty())
}
/// Web tools (search + fetch) with shared configuration.
pub struct WebTools {
config: Arc<RwLock<WebConfig>>,
/// Read out at construction, because the schema promises it and a schema is built synchronously
clone_repositories: bool,
}
impl WebTools {
pub fn new(config: WebConfig) -> Self {
let clone_repositories = config.clone_repositories;
Self { config: Arc::new(RwLock::new(config)), clone_repositories }
}
/// Create a web_search tool.
pub fn web_search(&self) -> WebSearchTool {
WebSearchTool { config: self.config.clone() }
}
/// Create a web_fetch tool.
pub fn web_fetch(&self) -> WebFetchTool {
WebFetchTool { config: self.config.clone(), clone_repositories: self.clone_repositories }
}
}
#[cfg(test)]
mod tests;