Files
luxe/crates/luxe-tools/src/web/search.rs
Evan Whittaker 09a7de2520 Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
2026-10-17 12:08:30 -05:00

339 lines
13 KiB
Rust

//! Search — the backends a `web_search` can run on, and the tool itself.
use super::render::{iso_date, render_hits};
use super::*;
/// A search backend (Brave, Tavily, Exa, Firecrawl, etc.).
///
/// Every backend is handed the whole [`SearchOptions`] set, because the tool schema advertises
/// those filters to the model on every request: a backend that quietly drops them answers a
/// different question from the one that was asked, and returns plausible results while doing it.
/// Where a vendor spells a filter differently — Brave has no domain parameter and takes `site:`
/// operators instead — the backend translates. Where it genuinely cannot, [`Self::honors`] says so
/// and the caller tells the model rather than letting it believe the filter applied.
#[async_trait]
pub trait SearchBackend: Send + Sync {
/// Run one search, shaped by everything the caller asked for.
async fn search(
&self,
query: &str,
n: usize,
options: &SearchOptions,
) -> Result<Vec<SearchHit>, WebError>;
/// What this backend does with the requested options, for the caller to report.
fn honors(&self, _options: &SearchOptions) -> Vec<String> {
Vec::new()
}
/// Whether one call covers every phrasing in [`SearchOptions::also`]. A plain search API does
/// not, so the caller runs them itself and merges.
fn batches_queries(&self) -> bool {
false
}
}
/// How recent a result has to be — the four windows `web_search` offers, in each vendor's terms.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Recency {
Day,
Week,
Month,
Year,
}
impl Recency {
/// Parse the tool schema's spelling.
pub fn parse(window: &str) -> Option<Self> {
match window {
"day" => Some(Self::Day),
"week" => Some(Self::Week),
"month" => Some(Self::Month),
"year" => Some(Self::Year),
_ => None,
}
}
/// The window as a count of days.
pub fn days(self) -> i64 {
match self {
Self::Day => 1,
Self::Week => 7,
Self::Month => 31,
Self::Year => 365,
}
}
/// The earliest publication date still inside the window, as `YYYY-MM-DD`.
pub fn since(self) -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_secs())
.unwrap_or_default()
.saturating_sub((self.days() * 86_400) as u64);
iso_date(secs)
}
/// Brave's `freshness` code.
pub fn brave_code(self) -> &'static str {
match self {
Self::Day => "pd",
Self::Week => "pw",
Self::Month => "pm",
Self::Year => "py",
}
}
/// Tavily's `time_range`.
pub fn tavily_range(self) -> &'static str {
match self {
Self::Day => "day",
Self::Week => "week",
Self::Month => "month",
Self::Year => "year",
}
}
/// The Google `tbs` window Firecrawl passes upstream.
pub fn tbs(self) -> &'static str {
match self {
Self::Day => "qdr:d",
Self::Week => "qdr:w",
Self::Month => "qdr:m",
Self::Year => "qdr:y",
}
}
}
/// What a caller can ask of a search, beyond the query itself.
///
/// Deliberately a small, provider-neutral set: the options that change *which* results come back
/// and whether their content rides along. Exa's own schema has thirty knobs; a tool schema that
/// exposed all of them would cost more context than the answers it fetches.
#[derive(Debug, Clone, Default)]
pub struct SearchOptions {
/// Extra query phrasings run alongside the main one, widening coverage in a single call.
pub also: Vec<String>,
/// Only these domains.
pub include_domains: Vec<String>,
/// Never these domains.
pub exclude_domains: Vec<String>,
/// Published no earlier than this window.
pub recency: Option<Recency>,
/// Carry extracted passages with each result instead of a bare snippet.
pub include_content: bool,
}
impl SearchOptions {
/// `site:`/`-site:` operators for a backend whose API has no domain filter of its own.
///
/// Every general web index understands them, which is why a missing parameter is not a missing
/// capability.
pub fn site_operators(&self) -> String {
let include = self.include_domains.iter().map(|domain| format!("site:{domain}"));
let exclude = self.exclude_domains.iter().map(|domain| format!("-site:{domain}"));
include.chain(exclude).collect::<Vec<_>>().join(" ")
}
/// `query` with those operators appended, which is how such a backend is asked.
pub fn applied_to(&self, query: &str) -> String {
let operators = self.site_operators();
match operators.is_empty() {
true => query.to_string(),
false => format!("{query} {operators}"),
}
}
}
/// Normalized search result.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchHit {
pub title: String,
pub url: String,
pub snippet: String,
}
/// Create a search backend based on configuration.
///
/// No key means Exa's keyless MCP endpoint rather than an error — see [`WebConfig::from_env`].
pub async fn create_backend(config: &WebConfig) -> Result<Box<dyn SearchBackend>, WebError> {
let Some(api_key) = config.search_api_key.as_deref() else {
return Ok(Box::new(crate::exa_mcp::ExaMcpBackend::new()));
};
match config.search_backend.as_deref().unwrap_or("exa_mcp") {
"brave" => Ok(Box::new(crate::brave::BraveBackend::new(api_key.to_string()))),
"tavily" => Ok(Box::new(crate::tavily::TavilyBackend::new(api_key.to_string()))),
"exa" => Ok(Box::new(crate::exa::ExaBackend::new(api_key.to_string()))),
"firecrawl" => Ok(Box::new(crate::firecrawl::FirecrawlBackend::new(api_key.to_string()))),
"exa_mcp" => Ok(Box::new(crate::exa_mcp::ExaMcpBackend::new())),
other => Err(WebError::Backend(format!("Unknown search backend: {other}"))),
}
}
/// Run every phrasing the caller asked for and merge the results.
///
/// One call per query for a backend that takes one query, first-seen order preserved and duplicate
/// URLs dropped — the same page found by two phrasings is one result, and the earlier phrasing is
/// the one that ranked it. A phrasing that errors is skipped rather than sinking the whole search;
/// the error only surfaces when *nothing* answered.
pub(super) async fn run_search(
backend: &dyn SearchBackend,
queries: &[String],
n: usize,
options: &SearchOptions,
) -> Result<Vec<SearchHit>, WebError> {
let Some((query, also)) = queries.split_first() else {
return Ok(Vec::new());
};
if backend.batches_queries() || also.is_empty() {
return backend.search(query, n, options).await;
}
// The extras are already in `queries`; passing them again would double a backend that reads
// both. One phrasing at a time, and the option set minus the fan-out it has already served.
let single = SearchOptions { also: Vec::new(), ..options.clone() };
let mut merged: Vec<SearchHit> = Vec::new();
let mut last_error = None;
for query in queries {
match backend.search(query, n, &single).await {
Ok(hits) => {
for hit in hits {
if !merged.iter().any(|seen| seen.url == hit.url) {
merged.push(hit);
}
}
}
Err(error) => last_error = Some(error),
}
}
match (merged.is_empty(), last_error) {
(true, Some(error)) => Err(error),
_ => Ok(merged),
}
}
/// web_search tool — pluggable search backend.
pub struct WebSearchTool {
pub(super) config: Arc<RwLock<WebConfig>>,
}
#[async_trait]
impl Tool for WebSearchTool {
fn name(&self) -> &str {
"web_search"
}
/// A search backend that has stopped answering is not worth a two-minute wait.
fn timeout(&self) -> Option<std::time::Duration> {
Some(luxe_core::tool::NETWORK_TOOL_TIMEOUT)
}
fn schema(&self) -> ToolSchema {
ToolSchema {
name: "web_search".to_string(),
description: "Search the web. Each result carries the passage the search engine \
matched, which often answers the question outright — `web_fetch` a URL \
only when you need more of the page than came back. Pass several \
`queries` to cover a topic from different angles in one call (varied \
phrasing beats repeating one query). Narrow with `domains` (prefix `-` \
to exclude) and `recency` when currency matters; a reply that leads with \
`Note:` is saying which of those the configured search vendor could not \
apply."
.to_string(),
parameters: serde_json::json!({
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query"},
"queries": {
"type": "array",
"items": {"type": "string"},
"description": "Two to four varied phrasings of the same question, searched together. Prefer this over one query for research."
},
"n": {"type": "integer", "description": "Results to return (default 5, max 20)"},
"domains": {
"type": "array",
"items": {"type": "string"},
"description": "Limit to these domains, e.g. ['docs.rs']; prefix with '-' to exclude, e.g. '-pinterest.com'."
},
"recency": {
"type": "string",
"enum": ["day", "week", "month", "year"],
"description": "Only pages published within this window."
},
"content": {
"type": "boolean",
"description": "Include the relevant passage from each result (default true). Set false for a bare link list."
}
},
"required": []
}),
}
}
async fn run(&self, args: serde_json::Value, _cx: &ToolContext) -> ToolResultData {
let queries = search_queries(&args);
let Some((_, also)) = queries.split_first() else {
return ToolResultData::error("Error: 'query' (or 'queries') is required");
};
let n = (args.get("n").and_then(serde_json::Value::as_u64).unwrap_or(5) as usize)
.clamp(1, MAX_RESULTS);
let options = SearchOptions {
also: also.to_vec(),
include_content: args
.get("content")
.and_then(serde_json::Value::as_bool)
.unwrap_or(true),
recency: args
.get("recency")
.and_then(serde_json::Value::as_str)
.and_then(Recency::parse),
..domain_filters(&args)
};
let config = self.config.read().await;
let backend = match create_backend(&config).await {
Ok(backend) => backend,
Err(e) => return ToolResultData::error(format!("Backend error: {e}")),
};
match run_search(backend.as_ref(), &queries, n, &options).await {
Ok(hits) => {
ToolResultData::text(render_hits(&queries, &hits, &backend.honors(&options)))
}
Err(e) => ToolResultData::error(format!("Search failed: {e}")),
}
}
}
/// Most results any backend will return for one query. Brave refuses more than 20 outright, so the
/// ceiling is the lowest that every vendor accepts rather than the highest one of them allows.
pub(super) const MAX_RESULTS: usize = 20;
/// The queries to run: `queries` when given, else the single `query`, trimmed and de-duplicated.
///
/// Owned by `luxe_core` because the tool card lists the same set under the call's header, and a
/// second implementation would let the card and the search disagree about what ran.
pub(super) fn search_queries(args: &serde_json::Value) -> Vec<String> {
luxe_core::web_search_queries(args)
}
/// Split `domains` into include/exclude sets; a leading `-` excludes.
pub(super) fn domain_filters(args: &serde_json::Value) -> SearchOptions {
let mut options = SearchOptions::default();
let Some(list) = args.get("domains").and_then(serde_json::Value::as_array) else {
return options;
};
for entry in list.iter().filter_map(serde_json::Value::as_str) {
let entry = entry.trim();
// A bare `-` names no domain either way.
if entry.is_empty() || entry == "-" {
continue;
}
match entry.strip_prefix('-') {
Some(excluded) => options.exclude_domains.push(excluded.to_string()),
None => options.include_domains.push(entry.to_string()),
}
}
options
}