Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
201 lines
7.6 KiB
Rust
201 lines
7.6 KiB
Rust
//! Live smoke tests for every `web_search` backend, run against the real vendors.
|
|
//!
|
|
//! Skipped unless `LUXE_TEST_LIVE_WEB=1`, so the suite stays offline; each keyed backend is
|
|
//! skipped again unless its key is in the environment. Unit tests pin the *request* each backend
|
|
//! builds, which is where the silent bugs live — a filter serialized under a name the vendor does
|
|
//! not read is accepted, ignored, and answered 200. These pin the other half: that the vendor
|
|
//! agrees, and that what comes back is shaped like an answer.
|
|
//!
|
|
//! Every backend here shipped once without this. Brave interpolated the query straight into a URL,
|
|
//! so `tokio select! & cancellation` reached it as `tokio select!`; Exa asked for `num_results`,
|
|
//! which it does not read, and never requested page text at all. Both returned confident results
|
|
//! for a question nobody asked.
|
|
//!
|
|
//! ```bash
|
|
//! LUXE_TEST_LIVE_WEB=1 cargo test -p luxe-tools --test live_web_search -- --nocapture
|
|
//! ```
|
|
|
|
use luxe_tools::brave::BraveBackend;
|
|
use luxe_tools::exa::ExaBackend;
|
|
use luxe_tools::firecrawl::FirecrawlBackend;
|
|
use luxe_tools::tavily::TavilyBackend;
|
|
use luxe_tools::web::{Recency, SearchBackend, SearchHit, SearchOptions};
|
|
|
|
fn enabled() -> bool {
|
|
std::env::var("LUXE_TEST_LIVE_WEB").is_ok_and(|value| value == "1")
|
|
}
|
|
|
|
/// The key for `var`, or `None` with a note saying which test was skipped and why.
|
|
fn key(var: &str) -> Option<String> {
|
|
if !enabled() {
|
|
eprintln!("skipped: set LUXE_TEST_LIVE_WEB=1 to run");
|
|
return None;
|
|
}
|
|
match std::env::var(var).ok().filter(|value| !value.trim().is_empty()) {
|
|
Some(key) => Some(key),
|
|
None => {
|
|
eprintln!("skipped: {var} is not set");
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Every backend owes the caller the same three things, whatever it costs to get them.
|
|
fn assert_usable(hits: &[SearchHit], what: &str) {
|
|
assert!(!hits.is_empty(), "{what}: a mainstream query must return something");
|
|
for hit in hits {
|
|
assert!(hit.url.starts_with("http"), "{what}: every hit carries a usable url: {hit:?}");
|
|
assert!(!hit.title.trim().is_empty(), "{what}: and a title: {hit:?}");
|
|
}
|
|
assert!(
|
|
hits.iter().any(|hit| hit.snippet.chars().count() > 80),
|
|
"{what}: a result with no text to read is a link the model has to spend a fetch on: \
|
|
{hits:#?}"
|
|
);
|
|
eprintln!("{what}: {} hits · first: {} ({})", hits.len(), hits[0].title, hits[0].url);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn brave_searches_the_query_it_was_given() {
|
|
let Some(api_key) = key("BRAVE_API_KEY") else { return };
|
|
let backend = BraveBackend::new(api_key);
|
|
|
|
let hits = backend
|
|
.search("tokio cancellation safety", 3, &SearchOptions::default())
|
|
.await
|
|
.expect("Brave answers");
|
|
assert_usable(&hits, "brave");
|
|
|
|
// The punctuation that used to truncate the query. Brave answers 200 either way, so the only
|
|
// observable difference is whether the results are about the right thing.
|
|
let punctuated = backend
|
|
.search("rust #[tokio::main] macro & async fn", 3, &SearchOptions::default())
|
|
.await
|
|
.expect("Brave accepts a punctuated query");
|
|
assert!(!punctuated.is_empty(), "a punctuated query must still search");
|
|
|
|
let filtered = backend
|
|
.search(
|
|
"cancellation",
|
|
5,
|
|
&SearchOptions {
|
|
include_domains: vec!["docs.rs".to_string()],
|
|
recency: Some(Recency::Year),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
.expect("Brave accepts site: operators and a freshness code");
|
|
assert!(
|
|
filtered.iter().all(|hit| hit.url.contains("docs.rs")),
|
|
"site: operators are the domain filter Brave actually reads: {filtered:#?}"
|
|
);
|
|
|
|
// Above 20 Brave answers 422 rather than clamping, so the clamp has to be ours.
|
|
assert!(backend.search("rust", 50, &SearchOptions::default()).await.is_ok());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tavily_applies_every_filter_the_schema_advertises() {
|
|
let Some(api_key) = key("TAVILY_API_KEY") else { return };
|
|
let backend = TavilyBackend::new(api_key);
|
|
|
|
let hits = backend
|
|
.search(
|
|
"tokio cancellation safety",
|
|
3,
|
|
&SearchOptions { include_content: true, ..Default::default() },
|
|
)
|
|
.await
|
|
.expect("Tavily answers a bearer-authenticated request");
|
|
assert_usable(&hits, "tavily");
|
|
|
|
let filtered = backend
|
|
.search(
|
|
"tokio",
|
|
5,
|
|
&SearchOptions {
|
|
include_domains: vec!["docs.rs".to_string()],
|
|
recency: Some(Recency::Year),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
.expect("Tavily accepts the filters");
|
|
assert!(
|
|
filtered.iter().all(|hit| hit.url.contains("docs.rs")),
|
|
"include_domains is honored server-side: {filtered:#?}"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn exa_returns_the_count_asked_for_and_the_page_text() {
|
|
let Some(api_key) = key("EXA_API_KEY") else { return };
|
|
let backend = ExaBackend::new(api_key);
|
|
|
|
let hits = backend
|
|
.search(
|
|
"tokio cancellation safety",
|
|
3,
|
|
&SearchOptions { include_content: true, ..Default::default() },
|
|
)
|
|
.await
|
|
.expect("Exa answers");
|
|
assert_usable(&hits, "exa");
|
|
// `num_results` was accepted and ignored, so this came back as Exa's default page size.
|
|
assert!(hits.len() <= 3, "numResults is the field Exa reads: {} hits", hits.len());
|
|
assert!(
|
|
hits.iter().any(|hit| hit.snippet.chars().count() > 200),
|
|
"contents.text is what makes a hit answerable without a fetch: {hits:#?}"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn firecrawl_searches_and_renders_a_page_a_plain_get_cannot_read() {
|
|
let Some(api_key) = key("FIRECRAWL_API_KEY") else { return };
|
|
let backend = FirecrawlBackend::new(api_key);
|
|
|
|
let hits = backend
|
|
.search(
|
|
"tokio cancellation safety",
|
|
3,
|
|
&SearchOptions { include_content: true, ..Default::default() },
|
|
)
|
|
.await
|
|
.expect("Firecrawl answers");
|
|
assert_usable(&hits, "firecrawl");
|
|
|
|
let filtered = backend
|
|
.search(
|
|
"cancellation",
|
|
5,
|
|
&SearchOptions {
|
|
include_domains: vec!["docs.rs".to_string()],
|
|
recency: Some(Recency::Year),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
.expect("Firecrawl accepts the filters");
|
|
assert!(
|
|
filtered.iter().all(|hit| hit.url.contains("docs.rs")),
|
|
"includeDomains is honored server-side: {filtered:#?}"
|
|
);
|
|
|
|
// The reason this backend earns its place in `web_fetch`: a rendered page, as markdown.
|
|
let page = backend
|
|
.scrape("https://openrouter.ai/docs/features/prompt-caching")
|
|
.await
|
|
.expect("Firecrawl renders a JavaScript-built docs page");
|
|
assert!(page.len() > 2_000, "a rendered page is a page, not a shell: {} chars", page.len());
|
|
assert!(page.contains("cach"), "and it is the page that was asked for");
|
|
eprintln!("firecrawl scrape: {} chars", page.len());
|
|
|
|
// Firecrawl answers 200 with a stub for an upstream 404; reporting that as content would hand
|
|
// the model a "Not Found" page as though it were the article. (A *soft* 404 — a 200 whose body
|
|
// says the page is missing, which is what docs.rs serves — is indistinguishable from content
|
|
// and is not what this checks.)
|
|
let missing = backend.scrape("https://www.rust-lang.org/no-such-page-luxe-xyz").await;
|
|
assert!(missing.is_err(), "an upstream 404 is a failure, not a page: {missing:?}");
|
|
}
|