Files
luxe/crates/luxe-cli/src/model_catalog/tests.rs
Evan Whittaker d2129a886f Luxe 0.1.3: your sessions on your phone, joinable sessions, and nested AGENTS.md
- luxe remote puts the sessions on your computer in reach of your own phone or tablet over
  Tailscale: write, steer, answer prompts, send photos, browse files, diffs and GitHub, with a
  paired device and your own Tailscale login.
- luxe join lets another program on the same machine join a running session over the RPC protocol.
- An AGENTS.md in a subfolder reaches the model the first time it works there.
- Long sessions stay fast: no stall before each model call, lighter memory, cheaper drawing.
- A session is open in one Luxe at a time, web_search shares results across its phrasings, and a
  grep glob starting with ! excludes.

The CHANGELOG has the full list.
2026-10-20 02:40:13 -05:00

267 lines
11 KiB
Rust

use super::*;
use luxe_providers::config::{ModelConfig, ProviderConfig};
fn config_with(models: &[(&str, ModelConfig)], providers: &[&str]) -> ModelsConfig {
ModelsConfig {
models: models.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(),
providers: providers.iter().map(|p| (p.to_string(), ProviderConfig::default())).collect(),
..Default::default()
}
}
/// Billing has to survive the whole resolution chain, or the ledger books a plan turn as spend.
#[test]
fn subscription_providers_resolve_as_plan_billed() {
use luxe_core::provider::Billing;
let config = ModelsConfig::default();
let codex = resolve_model(&config, "openai-codex", "gpt-5.3-codex").0;
assert_eq!(codex.billing, Billing::Subscription);
assert!(codex.context_window > 0, "codex borrows OpenAI's metadata: {codex:?}");
assert!(codex.cost.is_known(), "the rates are known — they're just not a charge");
let copilot = resolve_model(&config, "github-copilot", "claude-sonnet-5").0;
assert_eq!(copilot.billing, Billing::Subscription);
// An API key is real money.
assert_eq!(resolve_model(&config, "openai", "gpt-5.6").0.billing, Billing::PerToken);
assert_eq!(resolve_model(&config, "anthropic", "claude-sonnet-5").0.billing, Billing::PerToken);
// OpenRouter mints a key that spends credits per token.
assert_eq!(
resolve_model(&config, "openrouter", "anthropic/claude-sonnet-5").0.billing,
Billing::PerToken
);
}
/// A Copilot plan decides which models the account may use, and the login recorded that list;
/// the picker must respect it rather than offering the whole catalog.
#[test]
fn a_credential_that_enumerates_its_models_restricts_the_picker() {
let allowed = vec!["gpt-5.6-sol".to_string(), "claude-sonnet-5".to_string()];
assert!(offers_model("github-copilot", "gpt-5.6-sol", &allowed));
assert!(offers_model("github-copilot", "claude-sonnet-5", &allowed));
assert!(!offers_model("github-copilot", "gemini-3-flash-preview", &allowed));
}
/// Verified against the live backend: `gpt-5.3-codex` is an API model that the ChatGPT-account
/// Codex endpoint rejects outright, and `gpt-5.6-sol` is one it serves. A heuristic on the id
/// ("contains codex") got this exactly backwards, so the catalog names the served set.
#[test]
fn codex_offers_what_a_chatgpt_account_can_actually_run() {
assert!(offers_model("openai-codex", "gpt-5.6-sol", &[]));
assert!(offers_model("codex", "gpt-5.4", &[]), "aliases resolve too");
assert!(offers_model("openai-codex", "gpt-5.3-codex-spark", &[]));
assert!(
!offers_model("openai-codex", "gpt-5.3-codex", &[]),
"the API-only Codex model 400s on a ChatGPT account"
);
// The plain OpenAI provider serves its whole catalog.
assert!(offers_model("openai", "gpt-5.3-codex", &[]));
assert!(offers_model("openai", "gpt-5.6", &[]));
}
#[test]
fn a_provider_that_says_nothing_about_its_models_offers_all_of_them() {
assert!(offers_model("anthropic", "claude-sonnet-5", &[]));
assert!(offers_model("some-proxy", "whatever-1", &[]));
}
#[test]
fn a_model_released_after_this_build_still_resolves_from_the_snapshot() {
// The whole point of the embedded snapshot: no config entry, no cache, no network.
let (info, from) = resolve_model(&ModelsConfig::default(), "anthropic", "claude-opus-5");
assert_eq!(info.context_window, 1_000_000);
assert_eq!(info.max_tokens, 128_000);
assert!(info.reasoning);
assert!(info.cost.is_known());
assert!(matches!(from.context_window, catalog::Layer::Snapshot | catalog::Layer::Cache));
}
#[test]
fn config_still_wins_over_the_catalog() {
let config = config_with(
&[(
"anthropic/claude-opus-5",
ModelConfig {
name: Some("Opus".into()),
context_window: Some(200_000),
..Default::default()
},
)],
&[],
);
let (info, from) = resolve_model(&config, "anthropic", "claude-opus-5");
assert_eq!(info.context_window, 200_000);
assert_eq!(from.context_window, catalog::Layer::Config);
assert_eq!(info.display(), "Opus");
// Untouched fields still come from the catalog.
assert_eq!(info.max_tokens, 128_000);
}
#[test]
fn an_unknown_local_model_keeps_its_configured_shape() {
let config = config_with(
&[(
"lmstudio/some-local-build",
ModelConfig {
name: Some("local".into()),
context_window: Some(32_768),
..Default::default()
},
)],
&["lmstudio"],
);
let (info, _) = resolve_model(&config, "lmstudio", "some-local-build");
assert_eq!(info.context_window, 32_768);
assert_eq!(info.max_tokens, 0, "non-Anthropic omits max_tokens unless asked");
assert!(!info.cost.is_known());
}
/// What a local server serves is a fact about this machine, and models.dev cannot know it.
///
/// The first live run of this: LM Studio reported three models it had, and the picker offered
/// three entirely different ids that models.dev publishes under `lmstudio` — none of which
/// this machine had. The login's own choice was not among them either, so adopting it fell
/// through to "any model for that provider" and switched the session to something the server
/// would have refused.
#[test]
fn a_local_servers_own_list_replaces_the_catalogs_idea_of_it() {
use crate::engine_setup::test_env::{isolated_env, ENV_LOCK};
let _guard = ENV_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let dir = isolated_env();
invalidate_cached_catalog();
let luxe = dir.path().join("luxe");
std::fs::create_dir_all(&luxe).expect("the config dir");
std::fs::write(
luxe.join("models.json"),
r#"{
"providers": {
"lmstudio": {
"base_url": "http://127.0.0.1:1234/v1",
"models_dev": "lmstudio",
"served_models": ["qwen3.8-27b", "qwen3.6-27b-fable-fusion"]
}
},
"default_provider": "lmstudio"
}"#,
)
.expect("the config");
let settings = AgentSettings::resolve(std::path::PathBuf::from("."), Default::default());
let listed: Vec<String> =
build_menu(&settings).catalog.iter().map(ModelEntry::qualified_id).collect();
assert_eq!(
listed,
vec!["lmstudio/qwen3.6-27b-fable-fusion".to_string(), "lmstudio/qwen3.8-27b".to_string()],
"only what the endpoint said it serves"
);
drop(dir);
}
/// The menu is what the providers you configured can answer for — the reason a model comes
/// after a provider rather than before it.
///
/// A fresh machine used to open the picker on OpenAI's whole catalog: the fallback provider
/// name aliases to `openai` for metadata, and it was treated as routable whether or not
/// anything could authenticate it. Thirty-eight models, none of which would answer.
#[test]
fn an_unconfigured_machine_has_no_models_to_choose_between() {
use crate::engine_setup::test_env::{isolated_env, ENV_LOCK};
let _guard = ENV_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let dir = isolated_env();
invalidate_cached_catalog();
let empty = AgentSettings::resolve(std::path::PathBuf::from("."), Default::default());
let got = build_menu(&empty).catalog;
assert!(
got.is_empty(),
"nothing is configured, so there is nothing that could answer: {:?}",
got.iter().map(ModelEntry::qualified_id).take(8).collect::<Vec<_>>()
);
// A key in the environment is a configured provider, and its models are the ones that
// appear — this is the path that must not be broken by gating the rest.
std::env::set_var("ANTHROPIC_API_KEY", "sk-ant-not-a-real-key");
let with_key = AgentSettings::resolve(std::path::PathBuf::from("."), Default::default());
let catalog = build_menu(&with_key).catalog;
std::env::remove_var("ANTHROPIC_API_KEY");
assert!(!catalog.is_empty(), "an ambient key is a provider Luxe can route to");
assert!(
catalog.iter().all(|entry| entry.info.r#ref.provider == "anthropic"),
"and only that provider's models: {:?}",
catalog.iter().map(ModelEntry::qualified_id).collect::<Vec<_>>()
);
// So is a key handed to this launch, for whichever provider it was handed for.
let flagged = AgentSettings::resolve(
std::path::PathBuf::from("."),
crate::engine_setup::CliOverrides {
provider: Some("openai".into()),
api_key: Some("sk-not-a-real-key".into()),
..Default::default()
},
);
let catalog = build_menu(&flagged).catalog;
assert!(
catalog.iter().any(|entry| entry.info.r#ref.provider == "openai"),
"a launch that carries the credential can reach its provider"
);
// And so is naming a server that needs no credential: there is nothing else to say.
let local = AgentSettings::resolve(
std::path::PathBuf::from("."),
crate::engine_setup::CliOverrides {
provider: Some("lmstudio".into()),
..Default::default()
},
);
assert!(
build_menu(&local).catalog.iter().any(|entry| entry.info.r#ref.provider == "lmstudio"),
"`--provider lmstudio` has said everything it can say about reaching it"
);
drop(dir);
}
/// A choice names the model as `select_model` takes it, and offers the levels it honors as they
/// are: only `off` for a model that doesn't reason, and nothing it would quietly clamp.
///
/// The break this catches: offering every level for every model, so a phone sets `ultra` and the
/// engine silently runs `high`; or sending the display name where `select_model` wants the id.
#[test]
fn a_choice_offers_only_the_levels_its_model_honors() {
use luxe_core::provider::ThinkingLevel::{High, Low, Medium, Minimal, Off};
let mut info = ModelInfo {
r#ref: luxe_core::provider::ModelRef::new("anthropic", "claude-sonnet-5"),
display_name: Some("Claude Sonnet 5".into()),
reasoning: false,
..Default::default()
};
let plain = choice(&info);
assert_eq!((plain.provider.as_str(), plain.model.as_str()), ("anthropic", "claude-sonnet-5"));
assert_eq!(plain.name, "Claude Sonnet 5");
assert_eq!(plain.thinking_levels, [Off]);
info.reasoning = true;
assert_eq!(choice(&info).thinking_levels, [Off, Minimal, Low, Medium, High]);
}
/// A row only guessed for an endpoint that hasn't said what it serves isn't offered.
///
/// The break this catches: a phone offering an Azure resource deployments it doesn't have, which
/// fail on the first turn.
#[test]
fn a_guessed_row_isnt_offered() {
let entry = |id: &str, unverified: bool| ModelEntry {
info: ModelInfo {
r#ref: luxe_core::provider::ModelRef::new("azure", id),
..Default::default()
},
label: id.into(),
configured: false,
unverified,
};
let offered = offered(&[entry("served", false), entry("guessed", true)]);
assert_eq!(offered.iter().map(|choice| choice.model.as_str()).collect::<Vec<_>>(), ["served"]);
}