Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
370
crates/luxe-providers/src/openai_compat/request.rs
Normal file
370
crates/luxe-providers/src/openai_compat/request.rs
Normal file
@@ -0,0 +1,370 @@
|
||||
//! Chat-completions request building — core messages to `POST /chat/completions` wire JSON.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use luxe_core::message::ContentBlock;
|
||||
use luxe_core::provider::CompletionRequest;
|
||||
|
||||
use super::thinking::apply_thinking_config;
|
||||
use super::OpenaiCompatProvider;
|
||||
use crate::config::CompatFlags;
|
||||
use crate::wire_json::data_url;
|
||||
|
||||
impl OpenaiCompatProvider {
|
||||
/// The body a request would go out with, for tests elsewhere in the crate.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn body_for_tests(&self, req: &CompletionRequest) -> serde_json::Value {
|
||||
self.build_request_body(req)
|
||||
}
|
||||
|
||||
/// Build the JSON request body for `POST /chat/completions`.
|
||||
pub(super) fn build_request_body(&self, req: &CompletionRequest) -> serde_json::Value {
|
||||
let compat = self.compat_for(req);
|
||||
// Empty unless this endpoint needs the name, which is the only reason to walk the transcript
|
||||
// for it — an absent entry then simply means no name to attach.
|
||||
let tool_names = match compat.requires_tool_result_name {
|
||||
true => tool_names_by_call(&req.messages),
|
||||
false => HashMap::new(),
|
||||
};
|
||||
let mut messages: Vec<serde_json::Value> = Vec::with_capacity(req.messages.len());
|
||||
// A `tool` message has nowhere to put an image — this schema allows content parts only on
|
||||
// a `user` message — and a `user` message may not be interleaved between two `tool`
|
||||
// messages. Luxe emits consecutive tool messages routinely (refusals, then results, plus a
|
||||
// synthesized repair), so the pixels are collected across the whole *run* and handed over
|
||||
// once, after every tool message in it.
|
||||
let accepts_images = req.model.input.contains(&luxe_core::provider::Modality::Image);
|
||||
let mut index = 0;
|
||||
while index < req.messages.len() {
|
||||
if req.messages[index].role != luxe_core::message::Role::Tool {
|
||||
push_wire_messages(
|
||||
&req.messages[index],
|
||||
&mut messages,
|
||||
&tool_names,
|
||||
compat.copilot_headers,
|
||||
);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
let mut images: Vec<(String, String)> = Vec::new();
|
||||
while index < req.messages.len()
|
||||
&& req.messages[index].role == luxe_core::message::Role::Tool
|
||||
{
|
||||
push_tool_messages(
|
||||
&req.messages[index],
|
||||
&mut messages,
|
||||
accepts_images.then_some(&mut images),
|
||||
&tool_names,
|
||||
);
|
||||
index += 1;
|
||||
}
|
||||
if let Some(carrier) = image_carrier_message(&images) {
|
||||
messages.push(carrier);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(system) = &req.system {
|
||||
// `developer` is the newer OpenAI spelling of the system turn. The flag parsed,
|
||||
// resolved and displayed while this line said `system` regardless, which is a setting
|
||||
// that reads back what you set and does nothing.
|
||||
//
|
||||
// Honored rather than dropped because the local backends it defaults on for all take
|
||||
// it: Ollama forwards the role verbatim, llama.cpp accepts it and folds it into
|
||||
// `system` for ordinary templates while keeping the two distinct for GPT-OSS/Harmony,
|
||||
// where the difference is real, and LM Studio serves it fine.
|
||||
let role = if compat.supports_developer_role { "developer" } else { "system" };
|
||||
messages.insert(0, serde_json::json!({"role": role, "content": system}));
|
||||
}
|
||||
let mut body = serde_json::json!({
|
||||
"model": req.model.r#ref.id,
|
||||
"messages": serde_json::Value::Null,
|
||||
"stream": true,
|
||||
});
|
||||
body["messages"] = serde_json::Value::Array(messages); // moved in; `json!` would clone it
|
||||
|
||||
// Ask for usage in the streamed response so token/cost/context accounting works. Most
|
||||
// OpenAI-compatible servers require this opt-in; without it the stream omits usage.
|
||||
if compat.supports_usage_in_streaming {
|
||||
body["stream_options"] = serde_json::json!({ "include_usage": true });
|
||||
}
|
||||
|
||||
if !req.tools.is_empty() {
|
||||
body["tools"] = serde_json::Value::Array(
|
||||
req.tools
|
||||
.iter()
|
||||
.map(|t| {
|
||||
serde_json::json!({
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": t.name,
|
||||
"description": t.description,
|
||||
"parameters": t.parameters
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(max_tokens) = req.max_tokens {
|
||||
let field = if compat.max_tokens_field == "max_completion_tokens" {
|
||||
"max_completion_tokens"
|
||||
} else {
|
||||
"max_tokens"
|
||||
};
|
||||
body[field] = max_tokens.into();
|
||||
}
|
||||
|
||||
apply_thinking_config(&mut body, req, compat);
|
||||
apply_cache_control(&mut body, compat, req.reuse);
|
||||
// Last, so the model's own recipe outranks every field above it.
|
||||
crate::sampling::apply(&mut body, &req.model.sampling_params);
|
||||
body
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask a gateway that supports it to serve the stable prefix of this turn from cache.
|
||||
///
|
||||
/// Only OpenRouter's spelling lives here, and it is one field: the gateway places the breakpoint on
|
||||
/// the last cacheable block itself and advances it as the conversation grows, which is exactly the
|
||||
/// shape an agentic session wants and the reason not to hand-place markers the way the Anthropic
|
||||
/// adapter must. Worth it by a wide margin — a Claude turn through OpenRouter costs ~10x less from
|
||||
/// the second turn on — and the models that don't cache this way ignore the field rather than
|
||||
/// refusing it, so it needs no per-model gate. Anthropic is the one that needs asking: OpenRouter
|
||||
/// does *not* cache Claude implicitly, so without this every turn re-reads the whole transcript at
|
||||
/// full price.
|
||||
fn apply_cache_control(
|
||||
body: &mut serde_json::Value,
|
||||
compat: &CompatFlags,
|
||||
reuse: luxe_core::provider::PromptReuse,
|
||||
) {
|
||||
// A one-off prompt is never read back, so a breakpoint on it only buys the write premium.
|
||||
if compat.cache_control_format.as_deref() == Some("openrouter") && reuse.worth_caching() {
|
||||
body["cache_control"] = serde_json::json!({"type": "ephemeral"});
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one core [`Message`](luxe_core::message::Message) into one or more OpenAI wire
|
||||
/// messages, appending them to `out`.
|
||||
///
|
||||
/// A `Tool` message fans out to one `{role:"tool", tool_call_id, content}` per result block
|
||||
/// (OpenAI requires the id to pair a result with its call). Assistant tool calls become the
|
||||
/// `tool_calls` array; when the assistant only called tools its `content` is `null`.
|
||||
///
|
||||
/// `copilot` replays the reasoning Copilot handed back with the message (`reasoning_text` and
|
||||
/// `reasoning_opaque`), which no other chat-completions server knows.
|
||||
pub(super) fn push_wire_messages(
|
||||
msg: &luxe_core::message::Message,
|
||||
out: &mut Vec<serde_json::Value>,
|
||||
tool_names: &HashMap<&str, &str>,
|
||||
copilot: bool,
|
||||
) {
|
||||
use luxe_core::message::Role;
|
||||
|
||||
if msg.role == Role::Tool {
|
||||
push_tool_messages(msg, out, None, tool_names);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut text_parts: Vec<String> = Vec::new();
|
||||
let mut tool_calls = Vec::new();
|
||||
let mut image_blocks: Vec<(String, String)> = Vec::new(); // (mime, base64_data)
|
||||
let mut reasoning: Option<(String, String)> = None; // (text, opaque)
|
||||
for block in &msg.content {
|
||||
match block {
|
||||
ContentBlock::Text(text) => {
|
||||
text_parts.push(text.clone());
|
||||
}
|
||||
// Copilot's own reasoning goes back in its own fields, the way it came, so the model
|
||||
// keeps the thread of a multi-step call (Gemini's thought signatures ride in
|
||||
// `reasoning_opaque`)
|
||||
ContentBlock::Thinking { text, opaque: Some(opaque), .. } if copilot => {
|
||||
let (texts, opaques) = reasoning.get_or_insert_with(Default::default);
|
||||
texts.push_str(text);
|
||||
opaques.push_str(opaque);
|
||||
}
|
||||
// Anything else is transient on the chat-completions path: replaying it as assistant
|
||||
// text wastes tokens, breaks prefix caching, and isn't the shape this API expects.
|
||||
// Reasoning-capable models regenerate their own reasoning server-side.
|
||||
ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } => {}
|
||||
ContentBlock::ToolCall { id, name, arguments } => {
|
||||
tool_calls.push(serde_json::json!({
|
||||
"id": id,
|
||||
"type": "function",
|
||||
"function": { "name": name, "arguments": arguments.to_string() }
|
||||
}));
|
||||
}
|
||||
ContentBlock::ToolResult { content, .. } => {
|
||||
// A tool result nested inside a non-tool message has no run to be carried
|
||||
// alongside, so its images stay a marker.
|
||||
let text = tool_result_text(content, false);
|
||||
if !text.is_empty() {
|
||||
text_parts.push(text);
|
||||
}
|
||||
}
|
||||
ContentBlock::Image { data, mime, .. } => {
|
||||
image_blocks.push((mime.clone(), data.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut m = serde_json::json!({ "role": msg.role });
|
||||
if !image_blocks.is_empty() {
|
||||
// Image content: use array form with mixed text and image_url objects.
|
||||
let mut arr: Vec<serde_json::Value> = text_parts
|
||||
.into_iter()
|
||||
.map(|t| serde_json::json!({ "type": "text", "text": t }))
|
||||
.collect();
|
||||
for (mime, data) in &image_blocks {
|
||||
arr.push(serde_json::json!({
|
||||
"type": "image_url",
|
||||
"image_url": { "url": data_url(mime, data) }
|
||||
}));
|
||||
}
|
||||
m["content"] = serde_json::Value::Array(arr);
|
||||
if !tool_calls.is_empty() {
|
||||
m["tool_calls"] = serde_json::Value::Array(tool_calls);
|
||||
}
|
||||
} else if tool_calls.is_empty() {
|
||||
// No tool calls: use string content.
|
||||
m["content"] = serde_json::Value::String(text_parts.join("\n"));
|
||||
} else {
|
||||
// Tool calls present: use string content if there's text, null otherwise.
|
||||
let text = text_parts.join("\n");
|
||||
m["content"] =
|
||||
if text.is_empty() { serde_json::Value::Null } else { serde_json::Value::String(text) };
|
||||
m["tool_calls"] = serde_json::Value::Array(tool_calls);
|
||||
}
|
||||
if let Some((text, opaque)) = reasoning {
|
||||
if !text.is_empty() {
|
||||
m["reasoning_text"] = serde_json::Value::String(text);
|
||||
}
|
||||
m["reasoning_opaque"] = serde_json::Value::String(opaque);
|
||||
}
|
||||
out.push(m);
|
||||
}
|
||||
|
||||
/// The headers this endpoint wants beyond `Content-Type`, gathered in one place so the call site
|
||||
/// stays one line however many providers grow one.
|
||||
pub(super) fn request_headers(
|
||||
compat: &CompatFlags,
|
||||
req: &CompletionRequest,
|
||||
) -> Vec<(&'static str, &'static str)> {
|
||||
let mut headers = copilot_request_headers(compat, req);
|
||||
if compat.openrouter_headers {
|
||||
headers.extend([("HTTP-Referer", LUXE_HOME), ("X-Title", "Luxe")]);
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
/// Where OpenRouter's attribution header points a reader who wants to know what called it.
|
||||
pub(super) const LUXE_HOME: &str = "https://luxe.dev";
|
||||
|
||||
/// GitHub Copilot's per-request headers, empty for every other provider.
|
||||
///
|
||||
/// Whichever wire API Copilot is spoken in, so the Responses adapter sends them too; the headers
|
||||
/// themselves are [`crate::copilot::turn_headers`].
|
||||
pub(crate) fn copilot_request_headers(
|
||||
compat: &CompatFlags,
|
||||
req: &CompletionRequest,
|
||||
) -> Vec<(&'static str, &'static str)> {
|
||||
match compat.copilot_headers {
|
||||
true => crate::copilot::turn_headers(req),
|
||||
false => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Which tool produced each result, read off the calls in the same request.
|
||||
///
|
||||
/// A result carries the id of the call it answers, never the tool's name — so for an endpoint that
|
||||
/// wants `name` on a tool message, the name has to come from the assistant turn that made the call.
|
||||
/// Safe to read from the outgoing transcript: a call and its result are one unit that is never
|
||||
/// separated (`tool_pairing::repair` guarantees it on this copy), so the call is present whenever
|
||||
/// its result is.
|
||||
fn tool_names_by_call(messages: &[luxe_core::message::Message]) -> HashMap<&str, &str> {
|
||||
messages
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
.filter_map(|block| match block {
|
||||
ContentBlock::ToolCall { id, name, .. } => Some((id.as_str(), name.as_str())),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Emit one `tool` message per result block, draining any images into `images` when given.
|
||||
///
|
||||
/// `images` is `Some` only when the model declares it takes them; a text-only endpoint is billed
|
||||
/// for every base64 byte it is sent and the strict ones answer 400, so a model that cannot see is
|
||||
/// never handed pixels.
|
||||
fn push_tool_messages(
|
||||
msg: &luxe_core::message::Message,
|
||||
out: &mut Vec<serde_json::Value>,
|
||||
mut images: Option<&mut Vec<(String, String)>>,
|
||||
tool_names: &HashMap<&str, &str>,
|
||||
) {
|
||||
let mut emitted = false;
|
||||
for block in &msg.content {
|
||||
if let ContentBlock::ToolResult { call_id, content, .. } = block {
|
||||
if let Some(collected) = images.as_deref_mut() {
|
||||
collected.extend(
|
||||
luxe_core::ToolOutput::images(content)
|
||||
.map(|(mime, data)| (mime.to_string(), data.to_string())),
|
||||
);
|
||||
}
|
||||
let mut message = serde_json::json!({
|
||||
"role": "tool",
|
||||
"tool_call_id": call_id,
|
||||
"content": tool_result_text(content, images.is_some()),
|
||||
});
|
||||
// Only where the endpoint asks for it, and only when the call that produced this result
|
||||
// is in the same request — a name guessed from anywhere else would be worse than none.
|
||||
if let Some(name) = tool_names.get(call_id.as_str()) {
|
||||
message["name"] = serde_json::Value::String((*name).to_string());
|
||||
}
|
||||
out.push(message);
|
||||
emitted = true;
|
||||
}
|
||||
}
|
||||
// Defensive: a Tool message with no ToolResult blocks still needs a turn.
|
||||
if !emitted {
|
||||
out.push(serde_json::json!({ "role": "tool", "content": "" }));
|
||||
}
|
||||
}
|
||||
|
||||
/// The one `user` message carrying every image a run of tool results produced.
|
||||
///
|
||||
/// It leads with a line of text on purpose: several OpenAI-compatible servers reject a content
|
||||
/// array with no text part, and a bare wall of base64 tells the model nothing about where the
|
||||
/// pixels came from.
|
||||
fn image_carrier_message(images: &[(String, String)]) -> Option<serde_json::Value> {
|
||||
if images.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut parts = vec![serde_json::json!({
|
||||
"type": "text",
|
||||
"text": "Attached image(s) from tool result:",
|
||||
})];
|
||||
parts.extend(images.iter().map(|(mime, data)| {
|
||||
let url = data_url(mime, data);
|
||||
serde_json::json!({ "type": "image_url", "image_url": { "url": url } })
|
||||
}));
|
||||
Some(serde_json::json!({ "role": "user", "content": parts }))
|
||||
}
|
||||
|
||||
/// Join a tool result's outputs into a single text blob for the OpenAI `content` field.
|
||||
///
|
||||
/// `carried` says the images are being sent alongside, which changes what the text owes the model:
|
||||
/// a pointer to something it is about to receive, rather than a marker standing in for something it
|
||||
/// never will. A result with text of its own needs neither.
|
||||
fn tool_result_text(content: &[luxe_core::ToolOutput], carried: bool) -> String {
|
||||
let text = luxe_core::ToolOutput::joined_text(content);
|
||||
if !text.is_empty() {
|
||||
return text;
|
||||
}
|
||||
let images = luxe_core::ToolOutput::images(content).next().is_some();
|
||||
match (images, carried) {
|
||||
(true, true) => "(see attached image)".to_string(),
|
||||
(true, false) => "[image]".to_string(),
|
||||
_ => text,
|
||||
}
|
||||
}
|
||||
309
crates/luxe-providers/src/openai_compat/sse.rs
Normal file
309
crates/luxe-providers/src/openai_compat/sse.rs
Normal file
@@ -0,0 +1,309 @@
|
||||
//! Chat-completions SSE decoding — `data:` records to [`ProviderEvent`]s.
|
||||
|
||||
use luxe_core::provider::{ProviderEvent, StopReason, Usage};
|
||||
use serde::Deserialize;
|
||||
use tracing::warn;
|
||||
|
||||
/// SSE event from the OpenAI API.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct SseEvent {
|
||||
#[serde(default)]
|
||||
choices: Option<Vec<SseChoice>>,
|
||||
#[serde(default)]
|
||||
usage: Option<SseUsage>,
|
||||
#[serde(default)]
|
||||
error: Option<SseError>,
|
||||
}
|
||||
|
||||
/// A choice in an SSE event.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SseChoice {
|
||||
#[serde(default)]
|
||||
index: Option<usize>,
|
||||
#[serde(default)]
|
||||
delta: Option<SseDelta>,
|
||||
#[serde(default)]
|
||||
finish_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Delta in a streaming response.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct SseDelta {
|
||||
#[serde(default)]
|
||||
role: Option<String>,
|
||||
#[serde(default)]
|
||||
content: Option<String>,
|
||||
/// Reasoning text, DeepSeek/Qwen/LM Studio spelling.
|
||||
#[serde(default)]
|
||||
reasoning_content: Option<String>,
|
||||
/// Reasoning text, OpenRouter/Together spelling.
|
||||
#[serde(default)]
|
||||
reasoning: Option<String>,
|
||||
/// Reasoning text, GitHub Copilot spelling.
|
||||
#[serde(default)]
|
||||
reasoning_text: Option<String>,
|
||||
/// GitHub Copilot's reasoning state, replayed with the message it came on.
|
||||
#[serde(default)]
|
||||
reasoning_opaque: Option<String>,
|
||||
#[serde(default)]
|
||||
tool_calls: Option<Vec<SseToolCall>>,
|
||||
}
|
||||
|
||||
impl SseDelta {
|
||||
/// The reasoning fragment in this delta, whichever field the server chose to put it in.
|
||||
///
|
||||
/// There is no standard: DeepSeek-lineage servers (and LM Studio proxying them) send
|
||||
/// `reasoning_content`, OpenRouter and Together send `reasoning`, GitHub Copilot sends
|
||||
/// `reasoning_text`. All three mean the same thing.
|
||||
fn reasoning_delta(&self) -> Option<&str> {
|
||||
self.reasoning_content
|
||||
.as_deref()
|
||||
.or(self.reasoning.as_deref())
|
||||
.or(self.reasoning_text.as_deref())
|
||||
.filter(|text| !text.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool call in a streaming response.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SseToolCall {
|
||||
#[serde(default)]
|
||||
index: Option<usize>,
|
||||
#[serde(default)]
|
||||
id: Option<String>,
|
||||
#[serde(default)]
|
||||
function: Option<SseFunction>,
|
||||
}
|
||||
|
||||
/// A function call in a streaming response.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct SseFunction {
|
||||
#[serde(default)]
|
||||
name: Option<String>,
|
||||
#[serde(default)]
|
||||
arguments: Option<String>,
|
||||
}
|
||||
|
||||
/// Usage statistics from the API.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct SseUsage {
|
||||
#[serde(default)]
|
||||
prompt_tokens: Option<u32>,
|
||||
#[serde(default)]
|
||||
completion_tokens: Option<u32>,
|
||||
#[serde(default)]
|
||||
total_tokens: Option<u32>,
|
||||
/// Cached-input breakdown; `prompt_tokens` already includes these.
|
||||
#[serde(default)]
|
||||
prompt_tokens_details: Option<TokenDetails>,
|
||||
/// Reasoning-output breakdown; `completion_tokens` already includes these.
|
||||
#[serde(default)]
|
||||
completion_tokens_details: Option<TokenDetails>,
|
||||
/// Reasoning tokens reported beside the counts rather than inside a breakdown, as GitHub
|
||||
/// Copilot does.
|
||||
#[serde(default)]
|
||||
reasoning_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// The `*_tokens_details` sub-object (OpenAI reports cached/reasoning subsets here).
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct TokenDetails {
|
||||
#[serde(default)]
|
||||
cached_tokens: Option<u32>,
|
||||
/// Tokens written to the cache, which OpenRouter reports and bills at a premium. Counted
|
||||
/// inside `prompt_tokens` like the read side, so it needs no adjustment to fit [`Usage`].
|
||||
#[serde(default)]
|
||||
cache_write_tokens: Option<u32>,
|
||||
#[serde(default)]
|
||||
reasoning_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Error from the API.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SseError {
|
||||
message: String,
|
||||
#[serde(default)]
|
||||
r#type: Option<String>,
|
||||
#[serde(default)]
|
||||
code: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Translate one decoded chat-completions chunk into `ProviderEvent`s.
|
||||
pub(super) fn push_sse_events(event: &SseEvent, events: &mut Vec<ProviderEvent>) {
|
||||
if let Some(error) = &event.error {
|
||||
let (named, code) = (error.r#type.as_deref(), error.code.as_ref());
|
||||
let kind = crate::http::kind_of_frame(named, code, &error.message);
|
||||
events.push(ProviderEvent::Error { message: error.message.clone(), kind });
|
||||
return;
|
||||
}
|
||||
|
||||
// Emit usage BEFORE the choices' `Done` (the engine stops the stream on `Done`, so a trailing
|
||||
// usage would be missed). Also covers usage-only final chunks with no choices.
|
||||
if let Some(usage) = &event.usage {
|
||||
events.push(usage_event(usage));
|
||||
}
|
||||
|
||||
for choice in event.choices.iter().flatten() {
|
||||
let index = choice.index.unwrap_or(0);
|
||||
if let Some(delta) = &choice.delta {
|
||||
push_delta_events(index, delta, events);
|
||||
}
|
||||
if let Some(finish_reason) = &choice.finish_reason {
|
||||
push_finish_events(index, finish_reason, events);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One usage chunk as the normalized [`Usage`] the engine counts.
|
||||
fn usage_event(usage: &SseUsage) -> ProviderEvent {
|
||||
let prompt_details = usage.prompt_tokens_details.as_ref();
|
||||
let cache_read = prompt_details.and_then(|d| d.cached_tokens).unwrap_or(0);
|
||||
let cache_write = prompt_details.and_then(|d| d.cache_write_tokens).unwrap_or(0);
|
||||
let reasoning = usage
|
||||
.completion_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|d| d.reasoning_tokens)
|
||||
.or(usage.reasoning_tokens)
|
||||
.unwrap_or(0);
|
||||
let prompt = usage.prompt_tokens.unwrap_or(0);
|
||||
let total = usage.total_tokens.unwrap_or(0);
|
||||
// Reasoning is output, and inside `completion_tokens` by convention. Copilot's Gemini
|
||||
// leaves it out and puts it in the total instead (19 prompt + 1 completion of a total of
|
||||
// 1,460, with 1,440 reasoning), which priced a turn's output at a fraction of what it was.
|
||||
// Only a total that holds the reasoning on top of both counts says so
|
||||
let mut completion = usage.completion_tokens.unwrap_or(0);
|
||||
if reasoning > 0
|
||||
&& u64::from(total) >= u64::from(prompt) + u64::from(completion) + u64::from(reasoning)
|
||||
{
|
||||
completion += reasoning;
|
||||
}
|
||||
ProviderEvent::Usage(Usage {
|
||||
prompt_tokens: prompt,
|
||||
completion_tokens: completion,
|
||||
total_tokens: total,
|
||||
cache_read_tokens: cache_read,
|
||||
cache_write_tokens: cache_write,
|
||||
reasoning_tokens: reasoning,
|
||||
cache_write_1h_tokens: 0,
|
||||
})
|
||||
}
|
||||
|
||||
fn push_delta_events(index: usize, delta: &SseDelta, events: &mut Vec<ProviderEvent>) {
|
||||
// Role delta (usually first event).
|
||||
if delta.role.is_some() {
|
||||
events.push(ProviderEvent::TextStart { index });
|
||||
}
|
||||
|
||||
// Reasoning delta, before the content delta of the same chunk: a server that packs
|
||||
// both into one chunk produced the reasoning first.
|
||||
if let Some(reasoning) = delta.reasoning_delta() {
|
||||
events.push(ProviderEvent::ThinkingDelta { index, delta: reasoning.to_string() });
|
||||
}
|
||||
if let Some(opaque) = delta.reasoning_opaque.as_ref().filter(|opaque| !opaque.is_empty()) {
|
||||
events.push(ProviderEvent::ReasoningOpaque { index, opaque: opaque.clone() });
|
||||
}
|
||||
|
||||
if let Some(content) = &delta.content {
|
||||
events.push(ProviderEvent::TextDelta { index, delta: content.clone() });
|
||||
}
|
||||
|
||||
for call in delta.tool_calls.iter().flatten() {
|
||||
push_tool_call_events(call, events);
|
||||
}
|
||||
}
|
||||
|
||||
fn push_tool_call_events(call: &SseToolCall, events: &mut Vec<ProviderEvent>) {
|
||||
let index = call.index.unwrap_or(0);
|
||||
if let Some(id) = &call.id {
|
||||
events.push(ProviderEvent::ToolCallStart { index, id: id.clone(), name: String::new() });
|
||||
}
|
||||
let Some(function) = &call.function else { return };
|
||||
// The name arrives on the same chunk as the id, so it patches the `ToolCallStart` just
|
||||
// pushed rather than emitting an event of its own.
|
||||
if let Some(name) = &function.name {
|
||||
if let Some(ProviderEvent::ToolCallStart { name: slot, .. }) = events.last_mut() {
|
||||
*slot = name.clone();
|
||||
}
|
||||
}
|
||||
if let Some(arguments) = &function.arguments {
|
||||
events.push(ProviderEvent::ToolCallDelta { index, json_delta: arguments.clone() });
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool-call finish ends the turn too, and says why. Emitting only `ToolCallEnd` left the
|
||||
/// decoder with nothing deferred, so a body that then ended without `[DONE]` terminated the turn
|
||||
/// with no `Done` at all — and every tool-call turn that *did* get one reported `EndTurn`, while
|
||||
/// the Anthropic and Responses adapters reported `ToolCall` for the identical model.
|
||||
fn push_finish_events(index: usize, finish_reason: &str, events: &mut Vec<ProviderEvent>) {
|
||||
let reason = match finish_reason {
|
||||
"tool_calls" => {
|
||||
events.push(ProviderEvent::ToolCallEnd { index });
|
||||
StopReason::ToolCall
|
||||
}
|
||||
other => {
|
||||
events.push(ProviderEvent::TextEnd { index });
|
||||
match other {
|
||||
"stop" => StopReason::EndTurn,
|
||||
"length" => StopReason::Length,
|
||||
"content_filter" => StopReason::ContentFilter,
|
||||
_ => StopReason::Other(finish_reason.to_string()),
|
||||
}
|
||||
}
|
||||
};
|
||||
events.push(ProviderEvent::Done { reason });
|
||||
}
|
||||
|
||||
/// Record-at-a-time decoder for the chat-completions SSE shape.
|
||||
///
|
||||
/// `Done` is held back until the `[DONE]` sentinel (or the end of the body) because
|
||||
/// OpenAI-compatible servers report `include_usage` in a chunk *after* the `finish_reason` chunk,
|
||||
/// and the turn loop stops on the first `Done` — emitting it eagerly loses the turn's usage.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct CompatDecoder {
|
||||
deferred_done: Option<StopReason>,
|
||||
done_emitted: bool,
|
||||
}
|
||||
|
||||
impl CompatDecoder {
|
||||
fn flush_done(&mut self, out: &mut Vec<ProviderEvent>) {
|
||||
if self.done_emitted {
|
||||
return;
|
||||
}
|
||||
self.done_emitted = true;
|
||||
out.push(ProviderEvent::Done {
|
||||
reason: self.deferred_done.take().unwrap_or(StopReason::EndTurn),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::sse::SseDecoder for CompatDecoder {
|
||||
fn record(&mut self, record: &str, out: &mut Vec<ProviderEvent>) {
|
||||
for line in record.lines() {
|
||||
let Some(data) = line.trim().strip_prefix("data:").map(str::trim) else { continue };
|
||||
if data == "[DONE]" {
|
||||
return self.flush_done(out);
|
||||
}
|
||||
match serde_json::from_str::<SseEvent>(data) {
|
||||
Ok(event) => {
|
||||
let mut decoded = Vec::new();
|
||||
push_sse_events(&event, &mut decoded);
|
||||
for event in decoded {
|
||||
match event {
|
||||
ProviderEvent::Done { reason } => self.deferred_done = Some(reason),
|
||||
other => out.push(other),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => warn!(error = %error, data = %data, "failed to parse SSE event"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self, out: &mut Vec<ProviderEvent>) {
|
||||
// A body that ended without `[DONE]` still finished the turn — unconditionally, because
|
||||
// the caller cannot tell "the stream ended" from "the stream stopped" and a turn that
|
||||
// never terminates leaves the engine with no stop reason to report. A transport failure
|
||||
// does not come through here: it is a terminal `Error` event instead.
|
||||
self.flush_done(out);
|
||||
}
|
||||
}
|
||||
18
crates/luxe-providers/src/openai_compat/tests.rs
Normal file
18
crates/luxe-providers/src/openai_compat/tests.rs
Normal file
@@ -0,0 +1,18 @@
|
||||
#![allow(clippy::unwrap_used, clippy::collapsible_match)]
|
||||
|
||||
use super::{
|
||||
request::{push_wire_messages, LUXE_HOME},
|
||||
sse::{push_sse_events, SseEvent},
|
||||
*,
|
||||
};
|
||||
use crate::sse::SseDecoder;
|
||||
use luxe_core::message::{ContentBlock, Message, Role, ToolOutput};
|
||||
use luxe_core::provider::{ModelInfo, ModelRef, StopReason, ThinkingLevel};
|
||||
use std::collections::HashMap;
|
||||
|
||||
mod request;
|
||||
mod stream;
|
||||
mod wire;
|
||||
|
||||
use request::*;
|
||||
use stream::*;
|
||||
367
crates/luxe-providers/src/openai_compat/tests/request.rs
Normal file
367
crates/luxe-providers/src/openai_compat/tests/request.rs
Normal file
@@ -0,0 +1,367 @@
|
||||
//! Request — thinking, caching, sampling and templates in the body.
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(super) fn thinking_request(model: ModelInfo, thinking: ThinkingLevel) -> CompletionRequest {
|
||||
CompletionRequest {
|
||||
model,
|
||||
messages: vec![Message::user("Hello")],
|
||||
system: None,
|
||||
tools: vec![],
|
||||
thinking,
|
||||
max_tokens: Some(4096),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn provider(compat: CompatFlags) -> OpenaiCompatProvider {
|
||||
OpenaiCompatProvider::new(ProviderConfig::default(), compat)
|
||||
}
|
||||
|
||||
/// Copilot's chat completions carries reasoning in fields of its own. Its text is shown, its
|
||||
/// opaque state is kept, and both go back with the message they came on — to Copilot only.
|
||||
/// The records are copied from live Gemini and Kimi streams.
|
||||
#[test]
|
||||
fn copilots_reasoning_is_read_kept_and_replayed() {
|
||||
let events = decode_records(&[
|
||||
r#"data: {"choices":[{"index":0,"delta":{"role":"assistant","reasoning_text":"Checking divisors"}}]}"#,
|
||||
r#"data: {"choices":[{"index":0,"delta":{"reasoning_opaque":"CpcBAb4+9vs"}}]}"#,
|
||||
r#"data: {"choices":[{"index":0,"delta":{"content":"No."},"finish_reason":"stop"}]}"#,
|
||||
]);
|
||||
assert!(events.iter().any(|event| matches!(
|
||||
event,
|
||||
ProviderEvent::ThinkingDelta { delta, .. } if delta == "Checking divisors"
|
||||
)));
|
||||
assert!(events.iter().any(|event| matches!(
|
||||
event,
|
||||
ProviderEvent::ReasoningOpaque { opaque, .. } if opaque == "CpcBAb4+9vs"
|
||||
)));
|
||||
|
||||
let replayed = Message {
|
||||
role: luxe_core::message::Role::Assistant,
|
||||
content: vec![
|
||||
ContentBlock::Thinking {
|
||||
text: "Checking divisors".into(),
|
||||
signature: None,
|
||||
id: None,
|
||||
encrypted: None,
|
||||
opaque: Some("CpcBAb4+9vs".into()),
|
||||
},
|
||||
ContentBlock::ToolCall {
|
||||
id: "call_1".into(),
|
||||
name: "read".into(),
|
||||
arguments: serde_json::json!({}),
|
||||
},
|
||||
],
|
||||
};
|
||||
let mut copilot = Vec::new();
|
||||
push_wire_messages(&replayed, &mut copilot, &HashMap::new(), true);
|
||||
assert_eq!(copilot[0]["reasoning_opaque"], "CpcBAb4+9vs");
|
||||
assert_eq!(copilot[0]["reasoning_text"], "Checking divisors");
|
||||
let mut elsewhere = Vec::new();
|
||||
push_wire_messages(&replayed, &mut elsewhere, &HashMap::new(), false);
|
||||
assert!(elsewhere[0].get("reasoning_opaque").is_none(), "{}", elsewhere[0]);
|
||||
assert!(elsewhere[0].get("reasoning_text").is_none());
|
||||
}
|
||||
|
||||
/// Reasoning is output. Copilot reports it beside the counts, and for Gemini leaves it out of
|
||||
/// `completion_tokens` while the total holds it; for Kimi it is inside. Both live usage blocks.
|
||||
#[test]
|
||||
fn copilots_reasoning_tokens_count_as_output_once() {
|
||||
let usage_of = |record: &str| {
|
||||
decode_records(&[record])
|
||||
.into_iter()
|
||||
.find_map(|event| match event {
|
||||
ProviderEvent::Usage(usage) => Some(usage),
|
||||
_ => None,
|
||||
})
|
||||
.expect("a usage event")
|
||||
};
|
||||
let gemini = usage_of(
|
||||
r#"data: {"choices":[],"usage":{"completion_tokens":1,"prompt_tokens":19,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":1460,"reasoning_tokens":1440}}"#,
|
||||
);
|
||||
assert_eq!((gemini.completion_tokens, gemini.reasoning_tokens), (1441, 1440));
|
||||
let kimi = usage_of(
|
||||
r#"data: {"choices":[],"usage":{"completion_tokens":1500,"completion_tokens_details":{},"prompt_tokens":103,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":1603,"reasoning_tokens":1500}}"#,
|
||||
);
|
||||
assert_eq!((kimi.completion_tokens, kimi.reasoning_tokens), (1500, 1500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_options_requests_usage_when_supported() {
|
||||
let req = thinking_request(ModelInfo::default(), ThinkingLevel::Off);
|
||||
// Default compat has usage-in-streaming on → the request opts into streamed usage.
|
||||
let on = provider(CompatFlags::default()).build_request_body(&req);
|
||||
assert_eq!(on["stream_options"]["include_usage"], true);
|
||||
// Explicit opt-out omits the field entirely (for strict servers).
|
||||
let off_flags = CompatFlags { supports_usage_in_streaming: false, ..Default::default() };
|
||||
let off = provider(off_flags).build_request_body(&req);
|
||||
assert!(off.get("stream_options").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_effort_uses_model_specific_values_and_ultra_alias() {
|
||||
let model = ModelInfo {
|
||||
r#ref: ModelRef::new("openai", "gpt-5.6-sol"),
|
||||
reasoning: true,
|
||||
thinking_level_map: HashMap::from([
|
||||
("minimal".to_string(), Some("low".to_string())),
|
||||
("xhigh".to_string(), Some("xhigh".to_string())),
|
||||
("max".to_string(), Some("max".to_string())),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
let compat = CompatFlags {
|
||||
supports_reasoning_effort: true,
|
||||
thinking_format: Some("openai".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let provider = provider(compat);
|
||||
|
||||
let minimal =
|
||||
provider.build_request_body(&thinking_request(model.clone(), ThinkingLevel::Minimal));
|
||||
let ultra = provider.build_request_body(&thinking_request(model.clone(), ThinkingLevel::Ultra));
|
||||
let off = provider.build_request_body(&thinking_request(model, ThinkingLevel::Off));
|
||||
|
||||
assert_eq!(minimal["reasoning_effort"], "low");
|
||||
assert_eq!(ultra["reasoning_effort"], "max");
|
||||
assert_eq!(off["reasoning_effort"], "none");
|
||||
}
|
||||
|
||||
/// A models.dev entry for OpenRouter, with the reasoning dials it declares.
|
||||
pub(super) fn openrouter_model(
|
||||
id: &str,
|
||||
controls: luxe_core::provider::ThinkingControls,
|
||||
) -> ModelInfo {
|
||||
ModelInfo {
|
||||
r#ref: ModelRef::new("openrouter", id),
|
||||
reasoning: true,
|
||||
thinking_controls: controls,
|
||||
thinking_level_map: HashMap::from([
|
||||
("low".to_string(), Some("low".to_string())),
|
||||
("medium".to_string(), Some("medium".to_string())),
|
||||
("high".to_string(), Some("high".to_string())),
|
||||
]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The reported failure: OpenRouter answers 400 "Reasoning is mandatory for this endpoint and
|
||||
/// cannot be disabled" to `effort: "none"` on roughly a fifth of its catalog — grok-4.x,
|
||||
/// gemini-3.x, the thinking Qwens, deepseek-r1 — and off is the level a session starts on, so
|
||||
/// every turn on those models failed. Asking for nothing is the only spelling they accept.
|
||||
#[test]
|
||||
fn openrouter_asks_a_mandatory_reasoning_endpoint_for_nothing_rather_than_for_none() {
|
||||
let provider = provider(CompatFlags {
|
||||
supports_reasoning_effort: true,
|
||||
thinking_format: Some("openrouter".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
let toggleable = luxe_core::provider::ThinkingControls {
|
||||
effort: true,
|
||||
can_disable: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mandatory = luxe_core::provider::ThinkingControls { effort: true, ..Default::default() };
|
||||
|
||||
let off_where_allowed = provider.build_request_body(&thinking_request(
|
||||
openrouter_model("anthropic/claude-sonnet-5", toggleable),
|
||||
ThinkingLevel::Off,
|
||||
));
|
||||
let off_where_mandatory = provider.build_request_body(&thinking_request(
|
||||
openrouter_model("x-ai/grok-4.6", mandatory),
|
||||
ThinkingLevel::Off,
|
||||
));
|
||||
let on_where_mandatory = provider.build_request_body(&thinking_request(
|
||||
openrouter_model("x-ai/grok-4.6", mandatory),
|
||||
ThinkingLevel::High,
|
||||
));
|
||||
|
||||
assert_eq!(off_where_allowed["reasoning"]["effort"], "none");
|
||||
assert!(
|
||||
off_where_mandatory.get("reasoning").is_none(),
|
||||
"a mandatory endpoint 400s on every way of asking it to stop: {off_where_mandatory}"
|
||||
);
|
||||
assert_eq!(on_where_mandatory["reasoning"]["effort"], "high");
|
||||
}
|
||||
|
||||
/// OpenRouter does not cache Claude implicitly, so without this an agentic session re-reads its
|
||||
/// whole transcript at full price every turn. One field, and only for the gateway that reads it.
|
||||
#[test]
|
||||
fn only_the_openrouter_format_asks_for_prompt_caching() {
|
||||
let req = thinking_request(ModelInfo::default(), ThinkingLevel::Off);
|
||||
let asked = provider(CompatFlags {
|
||||
cache_control_format: Some("openrouter".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.build_request_body(&req);
|
||||
let anthropics_own = provider(CompatFlags {
|
||||
cache_control_format: Some("anthropic".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.build_request_body(&req);
|
||||
let plain = provider(CompatFlags::default()).build_request_body(&req);
|
||||
|
||||
assert_eq!(asked["cache_control"]["type"], "ephemeral");
|
||||
assert!(
|
||||
anthropics_own.get("cache_control").is_none(),
|
||||
"per-block markers belong to the Messages adapter, not this one"
|
||||
);
|
||||
assert!(plain.get("cache_control").is_none());
|
||||
|
||||
// A prompt nothing will read back is not worth the write premium, whatever the gateway.
|
||||
let mut one_off = thinking_request(ModelInfo::default(), ThinkingLevel::Off);
|
||||
one_off.reuse = luxe_core::provider::PromptReuse::OneOff;
|
||||
let summarizing = provider(CompatFlags {
|
||||
cache_control_format: Some("openrouter".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.build_request_body(&one_off);
|
||||
assert!(summarizing.get("cache_control").is_none(), "{summarizing}");
|
||||
}
|
||||
|
||||
/// The headers a request carries beyond `Content-Type` are opt-in per provider, and one
|
||||
/// provider's must never ride along on another's.
|
||||
#[test]
|
||||
fn attribution_headers_go_only_where_they_were_asked_for() {
|
||||
let req = thinking_request(ModelInfo::default(), ThinkingLevel::Off);
|
||||
let named =
|
||||
request_headers(&CompatFlags { openrouter_headers: true, ..Default::default() }, &req);
|
||||
assert_eq!(named, vec![("HTTP-Referer", LUXE_HOME), ("X-Title", "Luxe")]);
|
||||
assert!(request_headers(&CompatFlags::default(), &req).is_empty());
|
||||
assert!(
|
||||
!request_headers(&CompatFlags { copilot_headers: true, ..Default::default() }, &req)
|
||||
.iter()
|
||||
.any(|(name, _)| *name == "X-Title"),
|
||||
"Copilot has no use for OpenRouter's attribution"
|
||||
);
|
||||
}
|
||||
|
||||
/// Cache writes are billed at a premium and counted inside `prompt_tokens`, so reporting them
|
||||
/// as zero understates a turn's cost. OpenRouter sends them; this used to drop them.
|
||||
#[test]
|
||||
fn a_cache_write_is_read_off_the_usage_chunk_rather_than_assumed_absent() {
|
||||
let events = decode_records(&[
|
||||
r#"data: {"choices":[],"usage":{"prompt_tokens":9731,"completion_tokens":49,"total_tokens":9780,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":9729},"completion_tokens_details":{"reasoning_tokens":7}}}"#,
|
||||
"data: [DONE]",
|
||||
]);
|
||||
let usage = events
|
||||
.iter()
|
||||
.find_map(|event| match event {
|
||||
ProviderEvent::Usage(usage) => Some(usage),
|
||||
_ => None,
|
||||
})
|
||||
.expect("usage decoded");
|
||||
assert_eq!(usage.cache_write_tokens, 9729);
|
||||
assert_eq!(usage.cache_read_tokens, 0);
|
||||
assert_eq!(usage.reasoning_tokens, 7);
|
||||
assert_eq!(usage.prompt_tokens, 9731, "writes are counted inside the prompt, not beside it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen_formats_use_boolean_thinking_after_level_clamping() {
|
||||
let model = ModelInfo {
|
||||
r#ref: ModelRef::new("dashscope", "qwen3.7-max-2026-06-08"),
|
||||
reasoning: true,
|
||||
thinking_level_map: HashMap::from([
|
||||
("off".to_string(), Some("off".to_string())),
|
||||
("minimal".to_string(), None),
|
||||
("low".to_string(), None),
|
||||
("medium".to_string(), None),
|
||||
("high".to_string(), Some("on".to_string())),
|
||||
("xhigh".to_string(), Some("on".to_string())),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
let qwen =
|
||||
provider(CompatFlags { thinking_format: Some("qwen".to_string()), ..Default::default() });
|
||||
let local = provider(CompatFlags {
|
||||
thinking_format: Some("qwen-chat-template".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let clamped = qwen.build_request_body(&thinking_request(model.clone(), ThinkingLevel::Low));
|
||||
let off = qwen.build_request_body(&thinking_request(model.clone(), ThinkingLevel::Off));
|
||||
let local_high = local.build_request_body(&thinking_request(model, ThinkingLevel::High));
|
||||
|
||||
assert_eq!(clamped["enable_thinking"], true);
|
||||
assert_eq!(off["enable_thinking"], false);
|
||||
assert_eq!(local_high["chat_template_kwargs"]["enable_thinking"], true);
|
||||
assert_eq!(local_high["chat_template_kwargs"]["preserve_thinking"], true);
|
||||
}
|
||||
|
||||
/// A template's thinking vocabulary is the template's, not Luxe's: Qwen3.8 takes a
|
||||
/// `reasoning_effort` argument and can be told *not* to preserve thinking, neither of which
|
||||
/// the fixed two-key object could express.
|
||||
#[test]
|
||||
fn a_configured_chat_template_describes_its_own_arguments() {
|
||||
let model = ModelInfo {
|
||||
r#ref: ModelRef::new("llama-server", "qwen3.8-27b"),
|
||||
reasoning: true,
|
||||
thinking_level_map: HashMap::from([
|
||||
("off".to_string(), Some("none".to_string())),
|
||||
("high".to_string(), Some("high".to_string())),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
let configured = provider(CompatFlags {
|
||||
// Pi's spelling of the format, so an entry can be carried across unchanged.
|
||||
thinking_format: Some("chat-template".to_string()),
|
||||
chat_template_kwargs: match serde_json::json!({
|
||||
"enable_thinking": { "$var": "thinking.enabled" },
|
||||
"preserve_thinking": false,
|
||||
"reasoning_effort": { "$var": "thinking.level", "omit_when_off": true },
|
||||
}) {
|
||||
serde_json::Value::Object(map) => map,
|
||||
_ => unreachable!("fixture is an object"),
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let high = configured.build_request_body(&thinking_request(model.clone(), ThinkingLevel::High));
|
||||
assert_eq!(high["chat_template_kwargs"]["enable_thinking"], true);
|
||||
assert_eq!(high["chat_template_kwargs"]["preserve_thinking"], false);
|
||||
assert_eq!(high["chat_template_kwargs"]["reasoning_effort"], "high");
|
||||
|
||||
let off = configured.build_request_body(&thinking_request(model, ThinkingLevel::Off));
|
||||
assert_eq!(off["chat_template_kwargs"]["enable_thinking"], false);
|
||||
assert!(
|
||||
off["chat_template_kwargs"].get("reasoning_effort").is_none(),
|
||||
"an effort argument is meaningless with thinking off"
|
||||
);
|
||||
}
|
||||
|
||||
/// The acceptance case from the spec: a model carrying Qwen's thinking-mode recipe puts all
|
||||
/// seven keys on the wire, and each one beats the field the adapter would otherwise have sent.
|
||||
#[test]
|
||||
fn a_models_sampling_recipe_reaches_the_body_and_outranks_the_adapter() {
|
||||
let recipe = serde_json::json!({
|
||||
"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0,
|
||||
"presence_penalty": 0.0, "frequency_penalty": 0.0, "repeat_penalty": 1.0,
|
||||
});
|
||||
let model = ModelInfo {
|
||||
r#ref: ModelRef::new("lmstudio", "qwen3.8-27b"),
|
||||
reasoning: true,
|
||||
sampling_params: match recipe.clone() {
|
||||
serde_json::Value::Object(map) => map,
|
||||
_ => unreachable!("fixture is an object"),
|
||||
},
|
||||
thinking_level_map: HashMap::from([("off".to_string(), Some("none".to_string()))]),
|
||||
..Default::default()
|
||||
};
|
||||
let compat = CompatFlags {
|
||||
supports_reasoning_effort: true,
|
||||
thinking_format: Some("openai".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let body = provider(compat).build_request_body(&thinking_request(model, ThinkingLevel::Off));
|
||||
|
||||
for (key, value) in recipe.as_object().expect("fixture is an object") {
|
||||
assert_eq!(&body[key.as_str()], value, "{key} must reach the body verbatim");
|
||||
}
|
||||
// Nothing else about the request moved.
|
||||
assert_eq!(body["model"], "qwen3.8-27b");
|
||||
assert_eq!(body["stream"], true);
|
||||
assert_eq!(body["reasoning_effort"], "none");
|
||||
assert_eq!(body["max_tokens"], 4096);
|
||||
}
|
||||
184
crates/luxe-providers/src/openai_compat/tests/stream.rs
Normal file
184
crates/luxe-providers/src/openai_compat/tests/stream.rs
Normal file
@@ -0,0 +1,184 @@
|
||||
//! Stream — copilot headers, termination and reasoning deltas.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Drives the decoder the way the live reader does: one record at a time, in arrival order.
|
||||
pub(super) fn decode_records(records: &[&str]) -> Vec<ProviderEvent> {
|
||||
let mut decoder = CompatDecoder::default();
|
||||
let mut events = Vec::new();
|
||||
for record in records {
|
||||
decoder.record(record, &mut events);
|
||||
}
|
||||
decoder.finish(&mut events);
|
||||
events
|
||||
}
|
||||
|
||||
pub(super) fn request_with(messages: Vec<Message>) -> CompletionRequest {
|
||||
CompletionRequest {
|
||||
model: luxe_core::provider::ModelInfo::default(),
|
||||
messages,
|
||||
system: None,
|
||||
tools: vec![],
|
||||
thinking: ThinkingLevel::Off,
|
||||
max_tokens: None,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Copilot's endpoint expects to be told whether a request is the person asking or the agent
|
||||
/// continuing. Getting it backwards is invisible locally and shows up as throttling.
|
||||
#[test]
|
||||
fn copilot_headers_report_who_initiated_the_turn() {
|
||||
let compat = CompatFlags { copilot_headers: true, ..CompatFlags::default() };
|
||||
|
||||
let asked = request_with(vec![Message::user("hi")]);
|
||||
let headers = copilot_request_headers(&compat, &asked);
|
||||
assert!(headers.contains(&("X-Initiator", "user")), "{headers:?}");
|
||||
assert!(headers.contains(&("Openai-Intent", "conversation-edits")));
|
||||
|
||||
// A tool result means the agent is continuing on its own.
|
||||
let continuing =
|
||||
request_with(vec![Message::user("hi"), Message::tool_result("call-1", "output", false)]);
|
||||
let headers = copilot_request_headers(&compat, &continuing);
|
||||
assert!(headers.contains(&("X-Initiator", "agent")), "{headers:?}");
|
||||
|
||||
// So does an assistant turn being extended.
|
||||
let assistant = request_with(vec![Message::assistant("thinking…")]);
|
||||
assert!(copilot_request_headers(&compat, &assistant).contains(&("X-Initiator", "agent")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copilot_flags_a_turn_that_carries_an_image() {
|
||||
let compat = CompatFlags { copilot_headers: true, ..CompatFlags::default() };
|
||||
let with_image = request_with(vec![Message {
|
||||
role: luxe_core::message::Role::User,
|
||||
content: vec![ContentBlock::Image {
|
||||
data: "…".into(),
|
||||
mime: "image/png".into(),
|
||||
name: None,
|
||||
}],
|
||||
}]);
|
||||
assert!(
|
||||
copilot_request_headers(&compat, &with_image).contains(&("Copilot-Vision-Request", "true"))
|
||||
);
|
||||
|
||||
let text_only = request_with(vec![Message::user("hi")]);
|
||||
assert!(!copilot_request_headers(&compat, &text_only)
|
||||
.iter()
|
||||
.any(|(name, _)| *name == "Copilot-Vision-Request"));
|
||||
}
|
||||
|
||||
/// Every other provider must see none of this.
|
||||
#[test]
|
||||
fn copilot_headers_are_absent_for_everyone_else() {
|
||||
let headers =
|
||||
copilot_request_headers(&CompatFlags::default(), &request_with(vec![Message::user("hi")]));
|
||||
assert!(headers.is_empty(), "{headers:?}");
|
||||
}
|
||||
|
||||
/// `include_usage` puts the token counts in a chunk *after* `finish_reason`, and the turn loop
|
||||
/// stops on the first `Done` — so `Done` has to wait for `[DONE]`.
|
||||
#[test]
|
||||
fn trailing_usage_still_arrives_before_done() {
|
||||
let events = decode_records(&[
|
||||
r#"data: {"choices":[{"index":0,"delta":{"content":"hi"}}]}"#,
|
||||
r#"data: {"choices":[{"index":0,"finish_reason":"stop"}]}"#,
|
||||
r#"data: {"choices":[],"usage":{"prompt_tokens":100,"completion_tokens":5,"total_tokens":105}}"#,
|
||||
"data: [DONE]",
|
||||
]);
|
||||
let done = events.iter().position(|e| matches!(e, ProviderEvent::Done { .. })).unwrap();
|
||||
let usage = events.iter().position(|e| matches!(e, ProviderEvent::Usage(_))).unwrap();
|
||||
assert!(usage < done, "usage must precede Done so the engine processes it: {events:?}");
|
||||
assert_eq!(
|
||||
events.iter().filter(|e| matches!(e, ProviderEvent::Done { .. })).count(),
|
||||
1,
|
||||
"exactly one Done: {events:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A body that ends without the `[DONE]` sentinel still has to terminate the turn.
|
||||
#[test]
|
||||
fn a_body_without_the_done_sentinel_still_terminates() {
|
||||
let events = decode_records(&[
|
||||
r#"data: {"choices":[{"index":0,"delta":{"content":"hi"}}]}"#,
|
||||
r#"data: {"choices":[{"index":0,"finish_reason":"length"}]}"#,
|
||||
]);
|
||||
assert!(
|
||||
matches!(events.last(), Some(ProviderEvent::Done { reason: StopReason::Length })),
|
||||
"{events:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The reported failure, in the one shape that produced it: a **tool-call** turn whose body
|
||||
/// ends without `[DONE]`.
|
||||
///
|
||||
/// A `tool_calls` finish used to emit `ToolCallEnd` and nothing else, so the decoder had no
|
||||
/// deferred `Done` to flush and the stream simply stopped. The engine reads its stop reason
|
||||
/// off that event, so the turn ended reporting *nothing* — and the truncation warning, the
|
||||
/// only thing that reads the reason, could not fire on the turn that needed it. Every stream
|
||||
/// terminates now, whatever the body does.
|
||||
#[test]
|
||||
fn a_tool_call_turn_terminates_however_its_body_ends() {
|
||||
let call = r#"data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"c1","type":"function","function":{"name":"read","arguments":"{}"}}]}}]}"#;
|
||||
let finish = r#"data: {"choices":[{"index":0,"finish_reason":"tool_calls"}]}"#;
|
||||
|
||||
for (label, records) in [
|
||||
("no sentinel", vec![call, finish]),
|
||||
("with sentinel", vec![call, finish, "data: [DONE]"]),
|
||||
// OpenRouter repeats the finish chunk to carry usage; one turn is still one `Done`.
|
||||
("repeated finish", vec![call, finish, finish, "data: [DONE]"]),
|
||||
] {
|
||||
let events = decode_records(&records);
|
||||
let dones: Vec<_> = events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
ProviderEvent::Done { reason } => Some(reason.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(dones, vec![StopReason::ToolCall], "{label}: {events:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The same model reached three ways must report the same reason. Anthropic Messages and the
|
||||
/// Responses API both answer `ToolCall`; this adapter answered `EndTurn`, so a front-end
|
||||
/// reading the stop reason saw the provider, not the turn.
|
||||
#[test]
|
||||
fn a_tool_call_finish_reports_a_tool_call_not_an_end_of_turn() {
|
||||
let events = decode_records(&[
|
||||
r#"data: {"choices":[{"index":0,"finish_reason":"tool_calls"}]}"#,
|
||||
"data: [DONE]",
|
||||
]);
|
||||
assert!(
|
||||
matches!(events.last(), Some(ProviderEvent::Done { reason: StopReason::ToolCall })),
|
||||
"{events:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty body is still a finished call — the engine has to be able to stop waiting.
|
||||
#[test]
|
||||
fn a_body_with_nothing_in_it_still_terminates() {
|
||||
let events = decode_records(&[]);
|
||||
assert!(
|
||||
matches!(events.as_slice(), [ProviderEvent::Done { reason: StopReason::EndTurn }]),
|
||||
"{events:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Reasoning has no standard field name on this API; both spellings map to thinking.
|
||||
#[test]
|
||||
fn reasoning_deltas_are_decoded_under_either_field_name() {
|
||||
let deepseek =
|
||||
decode_records(&[r#"data: {"choices":[{"delta":{"reasoning_content":"pondering"}}]}"#]);
|
||||
let openrouter =
|
||||
decode_records(&[r#"data: {"choices":[{"delta":{"reasoning":"pondering"}}]}"#]);
|
||||
for events in [deepseek, openrouter] {
|
||||
assert!(
|
||||
events.iter().any(|event| matches!(
|
||||
event,
|
||||
ProviderEvent::ThinkingDelta { delta, .. } if delta == "pondering"
|
||||
)),
|
||||
"{events:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
519
crates/luxe-providers/src/openai_compat/tests/wire.rs
Normal file
519
crates/luxe-providers/src/openai_compat/tests/wire.rs
Normal file
@@ -0,0 +1,519 @@
|
||||
//! Wire — message expansion, SSE events and images.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tool_message_expands_with_tool_call_id() {
|
||||
let msg = Message {
|
||||
role: Role::Tool,
|
||||
content: vec![
|
||||
ContentBlock::ToolResult {
|
||||
call_id: "call_1".into(),
|
||||
content: vec![ToolOutput::Text("result A".into())],
|
||||
is_error: false,
|
||||
},
|
||||
ContentBlock::ToolResult {
|
||||
call_id: "call_2".into(),
|
||||
content: vec![ToolOutput::Text("result B".into())],
|
||||
is_error: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
push_wire_messages(&msg, &mut out, &HashMap::new(), false);
|
||||
// One tool message per result block, each carrying its own tool_call_id.
|
||||
assert_eq!(out.len(), 2);
|
||||
assert_eq!(out[0]["role"], "tool");
|
||||
assert_eq!(out[0]["tool_call_id"], "call_1");
|
||||
assert_eq!(out[0]["content"], "result A");
|
||||
assert_eq!(out[1]["tool_call_id"], "call_2");
|
||||
assert_eq!(out[1]["content"], "result B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_blocks_are_not_replayed_as_assistant_text() {
|
||||
let msg = Message {
|
||||
role: Role::Assistant,
|
||||
content: vec![
|
||||
ContentBlock::Thinking {
|
||||
text: "secret reasoning".into(),
|
||||
signature: None,
|
||||
id: None,
|
||||
encrypted: None,
|
||||
opaque: None,
|
||||
},
|
||||
ContentBlock::RedactedThinking { data: "ENC==".into() },
|
||||
ContentBlock::Text("the answer".into()),
|
||||
],
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
push_wire_messages(&msg, &mut out, &HashMap::new(), false);
|
||||
assert_eq!(out.len(), 1);
|
||||
let content = out[0]["content"].as_str().unwrap_or_default();
|
||||
assert_eq!(content, "the answer", "only the answer text is replayed");
|
||||
assert!(!content.contains("secret reasoning"), "reasoning is never inlined as text");
|
||||
assert!(!content.contains("thinking"), "no <thinking> wrapper leaks into the wire message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_tool_call_only_sends_null_content() {
|
||||
let msg = Message {
|
||||
role: Role::Assistant,
|
||||
content: vec![ContentBlock::ToolCall {
|
||||
id: "call_1".into(),
|
||||
name: "echo".into(),
|
||||
arguments: serde_json::json!({"x": 1}),
|
||||
}],
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
push_wire_messages(&msg, &mut out, &HashMap::new(), false);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0]["role"], "assistant");
|
||||
assert!(out[0]["content"].is_null(), "tool-call-only assistant must send null content");
|
||||
assert_eq!(out[0]["tool_calls"][0]["id"], "call_1");
|
||||
assert_eq!(out[0]["tool_calls"][0]["function"]["name"], "echo");
|
||||
// Arguments are serialized as a JSON string, per the OpenAI schema.
|
||||
assert!(out[0]["tool_calls"][0]["function"]["arguments"].is_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_text_with_tool_call_keeps_text() {
|
||||
let msg = Message {
|
||||
role: Role::Assistant,
|
||||
content: vec![
|
||||
ContentBlock::Text("let me check".into()),
|
||||
ContentBlock::ToolCall {
|
||||
id: "c1".into(),
|
||||
name: "echo".into(),
|
||||
arguments: serde_json::json!({}),
|
||||
},
|
||||
],
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
push_wire_messages(&msg, &mut out, &HashMap::new(), false);
|
||||
assert_eq!(out[0]["content"], "let me check");
|
||||
assert!(out[0]["tool_calls"].is_array());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_event_text_delta() {
|
||||
let sse_json = r#"{"choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}],"usage":null}"#;
|
||||
let event: SseEvent = serde_json::from_str(sse_json).unwrap();
|
||||
|
||||
let mut events = Vec::new();
|
||||
push_sse_events(&event, &mut events);
|
||||
|
||||
assert_eq!(events.len(), 2); // TextStart + TextDelta
|
||||
assert!(matches!(&events[0], ProviderEvent::TextStart { index: 0 }));
|
||||
assert!(matches!(&events[1], ProviderEvent::TextDelta { index: 0, delta } if delta == "Hello"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_event_tool_call() {
|
||||
let sse_json = r#"{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"read","arguments":"{\"path\":\"test.rs\"}"}}]},"finish_reason":null}],"usage":null}"#;
|
||||
let event: SseEvent = serde_json::from_str(sse_json).unwrap();
|
||||
|
||||
let mut events = Vec::new();
|
||||
push_sse_events(&event, &mut events);
|
||||
|
||||
assert_eq!(events.len(), 2); // ToolCallStart + ToolCallDelta (no ToolCallEnd without finish_reason)
|
||||
assert!(matches!(&events[0], ProviderEvent::ToolCallStart { id, .. } if id == "call_1"));
|
||||
assert!(
|
||||
matches!(&events[1], ProviderEvent::ToolCallDelta { json_delta, .. } if json_delta.contains("test.rs"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_event_done() {
|
||||
let sse_json = r#"{"choices":[{"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}"#;
|
||||
let event: SseEvent = serde_json::from_str(sse_json).unwrap();
|
||||
|
||||
let mut events = Vec::new();
|
||||
push_sse_events(&event, &mut events);
|
||||
|
||||
assert_eq!(events.len(), 3); // Usage + TextEnd + Done (usage precedes Done)
|
||||
assert!(
|
||||
matches!(&events[0], ProviderEvent::Usage(u) if u.prompt_tokens == 10 && u.completion_tokens == 5 && u.total_tokens == 15)
|
||||
);
|
||||
assert!(matches!(&events[1], ProviderEvent::TextEnd { index: 0 }));
|
||||
assert!(matches!(&events[2], ProviderEvent::Done { reason: StopReason::EndTurn }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_usage_captures_cache_and_reasoning_details() {
|
||||
let sse_json = r#"{"choices":[],"usage":{"prompt_tokens":1000,"completion_tokens":200,"total_tokens":1200,"prompt_tokens_details":{"cached_tokens":768},"completion_tokens_details":{"reasoning_tokens":150}}}"#;
|
||||
let event: SseEvent = serde_json::from_str(sse_json).unwrap();
|
||||
let mut events = Vec::new();
|
||||
push_sse_events(&event, &mut events);
|
||||
let usage = events.iter().find_map(|e| match e {
|
||||
ProviderEvent::Usage(u) => Some(u),
|
||||
_ => None,
|
||||
});
|
||||
let usage = usage.expect("a Usage event");
|
||||
assert_eq!(usage.prompt_tokens, 1000); // cache-inclusive
|
||||
assert_eq!(usage.cache_read_tokens, 768);
|
||||
assert_eq!(usage.reasoning_tokens, 150);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_event_error() {
|
||||
let sse_json = r#"{"error":{"message":"Rate limit exceeded","type":"rate_limit_exceeded"}}"#;
|
||||
let event: SseEvent = serde_json::from_str(sse_json).unwrap();
|
||||
|
||||
let mut events = Vec::new();
|
||||
push_sse_events(&event, &mut events);
|
||||
|
||||
assert_eq!(events.len(), 1);
|
||||
assert!(
|
||||
matches!(&events[0], ProviderEvent::Error { message, .. } if message == "Rate limit exceeded")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_event_empty_choices() {
|
||||
let sse_json = r#"{"choices":[]}"#;
|
||||
let event: SseEvent = serde_json::from_str(sse_json).unwrap();
|
||||
|
||||
let mut events = Vec::new();
|
||||
push_sse_events(&event, &mut events);
|
||||
|
||||
assert!(events.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_multiline_sse_fixture() {
|
||||
let events = decode_records(&[
|
||||
r#"data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}"#,
|
||||
r#"data: {"choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}"#,
|
||||
r#"data: {"choices":[{"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}"#,
|
||||
"data: [DONE]",
|
||||
]);
|
||||
|
||||
assert_eq!(events.len(), 5, "{events:?}");
|
||||
assert!(matches!(&events[0], ProviderEvent::TextStart { index: 0 }));
|
||||
assert!(matches!(&events[1], ProviderEvent::TextDelta { index: 0, delta } if delta == "Hello"));
|
||||
assert!(matches!(&events[2], ProviderEvent::Usage(u) if u.total_tokens == 15));
|
||||
assert!(matches!(&events[3], ProviderEvent::TextEnd { index: 0 }));
|
||||
// One terminal `Done`, after the usage: the `[DONE]` sentinel releases the one the
|
||||
// finish_reason chunk produced instead of adding a second.
|
||||
assert!(matches!(&events[4], ProviderEvent::Done { reason: StopReason::EndTurn }));
|
||||
}
|
||||
|
||||
/// A vision model that declares it takes images.
|
||||
pub(super) fn seeing(id: &str) -> ModelInfo {
|
||||
ModelInfo {
|
||||
r#ref: ModelRef::new("lmstudio", id),
|
||||
input: vec![luxe_core::provider::Modality::Text, luxe_core::provider::Modality::Image],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// One tool result carrying pixels, as `read` on an image or `screenshot` produces.
|
||||
pub(super) fn tool_result_with_image(call_id: &str, text: &str) -> Message {
|
||||
Message {
|
||||
role: luxe_core::message::Role::Tool,
|
||||
content: vec![ContentBlock::ToolResult {
|
||||
call_id: call_id.into(),
|
||||
content: vec![
|
||||
luxe_core::ToolOutput::Text(text.into()),
|
||||
luxe_core::ToolOutput::Image {
|
||||
data: "QUJD".into(),
|
||||
mime: "image/png".into(),
|
||||
name: Some("quad.png".into()),
|
||||
},
|
||||
],
|
||||
is_error: false,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn request(model: ModelInfo, messages: Vec<Message>) -> CompletionRequest {
|
||||
CompletionRequest {
|
||||
model,
|
||||
messages,
|
||||
system: None,
|
||||
tools: vec![],
|
||||
thinking: ThinkingLevel::Off,
|
||||
max_tokens: Some(4096),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The whole point of the run grouping: this schema has no image content on a `tool` message,
|
||||
/// so the pixels ride on a following `user` message — and that message may never land between
|
||||
/// two `tool` messages, which every provider rejects.
|
||||
#[test]
|
||||
fn a_tool_produced_image_reaches_a_vision_model_after_its_tool_message() {
|
||||
let body = provider(CompatFlags::default()).build_request_body(&request(
|
||||
seeing("qwen3.8-27b"),
|
||||
vec![Message::user("look at it"), tool_result_with_image("call-1", "quad.png: PNG")],
|
||||
));
|
||||
let messages = body["messages"].as_array().expect("messages");
|
||||
|
||||
let roles: Vec<&str> = messages.iter().filter_map(|m| m["role"].as_str()).collect();
|
||||
assert_eq!(roles, ["user", "tool", "user"], "the carrier follows the tool message");
|
||||
|
||||
let carrier = messages.last().expect("carrier");
|
||||
let parts = carrier["content"].as_array().expect("the carrier is a content array");
|
||||
assert_eq!(
|
||||
parts[0]["type"], "text",
|
||||
"a text part leads: some servers reject an array without one"
|
||||
);
|
||||
assert_eq!(parts[1]["type"], "image_url");
|
||||
assert_eq!(parts[1]["image_url"]["url"], "data:image/png;base64,QUJD");
|
||||
// The tool message keeps its own text and never carries the bytes.
|
||||
assert_eq!(messages[1]["content"], "quad.png: PNG");
|
||||
}
|
||||
|
||||
/// Sending base64 to a text-only endpoint is a large silent bill and, on a strict server, a
|
||||
/// 400. The declared modality is the gate, and nothing else.
|
||||
#[test]
|
||||
fn a_text_only_model_is_never_sent_the_bytes() {
|
||||
let body = provider(CompatFlags::default()).build_request_body(&request(
|
||||
ModelInfo::default(),
|
||||
vec![Message::user("look at it"), tool_result_with_image("call-1", "quad.png: PNG")],
|
||||
));
|
||||
let rendered = body.to_string();
|
||||
|
||||
assert!(!rendered.contains("QUJD"), "no base64 anywhere in the body: {rendered}");
|
||||
assert!(!rendered.contains("image_url"), "and no carrier message: {rendered}");
|
||||
let roles: Vec<&str> =
|
||||
body["messages"].as_array().unwrap().iter().filter_map(|m| m["role"].as_str()).collect();
|
||||
assert_eq!(roles, ["user", "tool"]);
|
||||
}
|
||||
|
||||
/// Luxe commits refusals and results as separate `Role::Tool` messages, and `tool_pairing` can
|
||||
/// synthesize a third. One carrier for the whole run, after all of them — a carrier per message
|
||||
/// would interleave a `user` turn between two `tool` turns and 400 the request.
|
||||
#[test]
|
||||
fn a_run_of_tool_messages_gets_exactly_one_carrier_after_all_of_them() {
|
||||
let body = provider(CompatFlags::default()).build_request_body(&request(
|
||||
seeing("qwen3.8-27b"),
|
||||
vec![
|
||||
Message::user("look"),
|
||||
tool_result_with_image("call-1", "first"),
|
||||
tool_result_with_image("call-2", "second"),
|
||||
],
|
||||
));
|
||||
let messages = body["messages"].as_array().expect("messages");
|
||||
let roles: Vec<&str> = messages.iter().filter_map(|m| m["role"].as_str()).collect();
|
||||
assert_eq!(roles, ["user", "tool", "tool", "user"], "never tool, user, tool");
|
||||
|
||||
let parts = messages.last().unwrap()["content"].as_array().expect("array");
|
||||
let images = parts.iter().filter(|part| part["type"] == "image_url").count();
|
||||
assert_eq!(images, 2, "both results' pixels ride on the one carrier");
|
||||
}
|
||||
|
||||
/// A result whose only output is an image would otherwise send empty content. It points at
|
||||
/// what follows rather than shrugging — the model is about to receive it.
|
||||
#[test]
|
||||
fn an_image_only_result_points_at_what_is_attached() {
|
||||
let mut message = tool_result_with_image("call-1", "");
|
||||
if let ContentBlock::ToolResult { content, .. } = &mut message.content[0] {
|
||||
content.retain(|output| matches!(output, luxe_core::ToolOutput::Image { .. }));
|
||||
}
|
||||
let body = provider(CompatFlags::default())
|
||||
.build_request_body(&request(seeing("qwen3.8-27b"), vec![message.clone()]));
|
||||
assert_eq!(body["messages"][0]["content"], "(see attached image)");
|
||||
|
||||
// With no way to send it, the marker stays a marker rather than a promise.
|
||||
let blind = provider(CompatFlags::default())
|
||||
.build_request_body(&request(ModelInfo::default(), vec![message]));
|
||||
assert_eq!(blind["messages"][0]["content"], "[image]");
|
||||
}
|
||||
|
||||
/// One endpoint, two builds, two differently shaped requests — from one adapter.
|
||||
///
|
||||
/// This is the whole point of per-model compat, and it could not work before: adapters are built
|
||||
/// once per provider, so a `thinking_format` written on a model reached nothing. The real setup
|
||||
/// it is for is a single LM Studio port serving two Qwen builds whose chat templates disagree —
|
||||
/// one handles a `developer` role, the other raises on it — where the provider-level format has
|
||||
/// to be whatever the model you are actually using wants.
|
||||
#[test]
|
||||
fn two_models_on_one_adapter_are_shaped_by_their_own_compat() {
|
||||
// The endpoint's flags: OpenAI-style effort, and the newer role spelling.
|
||||
let endpoint = CompatFlags {
|
||||
supports_reasoning_effort: true,
|
||||
thinking_format: Some("openai".into()),
|
||||
supports_developer_role: true,
|
||||
..Default::default()
|
||||
};
|
||||
let adapter = provider(endpoint.clone());
|
||||
let reasoning_model = |id: &str, compat: Option<CompatFlags>| ModelInfo {
|
||||
r#ref: ModelRef::new("lmstudio", id),
|
||||
reasoning: true,
|
||||
compat,
|
||||
..Default::default()
|
||||
};
|
||||
let ask = |model: ModelInfo| {
|
||||
adapter.build_request_body(&CompletionRequest {
|
||||
model,
|
||||
messages: vec![Message::user("hi")],
|
||||
system: Some("be brief".into()),
|
||||
tools: vec![],
|
||||
thinking: ThinkingLevel::High,
|
||||
max_tokens: Some(64),
|
||||
..Default::default()
|
||||
})
|
||||
};
|
||||
|
||||
// The model with no opinion runs on the endpoint's flags, exactly as before.
|
||||
let plain = ask(reasoning_model("qwen3.8-27b", None));
|
||||
assert_eq!(plain["messages"][0]["role"], "developer");
|
||||
assert_eq!(plain["reasoning_effort"], "high");
|
||||
assert_eq!(plain["messages"][0]["content"], "be brief");
|
||||
|
||||
// The build with its own flags is shaped differently on the same adapter: its template takes
|
||||
// thinking as a template argument and raises on a `developer` role.
|
||||
let fable = reasoning_model(
|
||||
"qwen3.6-27b-fable-fusion",
|
||||
Some(CompatFlags {
|
||||
thinking_format: Some("chat_template".into()),
|
||||
supports_developer_role: false,
|
||||
..endpoint.clone()
|
||||
}),
|
||||
);
|
||||
let templated = ask(fable);
|
||||
assert_eq!(templated["messages"][0]["role"], "system");
|
||||
assert!(templated.get("reasoning_effort").is_none(), "{templated}");
|
||||
assert_eq!(templated["chat_template_kwargs"]["enable_thinking"], true);
|
||||
|
||||
// And the adapter is unchanged by either request — the next model resolves for itself.
|
||||
let again = ask(reasoning_model("qwen3.8-27b", None));
|
||||
assert_eq!(again["messages"][0]["role"], "developer");
|
||||
assert_eq!(again["reasoning_effort"], "high");
|
||||
}
|
||||
|
||||
/// The other two flags a model can carry through the same path, each with a wire consequence.
|
||||
#[test]
|
||||
fn a_models_compat_decides_the_usage_opt_in_and_the_max_tokens_field() {
|
||||
let strict = CompatFlags {
|
||||
supports_usage_in_streaming: false,
|
||||
max_tokens_field: "max_completion_tokens".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let adapter = provider(CompatFlags::default());
|
||||
let model = ModelInfo {
|
||||
r#ref: ModelRef::new("proxy", "strict"),
|
||||
compat: Some(strict),
|
||||
..<_>::default()
|
||||
};
|
||||
|
||||
let body = adapter.build_request_body(&thinking_request(model, ThinkingLevel::Off));
|
||||
assert!(body.get("stream_options").is_none(), "this model's server rejects the field");
|
||||
assert_eq!(body["max_completion_tokens"], 4096);
|
||||
assert!(body.get("max_tokens").is_none());
|
||||
|
||||
// The adapter's own flags still answer for every other model on the endpoint.
|
||||
let sibling =
|
||||
adapter.build_request_body(&thinking_request(ModelInfo::default(), ThinkingLevel::Off));
|
||||
assert_eq!(sibling["stream_options"]["include_usage"], true);
|
||||
assert_eq!(sibling["max_tokens"], 4096);
|
||||
}
|
||||
|
||||
/// A tool result carries the *name* of the tool where the endpoint requires it.
|
||||
///
|
||||
/// Not decoration: `openai/harmony` — the format gpt-oss speaks — renders a tool message from
|
||||
/// its author's name and bails with `Tools should have a name!` when there is none, which a
|
||||
/// server reports as a 400. OpenAI itself ignores the field, so it stays opt-in.
|
||||
///
|
||||
/// The name has to be *found*: a result carries the id of the call it answers and nothing else,
|
||||
/// so it is read off the assistant turn that made the call, in the same request. That the call
|
||||
/// is there is guaranteed rather than hoped for — a call and its result are one unit, and the
|
||||
/// outgoing copy is repaired before it is built.
|
||||
#[test]
|
||||
fn a_tool_result_names_its_tool_where_the_endpoint_needs_it() {
|
||||
let call = |id: &str, name: &str| ContentBlock::ToolCall {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
arguments: serde_json::json!({}),
|
||||
};
|
||||
let result = |id: &str, text: &str| ContentBlock::ToolResult {
|
||||
call_id: id.to_string(),
|
||||
content: vec![luxe_core::ToolOutput::Text(text.to_string())],
|
||||
is_error: false,
|
||||
};
|
||||
let conversation = vec![
|
||||
Message::user("read both files"),
|
||||
Message {
|
||||
role: luxe_core::message::Role::Assistant,
|
||||
content: vec![call("call_1", "read"), call("call_2", "grep")],
|
||||
},
|
||||
Message {
|
||||
role: luxe_core::message::Role::Tool,
|
||||
// The third answers a call this request does not carry — a repaired half.
|
||||
content: vec![
|
||||
result("call_1", "fn main"),
|
||||
result("call_2", "3 matches"),
|
||||
result("call_orphan", "?"),
|
||||
],
|
||||
},
|
||||
];
|
||||
let request = |compat: Option<CompatFlags>| CompletionRequest {
|
||||
model: ModelInfo { compat, ..Default::default() },
|
||||
messages: conversation.clone(),
|
||||
system: None,
|
||||
tools: vec![],
|
||||
thinking: ThinkingLevel::Off,
|
||||
max_tokens: Some(64),
|
||||
..Default::default()
|
||||
};
|
||||
let tool_messages = |body: &serde_json::Value| -> Vec<serde_json::Value> {
|
||||
body["messages"]
|
||||
.as_array()
|
||||
.expect("messages")
|
||||
.iter()
|
||||
.filter(|message| message["role"] == "tool")
|
||||
.cloned()
|
||||
.collect()
|
||||
};
|
||||
|
||||
// Off by default: OpenAI has no such field, and a strict server rejects what it does not
|
||||
// model, so nothing is sent that was not asked for.
|
||||
let plain = provider(CompatFlags::default()).build_request_body(&request(None));
|
||||
for message in tool_messages(&plain) {
|
||||
assert!(message.get("name").is_none(), "{message}");
|
||||
}
|
||||
|
||||
// On, from the endpoint's own flags.
|
||||
let flags = CompatFlags { requires_tool_result_name: true, ..Default::default() };
|
||||
let named = provider(flags.clone()).build_request_body(&request(None));
|
||||
let named = tool_messages(&named);
|
||||
assert_eq!(named[0]["name"], "read", "each result names the call it answers");
|
||||
assert_eq!(named[0]["tool_call_id"], "call_1");
|
||||
assert_eq!(named[1]["name"], "grep");
|
||||
assert_eq!(named[1]["tool_call_id"], "call_2");
|
||||
assert!(
|
||||
named[2].get("name").is_none(),
|
||||
"a result with no call in this request gets no name rather than a guessed one: {}",
|
||||
named[2]
|
||||
);
|
||||
|
||||
// And on for one model on an endpoint whose other models do not want it — the same path
|
||||
// every other flag takes.
|
||||
let per_model = provider(CompatFlags::default()).build_request_body(&request(Some(flags)));
|
||||
assert_eq!(tool_messages(&per_model)[0]["name"], "read");
|
||||
}
|
||||
|
||||
/// A compat flag that parses, resolves and displays but changes nothing is worse than no flag:
|
||||
/// it reads back whatever you set, so the setting looks honored while the wire says otherwise.
|
||||
#[test]
|
||||
fn the_developer_role_flag_reaches_the_wire() {
|
||||
let request = CompletionRequest {
|
||||
model: ModelInfo::default(),
|
||||
messages: vec![Message::user("hi")],
|
||||
system: Some("be brief".into()),
|
||||
tools: vec![],
|
||||
thinking: ThinkingLevel::Off,
|
||||
max_tokens: Some(64),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let default = provider(CompatFlags::default()).build_request_body(&request);
|
||||
assert_eq!(default["messages"][0]["role"], "system", "unset stays the classic spelling");
|
||||
|
||||
let flags = CompatFlags { supports_developer_role: true, ..Default::default() };
|
||||
let opted_in = provider(flags).build_request_body(&request);
|
||||
assert_eq!(opted_in["messages"][0]["role"], "developer");
|
||||
assert_eq!(opted_in["messages"][0]["content"], "be brief", "and carries the same prompt");
|
||||
}
|
||||
67
crates/luxe-providers/src/openai_compat/thinking.rs
Normal file
67
crates/luxe-providers/src/openai_compat/thinking.rs
Normal file
@@ -0,0 +1,67 @@
|
||||
//! Thinking — how each chat-completions dialect asks for reasoning.
|
||||
|
||||
use luxe_core::provider::{CompletionRequest, ThinkingLevel};
|
||||
|
||||
use crate::config::CompatFlags;
|
||||
|
||||
pub(super) fn apply_thinking_config(
|
||||
body: &mut serde_json::Value,
|
||||
req: &CompletionRequest,
|
||||
compat: &CompatFlags,
|
||||
) {
|
||||
if !req.model.reasoning {
|
||||
return;
|
||||
}
|
||||
|
||||
let level = req.model.effective_thinking_level(req.thinking);
|
||||
let enabled = level != ThinkingLevel::Off;
|
||||
let effort = req.model.thinking_level_value(level).unwrap_or_else(|| match level {
|
||||
ThinkingLevel::Off => "none",
|
||||
ThinkingLevel::Ultra => "max",
|
||||
_ => level.as_str(),
|
||||
});
|
||||
|
||||
match compat.thinking_format.as_deref() {
|
||||
Some("qwen") => body["enable_thinking"] = enabled.into(),
|
||||
Some("qwen-chat-template" | "chat-template" | "chat_template") => {
|
||||
if let Some(kwargs) =
|
||||
crate::chat_template::kwargs(&compat.chat_template_kwargs, enabled, effort)
|
||||
{
|
||||
body["chat_template_kwargs"] = kwargs;
|
||||
}
|
||||
}
|
||||
Some("deepseek") => {
|
||||
body["thinking"] = serde_json::json!({
|
||||
"type": if enabled { "enabled" } else { "disabled" },
|
||||
});
|
||||
if enabled && compat.supports_reasoning_effort {
|
||||
body["reasoning_effort"] = effort.into();
|
||||
}
|
||||
}
|
||||
Some("openrouter") => {
|
||||
// A fifth of OpenRouter's catalog answers "Reasoning is mandatory for this endpoint"
|
||||
// to `effort: "none"` *and* to `enabled: false`, and off is the level a session starts
|
||||
// on — so those models would 400 on every turn. Omitting the key is the only spelling
|
||||
// that always works, and the catalog is trusted to say off is reachable before we ask
|
||||
// for it.
|
||||
if enabled || req.model.thinking_controls.can_disable {
|
||||
body["reasoning"] = serde_json::json!({"effort": effort});
|
||||
}
|
||||
}
|
||||
Some("together") => {
|
||||
body["reasoning"] = serde_json::json!({"enabled": enabled});
|
||||
if enabled && compat.supports_reasoning_effort {
|
||||
body["reasoning_effort"] = effort.into();
|
||||
}
|
||||
}
|
||||
Some("openai") => body["reasoning_effort"] = effort.into(),
|
||||
// `none` only where the model says it can stop thinking: Copilot answers it with a 400
|
||||
// for every model that cannot, and leaving the key out gets the model's own default
|
||||
None if compat.supports_reasoning_effort
|
||||
&& (enabled || req.model.thinking_controls.can_disable) =>
|
||||
{
|
||||
body["reasoning_effort"] = effort.into();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user