Files
luxe/crates/luxe-providers/tests/incremental_streaming.rs
Evan Whittaker 09a7de2520 Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
2026-10-17 12:08:30 -05:00

224 lines
8.9 KiB
Rust

//! Provider adapters must stream, not buffer.
//!
//! Every adapter used to call `response.text().await` before parsing, so reasoning and answer text
//! reached the UI only after the provider had finished the whole call. These tests keep an SSE body
//! deliberately open and assert the first events arrive while it is still open — the one property
//! whole-body parsing cannot satisfy.
use futures::StreamExt;
use luxe_core::message::Message;
use luxe_core::provider::{
CompletionRequest, ModelInfo, ModelRef, Provider, ProviderEvent, ThinkingLevel,
};
use luxe_providers::anthropic::{AnthropicCompat, AnthropicProvider};
use luxe_providers::config::{CompatFlags, ProviderConfig};
use luxe_providers::openai_compat::OpenaiCompatProvider;
use std::io::Write;
use std::time::Duration;
/// How long the server holds the body open after the first records.
const HOLD: Duration = Duration::from_millis(1_500);
/// Budget for opening the call *and* seeing its first event. Comfortably under `HOLD`, so passing
/// means "did not wait for the body to finish" rather than "was quick".
///
/// The budget has to cover `Provider::stream` too: whole-body parsing blocks *there*, before the
/// stream is even returned, so timing only the first `next()` would pass either way.
const FIRST_EVENT_BUDGET: Duration = Duration::from_millis(600);
/// Serves one HTTP response whose SSE body arrives in two halves, `HOLD` apart.
///
/// No `Content-Length`: HTTP/1.1 then frames the body by connection close, which is exactly how
/// the real providers stream, and lets us write the halves whenever we like.
fn serve_split_sse(first: String, second: String) -> u16 {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind");
let port = listener.local_addr().expect("addr").port();
std::thread::spawn(move || {
let (mut socket, _) = listener.accept().expect("accept");
read_request_head(&mut socket);
let head =
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n";
socket.write_all(head.as_bytes()).expect("head");
socket.write_all(first.as_bytes()).expect("first half");
socket.flush().expect("flush");
std::thread::sleep(HOLD);
socket.write_all(second.as_bytes()).expect("second half");
let _ = socket.flush();
});
port
}
/// Reads request headers (and any body) off `socket` far enough to reply.
fn read_request_head(socket: &mut std::net::TcpStream) {
use std::io::Read;
let mut buffer = Vec::new();
let mut chunk = [0u8; 2048];
loop {
let read = socket.read(&mut chunk).unwrap_or(0);
if read == 0 {
return;
}
buffer.extend_from_slice(&chunk[..read]);
let text = String::from_utf8_lossy(&buffer);
if let Some(head_end) = text.find("\r\n\r\n") {
let length = text
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length:")
.map(|value| value.trim().parse::<usize>().unwrap_or(0))
})
.unwrap_or(0);
if buffer.len() >= head_end + 4 + length {
return;
}
}
}
}
fn model(provider: &str) -> ModelInfo {
ModelInfo {
r#ref: ModelRef::new(provider, "test-model"),
max_tokens: 1024,
context_window: 8192,
..Default::default()
}
}
fn request(model: ModelInfo) -> CompletionRequest {
CompletionRequest {
model,
messages: vec![Message::user("hello")],
system: None,
tools: vec![],
thinking: ThinkingLevel::Off,
max_tokens: Some(256),
..Default::default()
}
}
/// Anthropic: a thinking delta in the first half must be observable before the second half exists.
#[tokio::test]
async fn anthropic_thinking_deltas_arrive_while_the_body_is_still_open() {
let first = "event: content_block_delta\n\
data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"pondering\"}}\n\n"
.to_string();
let second = "event: content_block_delta\n\
data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"answer\"}}\n\n\
event: message_stop\n\
data: {\"type\":\"message_stop\"}\n\n"
.to_string();
let port = serve_split_sse(first, second);
let model = model("anthropic");
let provider = AnthropicProvider::new(
"test-key".into(),
Some(format!("http://127.0.0.1:{port}/v1")),
model.clone(),
AnthropicCompat::default(),
);
let (_keep, cancel) = tokio::sync::watch::channel(false);
let (stream, event) = tokio::time::timeout(FIRST_EVENT_BUDGET, async {
let mut stream = provider.stream(request(model), cancel).await.expect("stream opens");
let event = stream.next().await;
(stream, event)
})
.await
.expect("the first reasoning delta must not wait for the whole response");
assert!(
matches!(event, Some(ProviderEvent::ThinkingDelta { ref delta, .. }) if delta == "pondering"),
"expected a streamed thinking delta, got {event:?}"
);
// The rest still arrives once the server sends it, terminator included.
let rest: Vec<ProviderEvent> = stream.collect().await;
assert!(
rest.iter().any(
|event| matches!(event, ProviderEvent::TextDelta { delta, .. } if delta == "answer")
),
"{rest:?}"
);
assert!(matches!(rest.last(), Some(ProviderEvent::Done { .. })), "{rest:?}");
}
/// OpenAI-compatible: same property, plus the reasoning field these servers actually use.
#[tokio::test]
async fn openai_compat_deltas_arrive_while_the_body_is_still_open() {
let first = concat!(
r#"data: {"choices":[{"index":0,"delta":{"reasoning_content":"pondering"}}]}"#,
"\n\n",
r#"data: {"choices":[{"index":0,"delta":{"content":"ans"}}]}"#,
"\n\n"
)
.to_string();
let second = concat!(
r#"data: {"choices":[{"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":2,"total_tokens":11}}"#,
"\n\n",
"data: [DONE]\n\n"
)
.to_string();
let port = serve_split_sse(first, second);
let provider = OpenaiCompatProvider::new(
ProviderConfig {
api_key: Some("test-key".into()),
base_url: Some(format!("http://127.0.0.1:{port}/v1")),
..Default::default()
},
CompatFlags::default(),
);
let (_keep, cancel) = tokio::sync::watch::channel(false);
let (stream, event) = tokio::time::timeout(FIRST_EVENT_BUDGET, async {
let mut stream =
provider.stream(request(model("openai_compat")), cancel).await.expect("stream opens");
let event = stream.next().await;
(stream, event)
})
.await
.expect("the first reasoning delta must not wait for the whole response");
assert!(
matches!(event, Some(ProviderEvent::ThinkingDelta { ref delta, .. }) if delta == "pondering"),
"expected a streamed reasoning delta, got {event:?}"
);
let rest: Vec<ProviderEvent> = stream.collect().await;
let usage_before_done = rest
.iter()
.position(|event| matches!(event, ProviderEvent::Usage(_)))
.zip(rest.iter().position(|event| matches!(event, ProviderEvent::Done { .. })))
.is_some_and(|(usage, done)| usage < done);
assert!(usage_before_done, "usage must still land before Done: {rest:?}");
}
/// A cancelled turn drops the connection instead of waiting out a provider that has gone quiet.
#[tokio::test]
async fn cancelling_ends_the_stream_without_waiting_for_the_body() {
let first = "event: content_block_delta\n\
data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
.to_string();
let port = serve_split_sse(first, String::new());
let model = model("anthropic");
let provider = AnthropicProvider::new(
"test-key".into(),
Some(format!("http://127.0.0.1:{port}/v1")),
model.clone(),
AnthropicCompat::default(),
);
let (cancel_tx, cancel) = tokio::sync::watch::channel(false);
let (mut stream, first_event) = tokio::time::timeout(FIRST_EVENT_BUDGET, async {
let mut stream = provider.stream(request(model), cancel).await.expect("stream opens");
let event = stream.next().await;
(stream, event)
})
.await
.expect("the first delta must not wait for the whole response");
assert!(matches!(first_event, Some(ProviderEvent::TextDelta { .. })), "{first_event:?}");
cancel_tx.send(true).expect("cancel");
let ended =
tokio::time::timeout(FIRST_EVENT_BUDGET, async { while stream.next().await.is_some() {} })
.await;
assert!(ended.is_ok(), "a cancelled stream must end without waiting out the body");
}