Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
420
crates/luxe-core/tests/engine/compaction/cancel.rs
Normal file
420
crates/luxe-core/tests/engine/compaction/cancel.rs
Normal file
@@ -0,0 +1,420 @@
|
||||
//! A cancel reaches the summarizer: no summary lands after it, and what the call billed stays billed.
|
||||
|
||||
use futures::StreamExt;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
use luxe_core::usage::UsageSource;
|
||||
|
||||
use super::*;
|
||||
use crate::fixtures::events_until;
|
||||
|
||||
/// Sets its flag when dropped, so a test can see the summarizer's stream was let go.
|
||||
struct DropFlag(Arc<AtomicBool>);
|
||||
|
||||
impl Drop for DropFlag {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// How the summarizer answers its nth call.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Summary {
|
||||
/// Bills, then says nothing ever again: still thinking when the reader gives up on it
|
||||
Stalls,
|
||||
/// Never even returns its stream
|
||||
NeverOpens,
|
||||
/// Bills, then finishes a real summary once the test releases it
|
||||
AfterRelease,
|
||||
}
|
||||
|
||||
/// Answers turns like [`CompactingProvider`], and each summarization as `script` says, the last
|
||||
/// entry repeating. `overflow_first` refuses the first turn call as too long for the window.
|
||||
struct ScriptedSummaryProvider {
|
||||
calls: Arc<Mutex<Vec<CompletionRequest>>>,
|
||||
summarizing: Arc<tokio::sync::Notify>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
dropped: Arc<AtomicBool>,
|
||||
script: Vec<Summary>,
|
||||
summaries: AtomicUsize,
|
||||
overflow_first: AtomicBool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Provider for ScriptedSummaryProvider {
|
||||
fn name(&self) -> &str {
|
||||
"scripted-summary-mock"
|
||||
}
|
||||
|
||||
async fn stream(
|
||||
&self,
|
||||
request: CompletionRequest,
|
||||
_cancel: tokio::sync::watch::Receiver<bool>,
|
||||
) -> Result<BoxStream<'static, ProviderEvent>, ProviderError> {
|
||||
let summarizing = CompactingProvider::is_summarization(&request);
|
||||
if !summarizing && self.overflow_first.swap(false, Ordering::SeqCst) {
|
||||
return Err(ProviderError::Other(
|
||||
"400: prompt is too long: maximum context length exceeded".to_string(),
|
||||
));
|
||||
}
|
||||
let occupancy = occupancy_of(&request);
|
||||
self.calls.lock().await.push(request);
|
||||
if !summarizing {
|
||||
return Ok(Box::pin(stream::iter(vec![
|
||||
ProviderEvent::TextDelta { index: 0, delta: "ok".to_string() },
|
||||
ProviderEvent::Usage(Usage {
|
||||
prompt_tokens: occupancy,
|
||||
completion_tokens: 1,
|
||||
total_tokens: occupancy,
|
||||
..Usage::default()
|
||||
}),
|
||||
ProviderEvent::Done { reason: StopReason::EndTurn },
|
||||
])));
|
||||
}
|
||||
let nth = self.summaries.fetch_add(1, Ordering::SeqCst);
|
||||
let step = self.script[nth.min(self.script.len() - 1)];
|
||||
self.summarizing.notify_one();
|
||||
let guard = DropFlag(self.dropped.clone());
|
||||
let billed = ProviderEvent::Usage(Usage {
|
||||
prompt_tokens: 1_234,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 1_234,
|
||||
..Usage::default()
|
||||
});
|
||||
// The mock ignores its cancel receiver, as a provider mid-request may: the engine has to
|
||||
// stop waiting on its own
|
||||
let rest: BoxStream<'static, ProviderEvent> = match step {
|
||||
Summary::NeverOpens => {
|
||||
let _held = guard;
|
||||
std::future::pending::<()>().await;
|
||||
unreachable!()
|
||||
}
|
||||
Summary::Stalls => Box::pin(stream::pending()),
|
||||
Summary::AfterRelease => {
|
||||
let release = self.release.clone();
|
||||
let summary = format!("## Goal\n{SUMMARY_MARKER}: carry on with what is left.");
|
||||
Box::pin(
|
||||
stream::once(async move {
|
||||
release.notified().await;
|
||||
ProviderEvent::TextDelta { index: 0, delta: summary }
|
||||
})
|
||||
.chain(stream::iter([ProviderEvent::Done { reason: StopReason::EndTurn }])),
|
||||
)
|
||||
}
|
||||
};
|
||||
let events = stream::iter([billed]).chain(rest).map(move |event| {
|
||||
let _held = &guard;
|
||||
event
|
||||
});
|
||||
Ok(Box::pin(events))
|
||||
}
|
||||
}
|
||||
|
||||
/// What a test holds of its provider while the engine owns it.
|
||||
struct Script {
|
||||
calls: Arc<Mutex<Vec<CompletionRequest>>>,
|
||||
summarizing: Arc<tokio::sync::Notify>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
dropped: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
fn scripted(script: &[Summary], overflow_first: bool) -> (Arc<ScriptedSummaryProvider>, Script) {
|
||||
let provider = Arc::new(ScriptedSummaryProvider {
|
||||
calls: Arc::new(Mutex::new(Vec::new())),
|
||||
summarizing: Arc::new(tokio::sync::Notify::new()),
|
||||
release: Arc::new(tokio::sync::Notify::new()),
|
||||
dropped: Arc::new(AtomicBool::new(false)),
|
||||
script: script.to_vec(),
|
||||
summaries: AtomicUsize::new(0),
|
||||
overflow_first: AtomicBool::new(overflow_first),
|
||||
});
|
||||
let held = Script {
|
||||
calls: provider.calls.clone(),
|
||||
summarizing: provider.summarizing.clone(),
|
||||
release: provider.release.clone(),
|
||||
dropped: provider.dropped.clone(),
|
||||
};
|
||||
(provider, held)
|
||||
}
|
||||
|
||||
fn stalled() -> (Arc<ScriptedSummaryProvider>, Script) {
|
||||
scripted(&[Summary::Stalls], false)
|
||||
}
|
||||
|
||||
async fn summarizer_asked(script: &Script) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), script.summarizing.notified())
|
||||
.await
|
||||
.expect("the summarizer was asked");
|
||||
}
|
||||
|
||||
fn compaction_billed(events: &[Event]) -> bool {
|
||||
events.iter().any(|event| {
|
||||
matches!(
|
||||
event,
|
||||
Event::TurnUsage { source: UsageSource::Compaction, prompt_tokens: 1_234, .. }
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn no_compaction_on_disk(root: &std::path::Path, session_id: &SessionId) -> bool {
|
||||
let entries = JsonlSessionStore::new(root.join("sessions")).load(session_id).unwrap();
|
||||
!entries.iter().any(|entry| matches!(entry.kind, EntryKind::Compaction { .. }))
|
||||
}
|
||||
|
||||
/// A session `turns` turns long on `provider`, with nothing compacted yet.
|
||||
async fn session_of(
|
||||
turns: usize,
|
||||
provider: Arc<ScriptedSummaryProvider>,
|
||||
tmp: &tempfile::TempDir,
|
||||
) -> (Sender, Receiver, SessionId) {
|
||||
let mut store = JsonlSessionStore::new(tmp.path().join("sessions"));
|
||||
let session_id = store.create("/tmp/test").unwrap();
|
||||
let (action_tx, mut event_rx) =
|
||||
Engine::spawn(config(store, session_id.clone(), provider, tmp.path()));
|
||||
for turn in 0..turns {
|
||||
run_turn(&action_tx, &mut event_rx, bulky(&format!("TURN-{turn}"))).await;
|
||||
}
|
||||
(action_tx, event_rx, session_id)
|
||||
}
|
||||
|
||||
type Sender = tokio::sync::mpsc::Sender<Action>;
|
||||
type Receiver = tokio::sync::mpsc::Receiver<Event>;
|
||||
|
||||
/// Sends `/compact`, waits for the summarizer to be asked, and cancels it; returns what the engine
|
||||
/// said up to the refresh sent after the cancel, which proves it is answering again.
|
||||
async fn compact_then_cancel(
|
||||
action_tx: &Sender,
|
||||
event_rx: &mut Receiver,
|
||||
script: &Script,
|
||||
) -> Vec<Event> {
|
||||
action_tx.send(Action::CompactNow { focus: None }).await.unwrap();
|
||||
summarizer_asked(script).await;
|
||||
action_tx.send(Action::Cancel).await.unwrap();
|
||||
action_tx.send(Action::RefreshWorkspace).await.unwrap();
|
||||
events_until(event_rx, "the engine to answer after the cancel", |event| {
|
||||
matches!(event, Event::WorkspaceRefreshRequested)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel during a threshold compaction ends the turn there: the summarizer's stream is dropped,
|
||||
/// no summary is committed, and no further provider call goes out.
|
||||
///
|
||||
/// The break this catches: compaction read its summary over a private cancel channel that was
|
||||
/// never raised, so a cancel landing mid-summary left the turn waiting on the summarizer — and a
|
||||
/// reply that arrived later still replaced the conversation the reader had just stopped.
|
||||
#[tokio::test]
|
||||
async fn a_cancel_during_threshold_compaction_ends_the_turn_and_commits_no_summary() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (provider, script) = stalled();
|
||||
let (action_tx, mut event_rx, session_id) = session_of(0, provider, &tmp).await;
|
||||
|
||||
let mut turn = 0;
|
||||
loop {
|
||||
assert!(turn < TURNS, "the threshold never fired");
|
||||
let prompt = bulky(&format!("TURN-{turn}"));
|
||||
let submit =
|
||||
Action::SubmitUserMessage { content: prompt, images: vec![], remediation: None };
|
||||
action_tx.send(submit).await.unwrap();
|
||||
turn += 1;
|
||||
tokio::select! {
|
||||
() = script.summarizing.notified() => break,
|
||||
_ = events_until(&mut event_rx, "a turn", |e| matches!(e, Event::TurnEnded { .. })) => {}
|
||||
}
|
||||
}
|
||||
|
||||
action_tx.send(Action::Cancel).await.unwrap();
|
||||
let seen = events_until(&mut event_rx, "the cancelled turn to end", |event| {
|
||||
matches!(event, Event::TurnEnded { .. })
|
||||
})
|
||||
.await;
|
||||
let Some(Event::TurnEnded { reason }) = seen.last() else { unreachable!() };
|
||||
assert_eq!(*reason, luxe_core::event::StopReason::Other("cancelled".into()));
|
||||
assert!(!seen.iter().any(|e| matches!(e, Event::CompactionDone { .. })), "{seen:?}");
|
||||
assert!(script.dropped.load(Ordering::SeqCst), "the summarizer's stream is let go");
|
||||
let calls = script.calls.lock().await;
|
||||
let last = calls.last().unwrap();
|
||||
assert!(CompactingProvider::is_summarization(last), "nothing was sent after the cancel");
|
||||
drop(calls);
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
assert!(no_compaction_on_disk(tmp.path(), &session_id), "no summary was committed");
|
||||
}
|
||||
|
||||
/// `/compact` is serviced like a turn, so a cancel reaches it, the engine answers what comes next,
|
||||
/// and what the summarizer billed stays billed.
|
||||
///
|
||||
/// The break this catches: the action loop awaited the summarizer inline, so a cancel sat in the
|
||||
/// inbox behind it and nothing after it was served until the summarizer gave up on its own.
|
||||
#[tokio::test]
|
||||
async fn a_cancel_during_compaction_on_request_stops_it_and_the_engine_answers_again() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (provider, script) = stalled();
|
||||
let (action_tx, mut event_rx, session_id) = session_of(6, provider, &tmp).await;
|
||||
|
||||
let seen = compact_then_cancel(&action_tx, &mut event_rx, &script).await;
|
||||
|
||||
assert!(!seen.iter().any(|e| matches!(e, Event::CompactionDone { .. })), "{seen:?}");
|
||||
assert!(
|
||||
!seen
|
||||
.iter()
|
||||
.any(|e| matches!(e, Event::Error(message) if message.contains("Nothing to compact"))),
|
||||
"a cancelled compaction is not reported as having nothing to do: {seen:?}"
|
||||
);
|
||||
assert!(script.dropped.load(Ordering::SeqCst), "the summarizer's stream is let go");
|
||||
assert!(compaction_billed(&seen), "what the summarizer billed stays billed: {seen:?}");
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
assert!(no_compaction_on_disk(tmp.path(), &session_id), "no summary was committed");
|
||||
}
|
||||
|
||||
/// A cancel reaches a summarizer that has not even returned its stream yet.
|
||||
#[tokio::test]
|
||||
async fn a_cancel_before_the_summarizer_answers_at_all_still_lands() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (provider, script) = scripted(&[Summary::NeverOpens], false);
|
||||
let (action_tx, mut event_rx, session_id) = session_of(6, provider, &tmp).await;
|
||||
|
||||
compact_then_cancel(&action_tx, &mut event_rx, &script).await;
|
||||
|
||||
assert!(script.dropped.load(Ordering::SeqCst), "the pending request is let go");
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
assert!(no_compaction_on_disk(tmp.path(), &session_id), "no summary was committed");
|
||||
}
|
||||
|
||||
/// Cancelled compactions are not failures: as many as the breaker allows, and the next time the
|
||||
/// window fills it still compacts.
|
||||
///
|
||||
/// The break this catches: a cancel unwound through the failure path, so a reader who stopped
|
||||
/// three summaries switched automatic compaction off for the rest of the session.
|
||||
#[tokio::test]
|
||||
async fn cancelled_compactions_do_not_trip_the_failure_breaker() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let breaker = luxe_core::compaction::MAX_COMPACTION_FAILURES as usize;
|
||||
let mut script = vec![Summary::Stalls; breaker];
|
||||
script.push(Summary::AfterRelease);
|
||||
let (provider, script) = scripted(&script, false);
|
||||
let (action_tx, mut event_rx, _) = session_of(6, provider, &tmp).await;
|
||||
for _ in 0..breaker {
|
||||
compact_then_cancel(&action_tx, &mut event_rx, &script).await;
|
||||
}
|
||||
|
||||
script.release.notify_one();
|
||||
for turn in 6..TURNS {
|
||||
run_turn(&action_tx, &mut event_rx, bulky(&format!("TURN-{turn}"))).await;
|
||||
}
|
||||
|
||||
let calls = script.calls.lock().await;
|
||||
let summaries = calls.iter().filter(|call| CompactingProvider::is_summarization(call)).count();
|
||||
assert!(summaries > breaker, "the threshold still compacts: {summaries} summaries");
|
||||
let last = calls.last().unwrap();
|
||||
assert!(texts(last).iter().any(|text| text.contains(SUMMARY_MARKER)), "and it took effect");
|
||||
drop(calls);
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
}
|
||||
|
||||
/// A cancel during the summary an overflowing prompt forces ends the turn as cancelled, not as an
|
||||
/// error.
|
||||
///
|
||||
/// The break this catches: the retry read an unchanged leaf as "nothing left to summarize" and
|
||||
/// failed the turn, which a loop counts toward its consecutive-error budget.
|
||||
#[tokio::test]
|
||||
async fn a_cancel_during_overflow_compaction_ends_the_turn_as_cancelled() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (provider, _) = scripted(&[Summary::Stalls], false);
|
||||
let (action_tx, event_rx, session_id) = session_of(8, provider, &tmp).await;
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
|
||||
let store = JsonlSessionStore::new(tmp.path().join("sessions"));
|
||||
let (provider, script) = scripted(&[Summary::Stalls], true);
|
||||
let (action_tx, mut event_rx2) = Engine::spawn(config(store, session_id, provider, tmp.path()));
|
||||
drop(event_rx);
|
||||
let submit =
|
||||
Action::SubmitUserMessage { content: "and now".into(), images: vec![], remediation: None };
|
||||
action_tx.send(submit).await.unwrap();
|
||||
summarizer_asked(&script).await;
|
||||
action_tx.send(Action::Cancel).await.unwrap();
|
||||
let seen = events_until(&mut event_rx2, "the turn to end", |event| {
|
||||
matches!(event, Event::TurnEnded { .. })
|
||||
})
|
||||
.await;
|
||||
|
||||
let Some(Event::TurnEnded { reason }) = seen.last() else { unreachable!() };
|
||||
assert_eq!(*reason, luxe_core::event::StopReason::Other("cancelled".into()), "{seen:?}");
|
||||
assert!(!seen.iter().any(|e| matches!(e, Event::Error(_))), "{seen:?}");
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
}
|
||||
|
||||
/// A steer sent while `/compact` runs gets a turn of its own once it finishes, as a steer with no
|
||||
/// turn running always does.
|
||||
///
|
||||
/// The break this catches: serviced like a turn, the compaction buffered the steer for a turn
|
||||
/// that never came, and it surfaced only after the reader's next message.
|
||||
#[tokio::test]
|
||||
async fn a_steer_sent_during_compaction_on_request_runs_as_a_turn_after_it() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (provider, script) = scripted(&[Summary::AfterRelease], false);
|
||||
let (action_tx, mut event_rx, _) = session_of(6, provider, &tmp).await;
|
||||
|
||||
action_tx.send(Action::CompactNow { focus: None }).await.unwrap();
|
||||
summarizer_asked(&script).await;
|
||||
action_tx.send(Action::SteerTurn { content: "STEERED-WHILE-COMPACTING".into() }).await.unwrap();
|
||||
buffered_mid_work(&action_tx, &mut event_rx).await;
|
||||
script.release.notify_one();
|
||||
let seen = events_until(&mut event_rx, "the steer's turn", |event| {
|
||||
matches!(event, Event::TurnEnded { .. })
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(seen.iter().any(|e| matches!(e, Event::CompactionDone { .. })), "{seen:?}");
|
||||
let calls = script.calls.lock().await;
|
||||
let last = calls.last().unwrap();
|
||||
assert!(texts(last).iter().any(|text| text == "STEERED-WHILE-COMPACTING"), "{:?}", texts(last));
|
||||
drop(calls);
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
}
|
||||
|
||||
/// Returns once the engine has answered a glance it serves mid-work, so whatever was sent before
|
||||
/// it reached the work in progress rather than waiting behind it.
|
||||
async fn buffered_mid_work(action_tx: &Sender, event_rx: &mut Receiver) {
|
||||
let request_id = luxe_core::project_files::ListingRequestId::new();
|
||||
let path = String::new();
|
||||
let listing = Action::ListDirectory { request_id, path, include_ignored: false };
|
||||
action_tx.send(listing).await.unwrap();
|
||||
events_until(event_rx, "the listing", |event| {
|
||||
matches!(event, Event::DirectoryListed { .. } | Event::DirectoryListFailed { .. })
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// A cancelled `/compact` drops the steers sent during it, the way a cancel with no turn running
|
||||
/// does: no turn runs for them.
|
||||
#[tokio::test]
|
||||
async fn a_cancelled_compaction_runs_no_turn_for_steers_sent_during_it() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (provider, script) = stalled();
|
||||
let (action_tx, mut event_rx, _) = session_of(6, provider, &tmp).await;
|
||||
let turns_before = script.calls.lock().await.len();
|
||||
|
||||
action_tx.send(Action::CompactNow { focus: None }).await.unwrap();
|
||||
summarizer_asked(&script).await;
|
||||
action_tx.send(Action::SteerTurn { content: "STEERED-THEN-CANCELLED".into() }).await.unwrap();
|
||||
buffered_mid_work(&action_tx, &mut event_rx).await;
|
||||
action_tx.send(Action::Cancel).await.unwrap();
|
||||
action_tx.send(Action::RefreshWorkspace).await.unwrap();
|
||||
let seen = events_until(&mut event_rx, "the engine to answer", |event| {
|
||||
matches!(event, Event::WorkspaceRefreshRequested)
|
||||
})
|
||||
.await;
|
||||
action_tx.send(Action::RefreshWorkspace).await.unwrap();
|
||||
let seen_after = events_until(&mut event_rx, "the engine to answer again", |event| {
|
||||
matches!(event, Event::WorkspaceRefreshRequested)
|
||||
})
|
||||
.await;
|
||||
|
||||
for event in seen.iter().chain(&seen_after) {
|
||||
assert!(!matches!(event, Event::TurnEnded { .. }), "a turn ran: {event:?}");
|
||||
}
|
||||
let calls = script.calls.lock().await;
|
||||
assert_eq!(calls.len(), turns_before + 1, "only the summary was asked for");
|
||||
drop(calls);
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
}
|
||||
178
crates/luxe-core/tests/engine/compaction/refusals.rs
Normal file
178
crates/luxe-core/tests/engine/compaction/refusals.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
//! Summaries that must not replace the conversation they were meant to summarize.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A summarizer that answers with nothing at all, which a local model on a bad day really does.
|
||||
struct EmptySummaryProvider {
|
||||
calls: Arc<Mutex<Vec<CompletionRequest>>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Provider for EmptySummaryProvider {
|
||||
fn name(&self) -> &str {
|
||||
"empty-summary-mock"
|
||||
}
|
||||
|
||||
async fn stream(
|
||||
&self,
|
||||
request: CompletionRequest,
|
||||
_cancel: tokio::sync::watch::Receiver<bool>,
|
||||
) -> Result<BoxStream<'static, ProviderEvent>, ProviderError> {
|
||||
let summarizing = CompactingProvider::is_summarization(&request);
|
||||
let occupancy = occupancy_of(&request);
|
||||
self.calls.lock().await.push(request);
|
||||
|
||||
let mut events = Vec::new();
|
||||
if summarizing {
|
||||
// Nothing. Not an error, not a refusal — an empty reply.
|
||||
events.push(ProviderEvent::TextDelta { index: 0, delta: String::new() });
|
||||
} else {
|
||||
events.push(ProviderEvent::TextDelta { index: 0, delta: "ok".to_string() });
|
||||
events.push(ProviderEvent::Usage(Usage {
|
||||
prompt_tokens: occupancy,
|
||||
completion_tokens: 1,
|
||||
total_tokens: occupancy,
|
||||
..Usage::default()
|
||||
}));
|
||||
}
|
||||
events.push(ProviderEvent::Done { reason: StopReason::EndTurn });
|
||||
Ok(Box::pin(stream::iter(events)))
|
||||
}
|
||||
}
|
||||
|
||||
/// An empty summary must not be committed over the transcript it was meant to summarize.
|
||||
///
|
||||
/// The break this catches, measured on a live loop with a 27B local summarizer: the model returned
|
||||
/// nothing usable, the formatter supplied the six required headings and the running file list
|
||||
/// anyway, and a 20,677-token transcript was replaced by that — logged as a successful compaction.
|
||||
/// The session lost everything it knew, silently, and the next turn started from an empty document
|
||||
/// of headings.
|
||||
#[tokio::test]
|
||||
async fn a_summary_that_says_nothing_does_not_replace_the_conversation() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mut store = JsonlSessionStore::new(tmp.path().join("sessions"));
|
||||
let session_id = store.create("/tmp/test").unwrap();
|
||||
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = Arc::new(EmptySummaryProvider { calls: calls.clone() });
|
||||
let (action_tx, mut event_rx) =
|
||||
Engine::spawn(config(store, session_id.clone(), provider, tmp.path()));
|
||||
for turn in 0..TURNS {
|
||||
run_turn(&action_tx, &mut event_rx, bulky(&format!("TURN-{turn}"))).await;
|
||||
}
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
|
||||
let calls = calls.lock().await;
|
||||
assert!(
|
||||
calls.iter().any(CompactingProvider::is_summarization),
|
||||
"the threshold still fired; this is about what happens next"
|
||||
);
|
||||
let last = calls.last().expect("a final turn request");
|
||||
assert!(!CompactingProvider::is_summarization(last), "the last call is the turn itself");
|
||||
let sent = texts(last);
|
||||
assert!(
|
||||
sent.iter().any(|text| text.starts_with("TURN-0")),
|
||||
"the conversation an unusable summary could not replace is still there: {sent:?}"
|
||||
);
|
||||
|
||||
// And nothing on disk claims a compaction happened.
|
||||
let store = JsonlSessionStore::new(tmp.path().join("sessions"));
|
||||
let entries = store.load(&session_id).unwrap();
|
||||
assert!(
|
||||
!entries.iter().any(|entry| matches!(entry.kind, EntryKind::Compaction { .. })),
|
||||
"a refused compaction leaves no summary behind"
|
||||
);
|
||||
}
|
||||
|
||||
/// How a summarizer's reply can stop short after writing a summary that looks complete.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum CutOff {
|
||||
Error,
|
||||
Length,
|
||||
EndOfStream,
|
||||
}
|
||||
|
||||
/// A summarizer that writes a substantive summary, then fails to finish it.
|
||||
struct CutOffSummaryProvider {
|
||||
calls: Arc<Mutex<Vec<CompletionRequest>>>,
|
||||
cut_off: CutOff,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Provider for CutOffSummaryProvider {
|
||||
fn name(&self) -> &str {
|
||||
"cut-off-summary-mock"
|
||||
}
|
||||
|
||||
async fn stream(
|
||||
&self,
|
||||
request: CompletionRequest,
|
||||
_cancel: tokio::sync::watch::Receiver<bool>,
|
||||
) -> Result<BoxStream<'static, ProviderEvent>, ProviderError> {
|
||||
let summarizing = CompactingProvider::is_summarization(&request);
|
||||
let occupancy = occupancy_of(&request);
|
||||
self.calls.lock().await.push(request);
|
||||
|
||||
if !summarizing {
|
||||
return Ok(Box::pin(stream::iter(vec![
|
||||
ProviderEvent::TextDelta { index: 0, delta: "ok".to_string() },
|
||||
ProviderEvent::Usage(Usage {
|
||||
prompt_tokens: occupancy,
|
||||
completion_tokens: 1,
|
||||
total_tokens: occupancy,
|
||||
..Usage::default()
|
||||
}),
|
||||
ProviderEvent::Done { reason: StopReason::EndTurn },
|
||||
])));
|
||||
}
|
||||
let mut events = vec![ProviderEvent::TextDelta {
|
||||
index: 0,
|
||||
delta: format!("## Goal\n{SUMMARY_MARKER}: half of what happened, and then"),
|
||||
}];
|
||||
match self.cut_off {
|
||||
CutOff::Error => events.push(ProviderEvent::Error {
|
||||
message: "connection reset".into(),
|
||||
kind: luxe_core::provider::ProviderErrorKind::Other("transport".into()),
|
||||
}),
|
||||
CutOff::Length => events.push(ProviderEvent::Done { reason: StopReason::Length }),
|
||||
CutOff::EndOfStream => {}
|
||||
}
|
||||
Ok(Box::pin(stream::iter(events)))
|
||||
}
|
||||
}
|
||||
|
||||
/// A summary the summarizer did not finish must not replace the conversation it summarizes.
|
||||
///
|
||||
/// The break this catches: the reply loop took an error, a length stop or a dropped stream as
|
||||
/// success, and the half-summary that had already arrived passed the content check, so the
|
||||
/// session's history was replaced by a fragment.
|
||||
#[tokio::test]
|
||||
async fn a_summary_cut_off_partway_does_not_replace_the_conversation() {
|
||||
for cut_off in [CutOff::Error, CutOff::Length, CutOff::EndOfStream] {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mut store = JsonlSessionStore::new(tmp.path().join("sessions"));
|
||||
let session_id = store.create("/tmp/test").unwrap();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = Arc::new(CutOffSummaryProvider { calls: calls.clone(), cut_off });
|
||||
let (action_tx, mut event_rx) =
|
||||
Engine::spawn(config(store, session_id.clone(), provider, tmp.path()));
|
||||
for turn in 0..TURNS {
|
||||
run_turn(&action_tx, &mut event_rx, bulky(&format!("TURN-{turn}"))).await;
|
||||
}
|
||||
action_tx.send(Action::Close).await.unwrap();
|
||||
|
||||
let calls = calls.lock().await;
|
||||
assert!(calls.iter().any(CompactingProvider::is_summarization), "{cut_off:?}: it ran");
|
||||
let sent = texts(calls.last().expect("a final turn request"));
|
||||
assert!(
|
||||
sent.iter().any(|text| text.starts_with("TURN-0")),
|
||||
"{cut_off:?}: the conversation is still sent whole"
|
||||
);
|
||||
let entries =
|
||||
JsonlSessionStore::new(tmp.path().join("sessions")).load(&session_id).unwrap();
|
||||
assert!(
|
||||
!entries.iter().any(|entry| matches!(entry.kind, EntryKind::Compaction { .. })),
|
||||
"{cut_off:?}: no summary was committed"
|
||||
);
|
||||
}
|
||||
}
|
||||
112
crates/luxe-core/tests/engine/compaction/summaries.rs
Normal file
112
crates/luxe-core/tests/engine/compaction/summaries.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
//! Summaries — compaction replaces the history it summarized in the next request.
|
||||
|
||||
use luxe_core::session::{JsonlSessionStore, SessionStore};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Plan, summarize, append — over a real parent chain, ending in the request the next turn sends.
|
||||
///
|
||||
/// The fixture used to append every entry with `parent: None`, a flat forest with no path through
|
||||
/// it, and assert only that a `Compaction` entry existed afterwards. That is why compaction could
|
||||
/// summarize, bill for the call, write the summary to disk and still send the entire transcript
|
||||
/// for months without a red test.
|
||||
#[tokio::test]
|
||||
async fn compaction_replaces_the_history_it_summarized() {
|
||||
use luxe_core::compaction::CompactionSettings;
|
||||
use luxe_core::session::EntryKind;
|
||||
|
||||
let tmp_dir = std::env::temp_dir().join(format!("luxe-test-compaction-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&tmp_dir).unwrap();
|
||||
let mut store = JsonlSessionStore::new(tmp_dir.clone());
|
||||
let session_id = store.create("/tmp/test").unwrap();
|
||||
|
||||
// A real chain. For a 128K window / 32K max the usable budget is 128K - min(32K, 16_384) =
|
||||
// 111_616 tokens and compaction fires at 90% of that (~100K); at ~1 token / 4 chars, 100
|
||||
// entries of ~5000 chars clears it.
|
||||
let long_text = "a".repeat(5000);
|
||||
let mut parent = luxe_core::session::latest_leaf(&store.load(&session_id).unwrap());
|
||||
for index in 0..100 {
|
||||
let kind = match index % 2 {
|
||||
0 => EntryKind::Message(luxe_core::message::Message::user(format!(
|
||||
"Message {index} {long_text}"
|
||||
))),
|
||||
_ => EntryKind::Message(luxe_core::message::Message::assistant(format!(
|
||||
"Response {index} {long_text}"
|
||||
))),
|
||||
};
|
||||
parent = Some(store.append(&session_id, parent, kind).unwrap());
|
||||
}
|
||||
let leaf = parent.expect("a leaf");
|
||||
|
||||
let settings = CompactionSettings {
|
||||
enabled: true,
|
||||
reserve_tokens: 16_384,
|
||||
keep_recent_tokens: 5_000,
|
||||
compact_at_fraction: 0.9,
|
||||
prune_protect_tokens: 40_000,
|
||||
prune_min_tokens: 20_000,
|
||||
prune_at_fraction: 0.8,
|
||||
summarizer: None,
|
||||
};
|
||||
let budget = luxe_core::compaction::ContextBudget {
|
||||
context_window: 128_000,
|
||||
max_tokens: 32_000,
|
||||
observed_tokens: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let entries = store.active_path(&session_id, &leaf).unwrap();
|
||||
let plan = luxe_core::compaction::find_compaction_point(
|
||||
&entries,
|
||||
&settings,
|
||||
budget,
|
||||
luxe_core::compaction::CompactionTrigger::Threshold,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!plan.messages_to_summarize.is_empty());
|
||||
assert!(!luxe_core::compaction::serialize_for_summary(&plan.messages_to_summarize).is_empty());
|
||||
|
||||
let formatted = luxe_core::compaction::format_structured_summary(
|
||||
"## Goal\nTest compaction.\n\n## Next Steps\nContinue testing.",
|
||||
&plan.file_ops,
|
||||
);
|
||||
assert!(formatted.contains("Test compaction"));
|
||||
|
||||
let compacted = store
|
||||
.compact(&session_id, Some(leaf), &formatted, &plan.first_kept, plan.tokens_before)
|
||||
.unwrap();
|
||||
|
||||
// The compaction is the tip, and the whole session is still reachable through it — nothing is
|
||||
// deleted, so a later `/tree` or an audit can still read what was summarized.
|
||||
let path = store.active_path(&session_id, &compacted).unwrap();
|
||||
assert_eq!(path.len(), 102, "header + 100 messages + compaction");
|
||||
|
||||
// But the request built from that path carries the summary in place of what it replaced.
|
||||
let messages = luxe_core::compaction::context_messages(&path);
|
||||
let rendered: Vec<String> = messages
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
.filter_map(|block| match block {
|
||||
luxe_core::message::ContentBlock::Text(text) => Some(text.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert!(messages.len() < 20, "the summarized span is gone: {} messages", messages.len());
|
||||
assert!(rendered[0].contains("Test compaction"), "the summary leads: {}", &rendered[0][..80]);
|
||||
assert!(rendered[0].contains("<summary>"), "and says what it is");
|
||||
assert!(
|
||||
!rendered[1..].iter().any(|text| text.starts_with("Message 0 ")),
|
||||
"the oldest message was summarized away"
|
||||
);
|
||||
assert!(
|
||||
rendered.iter().any(|text| text.starts_with("Response 99 ")),
|
||||
"the newest message survives verbatim"
|
||||
);
|
||||
|
||||
// And the occupancy the next threshold check reads is the smaller one it produced.
|
||||
let before = luxe_core::compaction::estimate_tokens(&path);
|
||||
let after = luxe_core::compaction::context_tokens(&path);
|
||||
assert!(after * 4 < before, "context shrank from {before} to {after} tokens");
|
||||
|
||||
std::fs::remove_dir_all(&tmp_dir).unwrap();
|
||||
}
|
||||
Reference in New Issue
Block a user