Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
292
crates/luxe-core/src/compaction/budget.rs
Normal file
292
crates/luxe-core/src/compaction/budget.rs
Normal file
@@ -0,0 +1,292 @@
|
||||
//! Budget — what a request occupies, and how much of the window it may use.
|
||||
|
||||
use super::context::context_entries;
|
||||
use super::CompactionSettings;
|
||||
use crate::message::{ContentBlock, Message, ToolOutput};
|
||||
use crate::provider::ToolSchema;
|
||||
use crate::session::Entry;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Estimate the token count for a set of entries.
|
||||
///
|
||||
/// ~1 token per 4 characters, over **every** kind of content that reaches the model. That breadth is
|
||||
/// the whole point: a coding session's context is almost entirely tool results and reasoning, so
|
||||
/// counting only `Text` blocks is off by two orders of magnitude and reads a full window as empty.
|
||||
///
|
||||
/// Still only an estimate. Where the provider has told us the real occupancy, prefer that:
|
||||
/// [`ContextBudget::observed_tokens`] and [`last_observed_context_tokens`].
|
||||
pub fn estimate_tokens(entries: &[Entry]) -> u64 {
|
||||
entries.iter().map(entry_to_token_estimate).sum()
|
||||
}
|
||||
|
||||
/// Estimate the tokens a provider-facing message list occupies.
|
||||
///
|
||||
/// The counterpart to [`estimate_tokens`] for the copy that is actually sent — after tool-call
|
||||
/// pairing and [`super::prune_tool_outputs`] have edited it, and for the synthetic one-message
|
||||
/// prompts the summarizer and the review guide build, which never existed as session entries.
|
||||
pub fn estimate_message_tokens(messages: &[Message]) -> u64 {
|
||||
messages.iter().map(msg_to_token_estimate).sum()
|
||||
}
|
||||
|
||||
/// Estimate the tokens a request spends before any conversation: the system prompt and the tool
|
||||
/// schemas.
|
||||
///
|
||||
/// Far from negligible — Luxe's harness plus two dozen JSON schemas is tens of thousands of
|
||||
/// tokens, and leaving it out of a prompt figure is how a request that "fits" turns into a 400.
|
||||
pub fn estimate_prefix_tokens(system: Option<&str>, tools: &[ToolSchema]) -> u64 {
|
||||
let system = system.map(|text| text.len() as u64 / 4).unwrap_or(0);
|
||||
let tools: u64 = tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
(tool.name.len() as u64
|
||||
+ tool.description.len() as u64
|
||||
+ tool.parameters.to_string().len() as u64)
|
||||
/ 4
|
||||
})
|
||||
.sum();
|
||||
system + tools
|
||||
}
|
||||
|
||||
/// Estimate tokens for a single entry.
|
||||
pub(super) fn entry_to_token_estimate(entry: &Entry) -> u64 {
|
||||
match &entry.kind {
|
||||
crate::session::EntryKind::Message(msg) => msg_to_token_estimate(msg),
|
||||
crate::session::EntryKind::Compaction { summary, .. } => summary.len() as u64 / 4,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Estimate tokens for a single message.
|
||||
fn msg_to_token_estimate(msg: &Message) -> u64 {
|
||||
msg.content.iter().map(block_token_estimate).sum()
|
||||
}
|
||||
|
||||
/// Estimate tokens for one content block.
|
||||
///
|
||||
/// A tool call costs its name plus its serialized arguments; reasoning costs its text, and its opaque
|
||||
/// replay payloads cost real tokens on the wire even though no one can read them. Images are the one
|
||||
/// thing characters cannot describe, so they take the same flat figure [`tool_result_tokens`] uses.
|
||||
fn block_token_estimate(block: &ContentBlock) -> u64 {
|
||||
match block {
|
||||
ContentBlock::Text(text) => text.len() as u64 / 4,
|
||||
ContentBlock::Thinking { text, signature, id, encrypted, opaque } => {
|
||||
let replayed = [signature, id, encrypted, opaque]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|value| value.len() as u64)
|
||||
.sum::<u64>();
|
||||
(text.len() as u64 + replayed) / 4
|
||||
}
|
||||
ContentBlock::RedactedThinking { data } => data.len() as u64 / 4,
|
||||
ContentBlock::ToolCall { name, arguments, .. } => {
|
||||
(name.len() as u64 + arguments.to_string().len() as u64) / 4
|
||||
}
|
||||
ContentBlock::ToolResult { content, .. } => tool_result_tokens(content),
|
||||
ContentBlock::Image { .. } => IMAGE_TOKEN_ESTIMATE,
|
||||
}
|
||||
}
|
||||
|
||||
/// The most recent context occupancy the provider actually reported, if this transcript carries one.
|
||||
///
|
||||
/// Reads the persisted [`crate::session::EntryKind::Usage`] ledger backwards, which is a measurement
|
||||
/// rather than a guess — the number the model billed for. Two rules make it trustworthy:
|
||||
///
|
||||
/// - **Stop at a compaction.** A usage record written before the cut describes the context that
|
||||
/// compaction just replaced, so carrying it forward would report a window that no longer exists and
|
||||
/// re-trigger compaction on a session that had just finished one.
|
||||
/// - **One call only.** A usage entry folds together every call in its flush, and while the turn loop
|
||||
/// writes one call per entry, a detached pass can batch several; summing their prompt tokens is not
|
||||
/// an occupancy figure of anything.
|
||||
///
|
||||
/// Returns `None` when the transcript has nothing measured to offer, and the caller falls back to
|
||||
/// [`estimate_tokens`].
|
||||
pub fn last_observed_context_tokens(entries: &[Entry]) -> Option<u64> {
|
||||
last_observed_measurement(entries).map(|(_, occupancy)| occupancy)
|
||||
}
|
||||
|
||||
/// [`last_observed_context_tokens`], plus where in the transcript it was measured, so a caller can
|
||||
/// estimate only what has been appended since.
|
||||
fn last_observed_measurement(entries: &[Entry]) -> Option<(usize, u64)> {
|
||||
for (index, entry) in entries.iter().enumerate().rev() {
|
||||
match &entry.kind {
|
||||
crate::session::EntryKind::Compaction { .. } => return None,
|
||||
crate::session::EntryKind::Usage { source, totals, .. }
|
||||
if *source == crate::usage::UsageSource::Agent && totals.calls == 1 =>
|
||||
{
|
||||
let occupancy = totals.prompt_tokens + totals.completion_tokens;
|
||||
if occupancy > 0 {
|
||||
return Some((index, occupancy));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// What the next request built from this transcript will occupy on the way in, in tokens.
|
||||
///
|
||||
/// Prefers the provider's own last measurement, which already counts the system prompt and tool
|
||||
/// schemas, plus an estimate of the entries appended since. With nothing measured (a fresh session,
|
||||
/// or one whose last usage predates a compaction) it estimates [`context_entries`] plus the prefix:
|
||||
/// what a compaction replaced is not sent, so it is not counted.
|
||||
pub fn prompt_tokens(entries: &[Entry], system: Option<&str>, tools: &[ToolSchema]) -> u64 {
|
||||
match last_observed_measurement(entries) {
|
||||
// The measurement already includes the prefix that call was sent with.
|
||||
Some((index, observed)) => observed + estimate_tokens(&entries[index + 1..]),
|
||||
None => context_tokens(entries) + estimate_prefix_tokens(system, tools),
|
||||
}
|
||||
}
|
||||
|
||||
/// Estimated tokens for the conversation a request would carry — [`estimate_tokens`] over
|
||||
/// [`context_entries`].
|
||||
pub fn context_tokens(entries: &[Entry]) -> u64 {
|
||||
context_entries(entries).into_iter().map(entry_to_token_estimate).sum()
|
||||
}
|
||||
|
||||
/// The occupancy at which a conversation gets summarized, or `None` for a window too small (or too
|
||||
/// unknown) to reason about.
|
||||
///
|
||||
/// One owner, because two callers need the same answer: the planner, which is about to load the
|
||||
/// whole active path to work out where to cut, and the turn loop, which asks once per tool
|
||||
/// iteration and must be able to say "nowhere near it" without paying for that load.
|
||||
pub fn compaction_threshold(usable_window: u64, settings: &CompactionSettings) -> Option<u64> {
|
||||
(usable_window > 0).then_some((usable_window as f64 * settings.compact_at_fraction) as u64)
|
||||
}
|
||||
|
||||
/// The usable input budget: the real context window minus a capped output reserve.
|
||||
///
|
||||
/// The reserve is `min(max_tokens, reserve_tokens)`, so we leave room for the model's reply without
|
||||
/// ever sacrificing the whole window to a large `max_tokens` (a 128K-output model would otherwise
|
||||
/// reserve 128K). When `max_tokens` is unknown (0), reserve the full cap. Mirrors Claude Code's
|
||||
/// `min(max_output, 20K)` and OpenCode's `context − maxOutput` fallback.
|
||||
pub fn usable_context_window(context_window: u64, max_tokens: u32, reserve_tokens: u32) -> u64 {
|
||||
let reserve = if max_tokens == 0 { reserve_tokens } else { max_tokens.min(reserve_tokens) };
|
||||
context_window.saturating_sub(reserve as u64)
|
||||
}
|
||||
|
||||
/// Held back from the window when capping a request's output.
|
||||
///
|
||||
/// The prompt figure is part measurement, part estimate, and a tokenizer that disagrees by a few
|
||||
/// percent on a 200K prompt is thousands of tokens out. Mirrors Pi's margin.
|
||||
pub const CONTEXT_SAFETY_TOKENS: u32 = 4_096;
|
||||
|
||||
/// The smallest output cap Luxe will ask for once the window is full. Useless as an answer, but a
|
||||
/// request that asks for zero (or a negative) output is a hard rejection rather than a short reply.
|
||||
const MIN_MAX_TOKENS: u32 = 1;
|
||||
|
||||
/// Cap one request's output so the prompt and the reply both fit inside the context window.
|
||||
///
|
||||
/// `max_tokens` is a ceiling, not an allocation, and must never claim room the prompt already
|
||||
/// occupies: llama.cpp and LM Studio silently clamp an oversized cap, while vLLM and OpenAI reject
|
||||
/// the request outright.
|
||||
///
|
||||
/// Returns `None` for a configured `0`, which every adapter but Anthropic reads as "omit the
|
||||
/// field". An unknown window (`0`) can't be reasoned about, so the cap passes through untouched.
|
||||
///
|
||||
/// Deliberately independent of [`usable_context_window`]: that one decides when to *summarize* a
|
||||
/// conversation, this one decides what a single request may ask for, and tying them together would
|
||||
/// make either knob move the other.
|
||||
pub fn clamp_max_tokens_to_context(
|
||||
context_window: u32,
|
||||
prompt_tokens: u64,
|
||||
max_tokens: u32,
|
||||
) -> Option<u32> {
|
||||
if max_tokens == 0 || context_window == 0 {
|
||||
return (max_tokens > 0).then_some(max_tokens);
|
||||
}
|
||||
let available = u64::from(context_window)
|
||||
.saturating_sub(prompt_tokens)
|
||||
.saturating_sub(u64::from(CONTEXT_SAFETY_TOKENS))
|
||||
.min(u64::from(u32::MAX)) as u32;
|
||||
if available < MIN_MAX_TOKENS {
|
||||
warn!(
|
||||
context_window,
|
||||
prompt_tokens, "the prompt fills the context window; asking for a minimal reply"
|
||||
);
|
||||
}
|
||||
let capped = max_tokens.min(available.max(MIN_MAX_TOKENS));
|
||||
if capped < max_tokens {
|
||||
debug!(
|
||||
max_tokens,
|
||||
capped, prompt_tokens, context_window, "capped output to fit the window"
|
||||
);
|
||||
}
|
||||
Some(capped)
|
||||
}
|
||||
|
||||
/// Percent of the usable window currently occupied. Returns 0.0 when either input is 0 (e.g. no
|
||||
/// usage observed yet), so callers read "unknown" as empty rather than full.
|
||||
pub fn context_fullness_percent(used_tokens: u64, usable: u64) -> f64 {
|
||||
if usable == 0 || used_tokens == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
(used_tokens as f64 / usable as f64) * 100.0
|
||||
}
|
||||
|
||||
/// Rough per-image context cost. Images aren't char/4 — providers price them by dimensions
|
||||
/// (~1K–1.6K tokens for a typical screenshot); a flat estimate is close enough for prune accounting.
|
||||
///
|
||||
/// Shared with the `/context` measurement, so a picture is worth the same in both.
|
||||
pub(crate) const IMAGE_TOKEN_ESTIMATE: u64 = 1_500;
|
||||
|
||||
/// Token estimate for a tool result's payload: text via char/4, images at a flat rate.
|
||||
pub(super) fn tool_result_tokens(content: &[ToolOutput]) -> u64 {
|
||||
content
|
||||
.iter()
|
||||
.map(|o| match o {
|
||||
ToolOutput::Text(t) => t.len() as u64 / 4,
|
||||
ToolOutput::Image { .. } => IMAGE_TOKEN_ESTIMATE,
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// The live model context budget used to decide compaction.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct ContextBudget {
|
||||
/// The model's real context window, in tokens.
|
||||
pub context_window: u64,
|
||||
/// The model's max output; caps the reserved reply slice.
|
||||
pub max_tokens: u32,
|
||||
/// Exact occupancy from the provider's last usage event, if known. Preferred over the char/4
|
||||
/// estimate — both Claude Code and OpenCode trigger on real usage.
|
||||
pub observed_tokens: Option<u64>,
|
||||
/// Estimated tokens the last request spent before any conversation: system prompt and tool
|
||||
/// schemas.
|
||||
///
|
||||
/// Needed for two things the cut cannot be made correctly without — splitting a measured
|
||||
/// occupancy into prefix and conversation, and remembering that the prefix occupies the window
|
||||
/// the kept tail has to share.
|
||||
pub prefix_tokens: u64,
|
||||
}
|
||||
|
||||
/// What the char/4 estimate has to be multiplied by to land on real tokens.
|
||||
///
|
||||
/// The estimate is a constant and tokenizers are not: prose runs near four characters per token,
|
||||
/// source nearer 2.4. That matters wherever the two units meet — `keep_recent_tokens` is a
|
||||
/// real-token budget, and the walk that honors it counts in estimated ones, so an uncalibrated tail
|
||||
/// comes out far larger than asked and leaves compaction nothing to free.
|
||||
///
|
||||
/// Derived from the one measurement available: what the provider said the last request cost against
|
||||
/// what this crate would have guessed for it. Blended across the prefix and the conversation,
|
||||
/// because splitting them would need a second measurement nobody reports, and tool schemas are JSON
|
||||
/// that tokenizes about as badly as code anyway.
|
||||
///
|
||||
/// Clamped, because it multiplies a cut: a wild ratio must not be able to summarize the whole
|
||||
/// session or none of it. `1.0` — believe the estimate — whenever nothing has been measured yet,
|
||||
/// and never less: a ratio below it would make the kept tail *larger* than the setting says, which
|
||||
/// is the failure this exists to fix.
|
||||
pub fn estimate_calibration(observed: Option<u64>, estimated: u64) -> f64 {
|
||||
/// Believe the estimate at worst; above this, no real tokenizer.
|
||||
const RANGE: std::ops::RangeInclusive<f64> = 1.0..=4.0;
|
||||
|
||||
match observed {
|
||||
Some(observed) if observed > 0 && estimated > 0 => {
|
||||
(observed as f64 / estimated as f64).clamp(*RANGE.start(), *RANGE.end())
|
||||
}
|
||||
_ => 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
318
crates/luxe-core/src/compaction/budget/tests.rs
Normal file
318
crates/luxe-core/src/compaction/budget/tests.rs
Normal file
@@ -0,0 +1,318 @@
|
||||
use super::*;
|
||||
use crate::compaction::*;
|
||||
use crate::message::{ContentBlock, Message, Role};
|
||||
use crate::session::{Entry, EntryKind};
|
||||
|
||||
#[test]
|
||||
fn usable_window_caps_the_output_reserve() {
|
||||
// A big max_tokens must not eat the whole window: the reserve is capped at reserve_tokens.
|
||||
assert_eq!(usable_context_window(1_000_000, 128_000, 20_000), 980_000);
|
||||
// A small max_tokens caps the reserve at itself — never over-reserve.
|
||||
assert_eq!(usable_context_window(200_000, 4_096, 20_000), 195_904);
|
||||
// Unknown max_tokens (0) reserves the full cap.
|
||||
assert_eq!(usable_context_window(200_000, 0, 20_000), 180_000);
|
||||
// Degenerate: reserve exceeds the window -> 0, which the caller reads as "don't compact".
|
||||
assert_eq!(usable_context_window(10_000, 0, 20_000), 0);
|
||||
}
|
||||
|
||||
/// `max_tokens` is a ceiling, not an allocation. A configured cap that overlaps the prompt is
|
||||
/// latent rather than safe: llama.cpp and LM Studio clamp it silently, vLLM and OpenAI reject
|
||||
/// the whole request — so the bug only appears when the endpoint changes.
|
||||
#[test]
|
||||
fn the_output_cap_never_claims_room_the_prompt_is_using() {
|
||||
let clamp = clamp_max_tokens_to_context;
|
||||
|
||||
// Room to spare: the configured cap is exactly what goes out.
|
||||
assert_eq!(clamp(262_144, 10_000, 131_072), Some(131_072));
|
||||
|
||||
// The reported case: a 242K prompt under a 262K window used to ship a 131K cap on top,
|
||||
// asking for 373K against a 262K window.
|
||||
let capped = clamp(262_144, 242_144, 131_072).expect("a cap");
|
||||
assert_eq!(capped, 262_144 - 242_144 - CONTEXT_SAFETY_TOKENS);
|
||||
assert!(242_144 + u64::from(capped) <= u64::from(262_144u32 - CONTEXT_SAFETY_TOKENS));
|
||||
|
||||
// An unknown window can't be reasoned about, so nothing is guessed.
|
||||
assert_eq!(clamp(0, 500_000, 131_072), Some(131_072));
|
||||
// A configured 0 still means "omit the field", which is not the same as "ask for none".
|
||||
assert_eq!(clamp(262_144, 10_000, 0), None);
|
||||
assert_eq!(clamp(0, 10_000, 0), None);
|
||||
|
||||
// A prompt that has eaten the window asks for a minimal reply rather than zero or a
|
||||
// negative, either of which is a hard rejection instead of a short answer.
|
||||
assert_eq!(clamp(8_192, 8_192, 32_000), Some(MIN_MAX_TOKENS));
|
||||
assert_eq!(clamp(8_192, 999_999, 32_000), Some(MIN_MAX_TOKENS));
|
||||
}
|
||||
|
||||
/// The prompt figure the clamp runs on: measured where the provider told us, estimated only
|
||||
/// for the tail it never saw.
|
||||
#[test]
|
||||
fn the_prompt_figure_prefers_what_the_provider_measured() {
|
||||
use crate::usage::{UsageSource, UsageTotals};
|
||||
|
||||
let text = |chars: usize| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user("x".repeat(chars))),
|
||||
};
|
||||
let usage = |prompt: u64, completion: u64| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Usage {
|
||||
source: UsageSource::Agent,
|
||||
model: "test-model".into(),
|
||||
totals: UsageTotals {
|
||||
calls: 1,
|
||||
prompt_tokens: prompt,
|
||||
completion_tokens: completion,
|
||||
..UsageTotals::default()
|
||||
},
|
||||
},
|
||||
};
|
||||
// `read` + 400 characters of description + `{}` = 406 characters, at four to the token.
|
||||
let tools = [ToolSchema {
|
||||
name: "read".into(),
|
||||
description: "d".repeat(400),
|
||||
parameters: serde_json::json!({}),
|
||||
}];
|
||||
assert_eq!(estimate_prefix_tokens(None, &tools), 101);
|
||||
assert_eq!(estimate_prefix_tokens(Some(&"s".repeat(8_000)), &tools), 2_101);
|
||||
assert_eq!(estimate_prefix_tokens(None, &[]), 0);
|
||||
|
||||
// Nothing measured: the whole transcript, plus the prefix the transcript can't see. The
|
||||
// system prompt and the tool schemas are most of a fresh request, so leaving them out is
|
||||
// how a request that "fits" turns into a 400.
|
||||
let fresh = [text(4_000)];
|
||||
assert_eq!(prompt_tokens(&fresh, None, &[]), 1_000);
|
||||
assert_eq!(prompt_tokens(&fresh, Some(&"s".repeat(8_000)), &tools), 1_000 + 2_101);
|
||||
|
||||
// Measured: the measurement already includes that call's prefix, so only what was appended
|
||||
// after it is estimated on top.
|
||||
let live = [text(4_000), usage(60_000, 500), text(8_000)];
|
||||
assert_eq!(prompt_tokens(&live, Some("system"), &tools), 60_500 + 2_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fullness_percent_is_used_over_usable() {
|
||||
assert_eq!(context_fullness_percent(90_000, 180_000), 50.0);
|
||||
// Unknowns read as empty, never full.
|
||||
assert_eq!(context_fullness_percent(0, 180_000), 0.0);
|
||||
assert_eq!(context_fullness_percent(90_000, 0), 0.0);
|
||||
}
|
||||
|
||||
/// Two units meet in the cut, and getting them wrong is what left compaction with nothing to
|
||||
/// free: `keep_recent_tokens` is a real-token budget, and the walk that honors it counts char/4
|
||||
/// estimates. On source, which tokenizes nearer 2.4 characters per token, a tail asked to be
|
||||
/// 20k came out around 33k.
|
||||
#[test]
|
||||
fn the_kept_tail_is_measured_in_the_tokens_the_provider_charges_for() {
|
||||
// A transcript the estimate reads as 20k that the provider measured at 34k.
|
||||
assert!((estimate_calibration(Some(34_000), 20_000) - 1.7).abs() < 1e-9);
|
||||
// Nothing measured yet, so nothing to correct with.
|
||||
assert_eq!(estimate_calibration(None, 20_000), 1.0);
|
||||
assert_eq!(estimate_calibration(Some(0), 20_000), 1.0);
|
||||
assert_eq!(estimate_calibration(Some(34_000), 0), 1.0);
|
||||
// A ratio under the estimate would make the tail *bigger* than the setting asked for,
|
||||
// which is the failure this exists to fix — so the estimate is the floor.
|
||||
assert_eq!(estimate_calibration(Some(500), 20_000), 1.0);
|
||||
// And no single measurement gets to summarize the entire session.
|
||||
assert_eq!(estimate_calibration(Some(9_000_000), 20_000), 4.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_tokens() {
|
||||
let entries = vec![Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message {
|
||||
role: Role::User,
|
||||
content: vec![ContentBlock::Text("Hello world".to_string())],
|
||||
}),
|
||||
}];
|
||||
|
||||
let tokens = estimate_tokens(&entries);
|
||||
// "Hello world" is 11 chars, so ~2-3 tokens.
|
||||
assert!(tokens >= 2);
|
||||
assert!(tokens <= 4);
|
||||
}
|
||||
|
||||
/// The bug that emptied the context reading on resume, in miniature: a coding session's context is
|
||||
/// tool results, tool calls and reasoning, and counting only `Text` blocks put a 70K-token
|
||||
/// transcript at 956. Estimating *less than a tenth* of the truth is not conservatism — it is a
|
||||
/// compaction check that never fires.
|
||||
#[test]
|
||||
fn the_estimate_counts_everything_that_reaches_the_model() {
|
||||
let entry = |content: Vec<ContentBlock>| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message { role: Role::Assistant, content }),
|
||||
};
|
||||
let body = "x".repeat(4_000); // ~1000 tokens each
|
||||
let cases: Vec<(&str, ContentBlock)> = vec![
|
||||
("text", ContentBlock::Text(body.clone())),
|
||||
(
|
||||
"reasoning",
|
||||
ContentBlock::Thinking {
|
||||
text: body.clone(),
|
||||
signature: None,
|
||||
id: None,
|
||||
encrypted: None,
|
||||
opaque: None,
|
||||
},
|
||||
),
|
||||
("redacted reasoning", ContentBlock::RedactedThinking { data: body.clone() }),
|
||||
(
|
||||
"tool call",
|
||||
ContentBlock::ToolCall {
|
||||
id: "call_1".into(),
|
||||
name: "read".into(),
|
||||
arguments: serde_json::json!({ "path": body.clone() }),
|
||||
},
|
||||
),
|
||||
(
|
||||
"tool result",
|
||||
ContentBlock::ToolResult {
|
||||
call_id: "call_1".into(),
|
||||
content: vec![ToolOutput::Text(body.clone())],
|
||||
is_error: false,
|
||||
},
|
||||
),
|
||||
];
|
||||
for (label, block) in cases {
|
||||
let tokens = estimate_tokens(&[entry(vec![block])]);
|
||||
assert!(tokens >= 900, "{label} contributed only {tokens} tokens");
|
||||
assert!(tokens <= 1_100, "{label} inflated 4,000 characters to {tokens} tokens");
|
||||
}
|
||||
// An image has no characters to count, so it takes the flat per-image figure.
|
||||
let image = ContentBlock::Image { data: "AAAA".into(), mime: "image/png".into(), name: None };
|
||||
assert_eq!(estimate_tokens(&[entry(vec![image])]), IMAGE_TOKEN_ESTIMATE);
|
||||
|
||||
// A summary is the whole of what a compacted conversation still occupies, so an entry that
|
||||
// counted as nothing would report a compacted session as an empty one.
|
||||
assert_eq!(
|
||||
estimate_tokens(&[Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: "s".repeat(4_000),
|
||||
first_kept: EntryId::new(),
|
||||
tokens_before: 90_000,
|
||||
},
|
||||
}]),
|
||||
1_000
|
||||
);
|
||||
|
||||
// The rate itself, pinned, over both entry- and message-shaped input. The estimator is
|
||||
// load-bearing twice over — it decides when to compact, and through `prompt_tokens` how
|
||||
// much output a request may ask for — so a rate that is out by a factor drives every
|
||||
// request to the one-token floor.
|
||||
assert_eq!(estimate_tokens(&[entry(vec![ContentBlock::Text(body.clone())])]), 1_000);
|
||||
assert_eq!(
|
||||
estimate_message_tokens(&[
|
||||
Message::user(body.clone()),
|
||||
Message { role: Role::Assistant, content: vec![ContentBlock::Text(body)] },
|
||||
]),
|
||||
2_000
|
||||
);
|
||||
assert_eq!(estimate_message_tokens(&[]), 0);
|
||||
}
|
||||
|
||||
/// Resume prefers the measurement over the guess, and knows which measurements are meaningless.
|
||||
#[test]
|
||||
fn observed_occupancy_reads_the_last_agent_call_and_stops_at_a_compaction() {
|
||||
use crate::usage::{UsageSource, UsageTotals};
|
||||
|
||||
let usage = |source: UsageSource, calls: u64, prompt: u64, completion: u64| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Usage {
|
||||
source,
|
||||
model: "test-model".into(),
|
||||
totals: UsageTotals {
|
||||
calls,
|
||||
prompt_tokens: prompt,
|
||||
completion_tokens: completion,
|
||||
..UsageTotals::default()
|
||||
},
|
||||
},
|
||||
};
|
||||
let compaction = || Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: "summarized".into(),
|
||||
first_kept: EntryId::new(),
|
||||
tokens_before: 90_000,
|
||||
},
|
||||
};
|
||||
|
||||
// The newest agent call wins, and it is prompt + completion — what the window actually held.
|
||||
assert_eq!(
|
||||
last_observed_context_tokens(&[
|
||||
usage(UsageSource::Agent, 1, 10_000, 100),
|
||||
usage(UsageSource::Agent, 1, 69_897, 415),
|
||||
]),
|
||||
Some(70_312)
|
||||
);
|
||||
|
||||
// A record from before a compaction describes a context that no longer exists.
|
||||
assert_eq!(
|
||||
last_observed_context_tokens(&[usage(UsageSource::Agent, 1, 90_000, 500), compaction(),]),
|
||||
None,
|
||||
"stale occupancy would re-trigger compaction on a session that just compacted"
|
||||
);
|
||||
// ...but a call made *after* the compaction is current again.
|
||||
assert_eq!(
|
||||
last_observed_context_tokens(&[
|
||||
usage(UsageSource::Agent, 1, 90_000, 500),
|
||||
compaction(),
|
||||
usage(UsageSource::Agent, 1, 12_000, 200),
|
||||
]),
|
||||
Some(12_200)
|
||||
);
|
||||
|
||||
// A batched entry's summed prompt tokens are not an occupancy figure, and another source's
|
||||
// call measures its own context, not the turn's.
|
||||
assert_eq!(last_observed_context_tokens(&[usage(UsageSource::Agent, 4, 200_000, 900)]), None);
|
||||
assert_eq!(last_observed_context_tokens(&[usage(UsageSource::Review, 1, 50_000, 100)]), None);
|
||||
assert_eq!(last_observed_context_tokens(&[]), None);
|
||||
|
||||
// A call that measured nothing measured nothing — it is not an empty context. Reading it
|
||||
// as one would report a full window as fresh and hand the next request the whole output
|
||||
// cap on top of it.
|
||||
assert_eq!(
|
||||
last_observed_context_tokens(&[
|
||||
usage(UsageSource::Agent, 1, 80_000, 400),
|
||||
usage(UsageSource::Agent, 1, 0, 0),
|
||||
]),
|
||||
Some(80_400)
|
||||
);
|
||||
}
|
||||
|
||||
/// A `Usage` entry is a cost record, not conversation — only messages reach the model, so only
|
||||
/// messages may count toward what a request will occupy.
|
||||
///
|
||||
/// Load-bearing beyond the obvious: it is why `prompt_tokens` may estimate the entries *after*
|
||||
/// the measurement it starts from without having to care whether the measurement itself is
|
||||
/// included.
|
||||
#[test]
|
||||
fn usage_entries_are_inert_for_estimation() {
|
||||
let ledger = Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Usage {
|
||||
source: crate::usage::UsageSource::Agent,
|
||||
model: "a-model-with-a-long-name".to_string(),
|
||||
totals: crate::usage::UsageTotals { calls: 1, prompt_tokens: 90_000, ..<_>::default() },
|
||||
},
|
||||
};
|
||||
assert_eq!(entry_to_token_estimate(&ledger), 0);
|
||||
assert_eq!(estimate_tokens(std::slice::from_ref(&ledger)), 0);
|
||||
}
|
||||
74
crates/luxe-core/src/compaction/context.rs
Normal file
74
crates/luxe-core/src/compaction/context.rs
Normal file
@@ -0,0 +1,74 @@
|
||||
//! Context — the entries and messages a request is built from, after the newest cut.
|
||||
|
||||
use crate::message::Message;
|
||||
use crate::session::Entry;
|
||||
|
||||
/// How a compaction summary introduces itself to the model.
|
||||
///
|
||||
/// It has to read as a *report about* a conversation, not as a turn in one — an unframed summary
|
||||
/// gets answered rather than absorbed. The delimiters matter for the same reason: the model needs
|
||||
/// to see where someone else's account of the history stops and the live conversation starts.
|
||||
const SUMMARY_PREFIX: &str =
|
||||
"The conversation history before this point was compacted into the following summary:\n<summary>\n";
|
||||
const SUMMARY_SUFFIX: &str = "\n</summary>";
|
||||
|
||||
/// The entries a request is actually built from.
|
||||
///
|
||||
/// A session log is append-only, so compacting does not delete anything: the summarized messages
|
||||
/// stay on the parent chain and the [`EntryKind::Compaction`](crate::session::EntryKind::Compaction)
|
||||
/// entry is appended after them, naming the first entry it kept. This is the cut — the newest
|
||||
/// compaction's summary stands in front, then the entries it kept, then everything committed
|
||||
/// since. Without it a compacted session sends the whole transcript *plus* a summary of it, which
|
||||
/// is how compaction comes to cost tokens instead of saving them.
|
||||
///
|
||||
/// Only the newest compaction on the path replaces anything. An older one is either inside the span
|
||||
/// this one replaced or, when the newer cut landed before it, on the kept side, still standing for
|
||||
/// what it summarized. That one reads first: it covers history older than the newest summary and
|
||||
/// everything kept after it. A `first_kept` that isn't on the path (a hand-edited or partially
|
||||
/// written log) degrades to summary-plus-everything-after rather than dropping the conversation.
|
||||
pub fn context_entries(entries: &[Entry]) -> Vec<&Entry> {
|
||||
let Some(cut) = entries
|
||||
.iter()
|
||||
.rposition(|entry| matches!(entry.kind, crate::session::EntryKind::Compaction { .. }))
|
||||
else {
|
||||
return entries.iter().collect();
|
||||
};
|
||||
let crate::session::EntryKind::Compaction { first_kept, .. } = &entries[cut].kind else {
|
||||
return entries.iter().collect();
|
||||
};
|
||||
|
||||
let kept_from = entries[..cut].iter().position(|entry| entry.id == *first_kept);
|
||||
let mut context = Vec::with_capacity(entries.len() - cut + 1);
|
||||
let kept = kept_from.map_or(&[][..], |at| &entries[at..cut]);
|
||||
// Summaries first, oldest first. A compaction on the kept side was appended after the span it
|
||||
// summarized, so left where the log has it, it would read after messages that came later than
|
||||
// what it stands for.
|
||||
let is_summary =
|
||||
|entry: &&Entry| matches!(entry.kind, crate::session::EntryKind::Compaction { .. });
|
||||
context.extend(kept.iter().filter(is_summary));
|
||||
context.push(&entries[cut]);
|
||||
context.extend(kept.iter().filter(|entry| !is_summary(entry)));
|
||||
context.extend(entries[cut + 1..].iter());
|
||||
context
|
||||
}
|
||||
|
||||
/// The model-facing message list for a transcript: [`context_entries`], with the compaction
|
||||
/// summary rendered as the user message that replaces what it summarized.
|
||||
///
|
||||
/// The one place a transcript becomes a request, so the cut cannot be applied in some callers and
|
||||
/// forgotten in others.
|
||||
pub fn context_messages(entries: &[Entry]) -> Vec<Message> {
|
||||
context_entries(entries)
|
||||
.into_iter()
|
||||
.filter_map(|entry| match &entry.kind {
|
||||
crate::session::EntryKind::Message(message) => Some(message.clone()),
|
||||
crate::session::EntryKind::Compaction { summary, .. } => {
|
||||
Some(Message::user(format!("{SUMMARY_PREFIX}{summary}{SUMMARY_SUFFIX}")))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
108
crates/luxe-core/src/compaction/context/tests.rs
Normal file
108
crates/luxe-core/src/compaction/context/tests.rs
Normal file
@@ -0,0 +1,108 @@
|
||||
use super::*;
|
||||
use crate::compaction::*;
|
||||
use crate::message::{ContentBlock, Message, Role};
|
||||
use crate::session::{Entry, EntryKind};
|
||||
|
||||
/// A summary stands in front of the conversation it replaced, and only the newest one counts —
|
||||
/// an older compaction is inside the span this one already covers.
|
||||
#[test]
|
||||
fn the_context_is_the_summary_plus_what_it_kept_plus_what_came_after() {
|
||||
let text = |body: &str| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(body.to_string())),
|
||||
};
|
||||
let compaction = |summary: &str, first_kept: &EntryId| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: summary.to_string(),
|
||||
first_kept: first_kept.clone(),
|
||||
tokens_before: 1_000,
|
||||
},
|
||||
};
|
||||
let said = |messages: &[Message]| -> Vec<String> {
|
||||
messages
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
.filter_map(|block| match block {
|
||||
ContentBlock::Text(text) => Some(text.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let path: Vec<Entry> = vec![text("a"), text("b"), text("c"), text("d")];
|
||||
// No compaction: the path is the context, unchanged.
|
||||
assert_eq!(said(&context_messages(&path)), ["a", "b", "c", "d"]);
|
||||
|
||||
// Compacted at `c`, then two more turns.
|
||||
let mut compacted = path.clone();
|
||||
compacted.push(compaction("FIRST", &path[2].id));
|
||||
compacted.push(text("e"));
|
||||
let messages = context_messages(&compacted);
|
||||
assert_eq!(said(&messages)[1..], ["c", "d", "e"], "a and b were summarized away");
|
||||
assert!(said(&messages)[0].contains("FIRST"));
|
||||
assert!(said(&messages)[0].contains("<summary>"), "framed as a report, not a turn");
|
||||
assert_eq!(messages[0].role, Role::User);
|
||||
|
||||
// A second compaction supersedes the first — the first is inside the span it replaced.
|
||||
let mut twice = compacted.clone();
|
||||
twice.push(compaction("SECOND", &twice[5].id));
|
||||
let messages = context_messages(&twice);
|
||||
assert_eq!(said(&messages).len(), 2);
|
||||
assert!(said(&messages)[0].contains("SECOND"));
|
||||
assert!(!said(&messages)[0].contains("FIRST"), "only the newest cut applies");
|
||||
assert_eq!(said(&messages)[1], "e");
|
||||
|
||||
// A `first_kept` that isn't on the path (a truncated or hand-edited log) degrades to the
|
||||
// summary plus everything after it, rather than dropping the conversation.
|
||||
let mut orphaned = path.clone();
|
||||
orphaned.push(compaction("ORPHAN", &EntryId::new()));
|
||||
orphaned.push(text("f"));
|
||||
assert_eq!(said(&context_messages(&orphaned))[1..], ["f"]);
|
||||
}
|
||||
|
||||
/// A summary the newer cut left on the kept side is read before the newer one, and before anything
|
||||
/// kept: it stands for older history than either.
|
||||
///
|
||||
/// The break this catches: it stayed where the log had appended it, so a request read the newer
|
||||
/// summary, then the kept messages, then the older summary — history out of order.
|
||||
#[test]
|
||||
fn an_older_summary_on_the_kept_side_leads() {
|
||||
let text = |body: &str| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(body.to_string())),
|
||||
};
|
||||
let compaction = |summary: &str, first_kept: &EntryId| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: summary.to_string(),
|
||||
first_kept: first_kept.clone(),
|
||||
tokens_before: 1_000,
|
||||
},
|
||||
};
|
||||
let mut log: Vec<Entry> = ["a", "b", "c", "d"].into_iter().map(text).collect();
|
||||
log.push(compaction("OLDER", &log[2].id.clone()));
|
||||
log.push(text("e"));
|
||||
log.push(compaction("NEWER", &log[3].id.clone()));
|
||||
log.push(text("f"));
|
||||
|
||||
let said: Vec<String> = context_messages(&log)
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
.filter_map(|block| match block {
|
||||
ContentBlock::Text(text) => Some(text.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert!(said[0].contains("OLDER") && !said[0].contains("NEWER"), "{said:?}");
|
||||
assert!(said[1].contains("NEWER") && !said[1].contains("OLDER"), "{said:?}");
|
||||
assert_eq!(said[2..], ["d", "e", "f"]);
|
||||
}
|
||||
148
crates/luxe-core/src/compaction/planner.rs
Normal file
148
crates/luxe-core/src/compaction/planner.rs
Normal file
@@ -0,0 +1,148 @@
|
||||
//! Planner — where a compaction cuts, and what it hands the summarizer.
|
||||
|
||||
use super::budget::{
|
||||
compaction_threshold, context_tokens, entry_to_token_estimate, estimate_calibration,
|
||||
usable_context_window, ContextBudget,
|
||||
};
|
||||
use super::summary::{contributes_to_summary, files_listed_in, files_touched};
|
||||
use super::{CompactionError, CompactionPlan, CompactionSettings, CompactionTrigger};
|
||||
use crate::session::Entry;
|
||||
use tracing::debug;
|
||||
|
||||
/// Find the compaction point in loaded entries, given the active model's real context budget.
|
||||
///
|
||||
/// The budget comes from the live model so the trigger tracks the actual window, not a hardcoded
|
||||
/// default. Returns `Cancelled` when compaction isn't needed — or when the window is unknown (0),
|
||||
/// since compacting blind is worse than not compacting (OpenCode does the same).
|
||||
pub fn find_compaction_point(
|
||||
entries: &[Entry],
|
||||
settings: &CompactionSettings,
|
||||
budget: ContextBudget,
|
||||
trigger: CompactionTrigger,
|
||||
) -> Result<CompactionPlan, CompactionError> {
|
||||
// Prefer the provider's exact usage when we have it; fall back to the estimate over what a
|
||||
// request would actually carry, so a compaction that just landed reads as the smaller context
|
||||
// it produced rather than re-triggering on the history it replaced.
|
||||
let tokens_before =
|
||||
budget.observed_tokens.filter(|t| *t > 0).unwrap_or_else(|| context_tokens(entries));
|
||||
|
||||
let usable =
|
||||
usable_context_window(budget.context_window, budget.max_tokens, settings.reserve_tokens);
|
||||
if trigger == CompactionTrigger::Threshold {
|
||||
let Some(threshold) = compaction_threshold(usable, settings) else {
|
||||
return Err(CompactionError::Cancelled);
|
||||
};
|
||||
if tokens_before < threshold {
|
||||
debug!(tokens_before, threshold, "below compaction threshold");
|
||||
return Err(CompactionError::Cancelled);
|
||||
}
|
||||
}
|
||||
let conversation = context_tokens(entries);
|
||||
let calibration =
|
||||
estimate_calibration(budget.observed_tokens, conversation + budget.prefix_tokens);
|
||||
let prefix = (budget.prefix_tokens as f64 * calibration) as u64;
|
||||
|
||||
// `keep_recent_tokens` is a fixed count and the window is not: on a small model the default
|
||||
// tail is more than the whole window, and the walk below would never find a cut. So the tail
|
||||
// and the prefix (which shares that window) may hold at most half of it.
|
||||
let keep_recent = match usable {
|
||||
0 => u64::from(settings.keep_recent_tokens),
|
||||
usable => u64::from(settings.keep_recent_tokens).min((usable / 2).saturating_sub(prefix)),
|
||||
};
|
||||
|
||||
// Everything a previous compaction already replaced is out of scope: summarizing it again
|
||||
// would re-read the whole session on every compaction and fold the same history in twice.
|
||||
// The span starts at the entries that compaction *kept*, which are the ones still in play.
|
||||
let (boundary, previous_summary) = summarized_span_start(entries);
|
||||
|
||||
// Walk back from the end until the tail is worth `keep_recent` *real* tokens, which is
|
||||
// `keep_recent / calibration` of the estimated ones this walk can actually count.
|
||||
let target = (keep_recent as f64 / calibration) as u64;
|
||||
let mut accumulated = 0u64;
|
||||
let mut first_kept_idx = entries.len();
|
||||
|
||||
for (i, entry) in entries.iter().enumerate().skip(boundary).rev() {
|
||||
accumulated += entry_to_token_estimate(entry);
|
||||
if accumulated >= target {
|
||||
first_kept_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If the whole tail fits within keep_recent, there's nothing older to summarize — bail rather
|
||||
// than index past the end. This happens when occupancy is dominated by content that isn't in
|
||||
// the transcript entries (tool schemas, system prompt), so real usage can trip the threshold
|
||||
// while the entries themselves are small.
|
||||
if first_kept_idx >= entries.len() {
|
||||
debug!(entries = entries.len(), "nothing older than keep_recent to summarize");
|
||||
return Err(CompactionError::Cancelled);
|
||||
}
|
||||
// A cut that lands on a tool result keeps the result and summarizes away the call it answers,
|
||||
// which is an unpaired half on the wire and a hard 400 on both formats. Move the boundary
|
||||
// forward to the next entry that can legally start a request.
|
||||
let Some(first_kept_idx) =
|
||||
(first_kept_idx..entries.len()).find(|i| starts_context(&entries[*i]))
|
||||
else {
|
||||
debug!("no valid cut point in the recent tail");
|
||||
return Err(CompactionError::Cancelled);
|
||||
};
|
||||
|
||||
let messages_to_summarize = entries[boundary..first_kept_idx].to_vec();
|
||||
// Not merely non-empty: non-empty *of conversation*. A span of cost records and model switches
|
||||
// reaches the summarizer as a blank document, which is a paid call to summarize nothing
|
||||
if !messages_to_summarize.iter().any(contributes_to_summary) {
|
||||
debug!(entries = messages_to_summarize.len(), "nothing in the span reaches a summarizer");
|
||||
return Err(CompactionError::Cancelled);
|
||||
}
|
||||
let first_kept = entries[first_kept_idx].id.clone();
|
||||
|
||||
// What the last summary recorded, plus what this segment touched — the union is the running
|
||||
// record, and the second half is the only part that can ever add to it.
|
||||
let mut file_ops = previous_summary.as_deref().map(files_listed_in).unwrap_or_default();
|
||||
file_ops.extend(files_touched(&messages_to_summarize));
|
||||
|
||||
Ok(CompactionPlan {
|
||||
messages_to_summarize,
|
||||
turn_prefix: vec![],
|
||||
first_kept,
|
||||
previous_summary,
|
||||
file_ops,
|
||||
tokens_before,
|
||||
})
|
||||
}
|
||||
|
||||
/// Where the next summary's input starts, and the summary it continues from.
|
||||
///
|
||||
/// The entry after a previous compaction's `first_kept`, so each pass folds the last summary plus
|
||||
/// the messages that survived it into one — rather than starting from the beginning of a session
|
||||
/// whose early messages were replaced several compactions ago. A `first_kept` that isn't on the
|
||||
/// path falls back to the entry after the compaction itself.
|
||||
pub(super) fn summarized_span_start(entries: &[Entry]) -> (usize, Option<String>) {
|
||||
let Some(cut) = entries
|
||||
.iter()
|
||||
.rposition(|entry| matches!(entry.kind, crate::session::EntryKind::Compaction { .. }))
|
||||
else {
|
||||
return (0, None);
|
||||
};
|
||||
let crate::session::EntryKind::Compaction { summary, first_kept, .. } = &entries[cut].kind
|
||||
else {
|
||||
return (0, None);
|
||||
};
|
||||
let start = entries[..cut].iter().position(|entry| entry.id == *first_kept).unwrap_or(cut + 1);
|
||||
(start, Some(summary.clone()))
|
||||
}
|
||||
|
||||
/// Whether the kept side of a cut may begin with this entry.
|
||||
///
|
||||
/// A tool result may not: it answers a call that would be on the summarized side. Anything that
|
||||
/// carries no message at all may not either — the cut names an entry, and naming one the request
|
||||
/// never sees leaves the first *message* free to be a tool result again.
|
||||
fn starts_context(entry: &Entry) -> bool {
|
||||
match &entry.kind {
|
||||
crate::session::EntryKind::Message(message) => message.role != crate::message::Role::Tool,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
411
crates/luxe-core/src/compaction/planner/tests.rs
Normal file
411
crates/luxe-core/src/compaction/planner/tests.rs
Normal file
@@ -0,0 +1,411 @@
|
||||
use super::*;
|
||||
use crate::compaction::*;
|
||||
use crate::message::{ContentBlock, Message, Role, ToolOutput};
|
||||
use crate::session::{Entry, EntryKind};
|
||||
|
||||
#[test]
|
||||
fn find_compaction_point_tracks_the_real_window() {
|
||||
// ~160K estimated tokens across four entries (160K chars / 4 each).
|
||||
let chunk = "x".repeat(160_000);
|
||||
let entries: Vec<Entry> = (0..4)
|
||||
.map(|_| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(chunk.clone())),
|
||||
})
|
||||
.collect();
|
||||
let settings = CompactionSettings { keep_recent_tokens: 100, ..Default::default() };
|
||||
let budget = |window, max| ContextBudget {
|
||||
context_window: window,
|
||||
max_tokens: max,
|
||||
observed_tokens: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 200K window / 32K max: usable 180K, trigger 162K. 160K < 162K -> no compaction.
|
||||
assert!(matches!(
|
||||
find_compaction_point(
|
||||
&entries,
|
||||
&settings,
|
||||
budget(200_000, 32_000),
|
||||
CompactionTrigger::Threshold
|
||||
),
|
||||
Err(CompactionError::Cancelled)
|
||||
));
|
||||
// 128K window / 32K max: usable 108K, trigger ~97K. 160K over -> compaction planned.
|
||||
assert!(find_compaction_point(
|
||||
&entries,
|
||||
&settings,
|
||||
budget(128_000, 32_000),
|
||||
CompactionTrigger::Threshold
|
||||
)
|
||||
.is_ok());
|
||||
// Unknown window (0): never compact blind.
|
||||
assert!(matches!(
|
||||
find_compaction_point(&entries, &settings, budget(0, 32_000), CompactionTrigger::Threshold),
|
||||
Err(CompactionError::Cancelled)
|
||||
));
|
||||
|
||||
// The two budgets stay decoupled. When a conversation gets summarized is
|
||||
// `usable_context_window`'s call alone; what one request may ask for is
|
||||
// `clamp_max_tokens_to_context`'s. The 200K/32K case above is the pin — fold the request's
|
||||
// safety margin into the reserve and its threshold falls from 162K to ~158K, which starts
|
||||
// compacting a 160K transcript that had room to spare.
|
||||
assert_eq!(usable_context_window(200_000, 32_000, settings.reserve_tokens), 180_000);
|
||||
}
|
||||
|
||||
/// The prefix occupies the window too, and a cap that forgot it left compaction nothing to free
|
||||
/// on any model whose schemas are a real share of its context: keep half the window while the
|
||||
/// system prompt and tools hold another quarter, and a quarter is all there is to cut from.
|
||||
#[test]
|
||||
fn the_kept_tail_leaves_room_for_the_prompt_that_sits_above_it() {
|
||||
// Six messages the estimate reads as 10k each.
|
||||
let entries = user_entries(6, 40_000);
|
||||
let settings = CompactionSettings { keep_recent_tokens: 20_000, ..Default::default() };
|
||||
// Usable 39k, so the tail may hold 19.5k — of which the prefix has already taken 10k.
|
||||
let budget = ContextBudget {
|
||||
context_window: 45_000,
|
||||
max_tokens: 6_000,
|
||||
observed_tokens: Some(42_000),
|
||||
prefix_tokens: 10_000,
|
||||
};
|
||||
|
||||
let plan = find_compaction_point(&entries, &settings, budget, CompactionTrigger::Threshold)
|
||||
.expect("a full window yields a cut");
|
||||
// 9.5k of tail is one message; forget the prefix and it is two, and the window stays full.
|
||||
assert_eq!(plan.messages_to_summarize.len(), 5);
|
||||
|
||||
// The prefix is an estimate like everything else, so it is scaled by the same calibration
|
||||
// before being subtracted — a 6k estimate on a transcript measured at twice its guess is
|
||||
// 12k of real window gone, and taking it at face value gives the tail back room that is
|
||||
// not there.
|
||||
let plan = find_compaction_point(
|
||||
&user_entries(8, 10_000),
|
||||
&settings,
|
||||
ContextBudget {
|
||||
context_window: 60_000,
|
||||
max_tokens: 8_000,
|
||||
observed_tokens: Some(52_000),
|
||||
prefix_tokens: 6_000,
|
||||
},
|
||||
CompactionTrigger::Threshold,
|
||||
)
|
||||
.expect("a full window yields a cut");
|
||||
assert_eq!(plan.messages_to_summarize.len(), 5);
|
||||
}
|
||||
|
||||
/// The other half of the same arithmetic, isolated: with the cap far from binding, the walk
|
||||
/// still has to stop where `keep_recent_tokens` of *real* tokens is, not char/4 of them.
|
||||
#[test]
|
||||
fn a_calibrated_walk_stops_where_the_real_tokens_run_out() {
|
||||
// Eight messages the estimate reads as 2.5k each, measured at 1.7x that.
|
||||
let entries = user_entries(8, 10_000);
|
||||
let settings = CompactionSettings { keep_recent_tokens: 8_000, ..Default::default() };
|
||||
let budget = ContextBudget {
|
||||
context_window: 50_000,
|
||||
max_tokens: 6_000,
|
||||
observed_tokens: Some(42_500),
|
||||
prefix_tokens: 5_000,
|
||||
};
|
||||
|
||||
let plan = find_compaction_point(&entries, &settings, budget, CompactionTrigger::Threshold)
|
||||
.expect("a full window yields a cut");
|
||||
// 8k real is 4.7k estimated, which is two messages. Uncalibrated it reads as four, and the
|
||||
// tail kept is half again as large as the setting asked for.
|
||||
assert_eq!(plan.messages_to_summarize.len(), 6);
|
||||
}
|
||||
|
||||
/// `keep_recent_tokens` is a fixed count and the window is not, so on a small model the default
|
||||
/// 20K tail is larger than everything the model can hold. Capping it against the usable window
|
||||
/// is what leaves a history worth summarizing; uncapped, the walk never accumulates a cut and
|
||||
/// compaction silently never happens — on exactly the sessions that need it most.
|
||||
#[test]
|
||||
fn the_recent_tail_can_never_swallow_the_window_it_has_to_fit_in() {
|
||||
// Six ~2K-token entries, and a 16K window whose usable budget is 12K.
|
||||
let entries: Vec<Entry> = (0..6)
|
||||
.map(|_| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user("x".repeat(8_000))),
|
||||
})
|
||||
.collect();
|
||||
let settings = CompactionSettings { keep_recent_tokens: 20_000, ..Default::default() };
|
||||
let budget = ContextBudget { context_window: 16_000, max_tokens: 4_000, ..Default::default() };
|
||||
|
||||
let plan = find_compaction_point(&entries, &settings, budget, CompactionTrigger::Threshold)
|
||||
.expect("a full window has to yield a cut");
|
||||
assert!(
|
||||
!plan.messages_to_summarize.is_empty(),
|
||||
"a tail bigger than the window leaves nothing to summarize"
|
||||
);
|
||||
assert!(
|
||||
plan.messages_to_summarize.len() < entries.len(),
|
||||
"and it still has to keep a recent tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observed_tokens_override_the_estimate() {
|
||||
// Three ~40K-token entries => ~120K estimated, below the 162K trigger for a 200K window.
|
||||
let chunk = "x".repeat(160_000);
|
||||
let entries: Vec<Entry> = (0..3)
|
||||
.map(|_| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(chunk.clone())),
|
||||
})
|
||||
.collect();
|
||||
let settings = CompactionSettings { keep_recent_tokens: 100, ..Default::default() };
|
||||
let base = ContextBudget { context_window: 200_000, max_tokens: 32_000, ..Default::default() };
|
||||
|
||||
// Estimate alone (~120K < 162K) doesn't fire.
|
||||
assert!(matches!(
|
||||
find_compaction_point(&entries, &settings, base, CompactionTrigger::Threshold),
|
||||
Err(CompactionError::Cancelled)
|
||||
));
|
||||
// The provider's real usage (170K > 162K) flips the decision and still yields a real cut.
|
||||
let observed = ContextBudget { observed_tokens: Some(170_000), ..base };
|
||||
let plan =
|
||||
find_compaction_point(&entries, &settings, observed, CompactionTrigger::Threshold).unwrap();
|
||||
assert!(!plan.messages_to_summarize.is_empty());
|
||||
}
|
||||
|
||||
/// The cut names the first entry the request keeps, so it may not name a tool result: the call
|
||||
/// it answers would be on the summarized side, and an unpaired half is a hard 400 on both wire
|
||||
/// formats rather than a degraded reply.
|
||||
#[test]
|
||||
fn a_cut_never_strands_a_tool_result_from_its_call() {
|
||||
let entry = |kind: EntryKind| Entry { id: EntryId::new(), parent_id: None, ts: 0, kind };
|
||||
let assistant_call = |body: &str| {
|
||||
entry(EntryKind::Message(Message {
|
||||
role: Role::Assistant,
|
||||
content: vec![
|
||||
ContentBlock::Text(body.to_string()),
|
||||
ContentBlock::ToolCall {
|
||||
id: "call-1".into(),
|
||||
name: "read".into(),
|
||||
arguments: serde_json::json!({}),
|
||||
},
|
||||
],
|
||||
}))
|
||||
};
|
||||
let result = |body: &str| {
|
||||
entry(EntryKind::Message(Message {
|
||||
role: Role::Tool,
|
||||
content: vec![ContentBlock::ToolResult {
|
||||
call_id: "call-1".into(),
|
||||
content: vec![ToolOutput::Text(body.to_string())],
|
||||
is_error: false,
|
||||
}],
|
||||
}))
|
||||
};
|
||||
let user = |body: &str| entry(EntryKind::Message(Message::user(body.to_string())));
|
||||
|
||||
// Sized so the token walk lands squarely on the tool result.
|
||||
let big = "x".repeat(40_000);
|
||||
let entries = vec![user(&big), assistant_call(&big), result(&big), user("the next thing")];
|
||||
let settings = CompactionSettings { keep_recent_tokens: 9_000, ..Default::default() };
|
||||
let plan = find_compaction_point(
|
||||
&entries,
|
||||
&settings,
|
||||
ContextBudget {
|
||||
context_window: 40_000,
|
||||
max_tokens: 4_000,
|
||||
observed_tokens: Some(39_000),
|
||||
..Default::default()
|
||||
},
|
||||
CompactionTrigger::Threshold,
|
||||
)
|
||||
.expect("a plan");
|
||||
|
||||
let kept = entries.iter().position(|entry| entry.id == plan.first_kept).expect("on path");
|
||||
assert!(
|
||||
!matches!(&entries[kept].kind,
|
||||
EntryKind::Message(message) if message.role == Role::Tool),
|
||||
"the cut moved past the tool result rather than orphaning its call"
|
||||
);
|
||||
assert_eq!(kept, 3, "forward to the next entry that can legally start a request");
|
||||
}
|
||||
|
||||
/// The three edges of the trigger decision, each of which decides whether a provider call is
|
||||
/// spent: exactly at the threshold, a measurement of zero, and a summary whose `first_kept` is
|
||||
/// no longer on the path.
|
||||
#[test]
|
||||
fn the_trigger_edges_are_pinned() {
|
||||
let chunk = "x".repeat(160_000); // 40K tokens
|
||||
let entry = || Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(chunk.clone())),
|
||||
};
|
||||
let entries: Vec<Entry> = (0..5).map(|_| entry()).collect();
|
||||
let settings = CompactionSettings { keep_recent_tokens: 100, ..Default::default() };
|
||||
// usable = 200_000 - min(32_000, 20_000) = 180_000; the trigger is 90% of that.
|
||||
let budget = |observed| ContextBudget {
|
||||
context_window: 200_000,
|
||||
max_tokens: 32_000,
|
||||
observed_tokens: observed,
|
||||
..Default::default()
|
||||
};
|
||||
let plan = |observed| {
|
||||
find_compaction_point(&entries, &settings, budget(observed), CompactionTrigger::Threshold)
|
||||
};
|
||||
|
||||
// Landing exactly on the threshold compacts — it is "at or over", not "over".
|
||||
assert!(plan(Some(162_000)).is_ok(), "162_000 is the threshold, and it counts");
|
||||
assert!(plan(Some(161_999)).is_err(), "one token under it does not");
|
||||
|
||||
// A measurement of zero is not an empty context, it is no measurement — fall back to the
|
||||
// estimate (5 x 40K here), which is well over.
|
||||
assert!(plan(Some(0)).is_ok(), "a zero reading defers to the estimate, not to itself");
|
||||
|
||||
// Manual compaction ignores the threshold entirely.
|
||||
assert!(find_compaction_point(&entries, &settings, budget(Some(1)), CompactionTrigger::Manual)
|
||||
.is_ok());
|
||||
|
||||
// A small window cannot keep a fixed 20K tail and still have anything to summarize. Left
|
||||
// unclamped the walk never finds a cut, and compaction silently never happens — on the
|
||||
// sessions most in need of it.
|
||||
let cramped = CompactionSettings { keep_recent_tokens: 20_000, ..settings.clone() };
|
||||
let plan = find_compaction_point(
|
||||
&entries,
|
||||
&cramped,
|
||||
ContextBudget {
|
||||
context_window: 24_000,
|
||||
max_tokens: 4_000,
|
||||
observed_tokens: Some(23_000),
|
||||
..Default::default()
|
||||
},
|
||||
CompactionTrigger::Threshold,
|
||||
);
|
||||
assert!(plan.is_ok(), "a 20K tail against a 20K usable window still compacts");
|
||||
|
||||
// A previous compaction whose `first_kept` is no longer on the path (a truncated log)
|
||||
// resumes from just after that compaction rather than from the top of the session.
|
||||
let mut orphaned = entries.clone();
|
||||
orphaned.insert(
|
||||
2,
|
||||
Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: "OLD".into(),
|
||||
first_kept: EntryId::new(),
|
||||
tokens_before: 1,
|
||||
},
|
||||
},
|
||||
);
|
||||
let (start, summary) = summarized_span_start(&orphaned);
|
||||
assert_eq!(start, 3, "the entry after the compaction, not the one before it");
|
||||
assert_eq!(summary.as_deref(), Some("OLD"));
|
||||
}
|
||||
|
||||
/// A second compaction folds in only what the first one kept, wherever its own cut lands.
|
||||
///
|
||||
/// After the cut: the prior summary is inside the span, so the summarizer reads it and the new
|
||||
/// summary replaces both. Before the cut: the prior summary stays on the kept side, still standing
|
||||
/// for what it replaced, and the new one covers only the entries in between. Either way nothing
|
||||
/// from before the first cut is read again, and nothing is lost.
|
||||
#[test]
|
||||
fn a_second_compaction_folds_in_only_what_the_first_one_kept() {
|
||||
let text = |body: &str| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(format!("{body}{}", " ".repeat(3_999)))),
|
||||
};
|
||||
let compaction = |summary: &str, first_kept: &EntryId| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: summary.to_string(),
|
||||
first_kept: first_kept.clone(),
|
||||
tokens_before: 1_000,
|
||||
},
|
||||
};
|
||||
let said = |messages: &[Message]| -> Vec<String> {
|
||||
let text = |block: &ContentBlock| match block {
|
||||
ContentBlock::Text(text) => Some(text.trim().to_string()),
|
||||
_ => None,
|
||||
};
|
||||
messages.iter().flat_map(|message| &message.content).filter_map(text).collect()
|
||||
};
|
||||
// a..d at 1,000 tokens each, compacted at `c`, then two more turns
|
||||
let mut log: Vec<Entry> = ["a", "b", "c", "d"].into_iter().map(text).collect();
|
||||
log.push(compaction("FIRST", &log[2].id));
|
||||
log.extend(["e", "f"].into_iter().map(text));
|
||||
let plan = |keep_recent_tokens| {
|
||||
let settings = CompactionSettings { keep_recent_tokens, ..Default::default() };
|
||||
find_compaction_point(&log, &settings, ContextBudget::default(), CompactionTrigger::Manual)
|
||||
.expect("a plan")
|
||||
};
|
||||
let compacted_again = |plan: &CompactionPlan| {
|
||||
let mut twice = log.clone();
|
||||
twice.push(compaction("SECOND", &plan.first_kept));
|
||||
twice.push(text("g"));
|
||||
said(&context_messages(&twice))
|
||||
};
|
||||
|
||||
// The cut lands after the prior summary: `c`, `d` and FIRST are folded into SECOND
|
||||
let after = plan(1_500);
|
||||
assert_eq!(after.first_kept, log[5].id);
|
||||
assert_eq!(after.previous_summary.as_deref(), Some("FIRST"));
|
||||
let read = serialize_for_summary(&after.messages_to_summarize);
|
||||
assert!(read.contains("[User]: c") && read.contains("[Compaction]: FIRST"), "{read}");
|
||||
assert!(!read.contains("[User]: a") && !read.contains("[User]: b"), "{read}");
|
||||
let context = compacted_again(&after);
|
||||
assert!(context[0].contains("SECOND") && !context[0].contains("FIRST"));
|
||||
assert_eq!(context[1..], ["e", "f", "g"]);
|
||||
|
||||
// The cut lands before it: only `c` is folded, and FIRST rides on the kept side
|
||||
let before = plan(2_500);
|
||||
assert_eq!(before.first_kept, log[3].id);
|
||||
assert_eq!(before.previous_summary.as_deref(), Some("FIRST"));
|
||||
let read = serialize_for_summary(&before.messages_to_summarize);
|
||||
assert!(read.contains("[User]: c") && !read.contains("FIRST"), "{read}");
|
||||
let context = compacted_again(&before);
|
||||
// Oldest first: FIRST stands for a and b, SECOND for c, and only then what was kept
|
||||
assert!(context[0].contains("FIRST") && !context[0].contains("SECOND"), "{context:?}");
|
||||
assert!(context[1].contains("SECOND") && !context[1].contains("FIRST"), "{context:?}");
|
||||
assert_eq!(context[2..], ["d", "e", "f", "g"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_to_summarize_is_cancelled_not_a_panic() {
|
||||
// Occupancy trips the trigger (observed) but the transcript is tiny — must not index OOB.
|
||||
let entries = vec![Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user("hi".to_string())),
|
||||
}];
|
||||
let settings = CompactionSettings { keep_recent_tokens: 100, ..Default::default() };
|
||||
let budget = ContextBudget {
|
||||
context_window: 200_000,
|
||||
max_tokens: 32_000,
|
||||
observed_tokens: Some(170_000),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(
|
||||
find_compaction_point(&entries, &settings, budget, CompactionTrigger::Threshold),
|
||||
Err(CompactionError::Cancelled)
|
||||
));
|
||||
}
|
||||
|
||||
/// `count` user messages of `chars` each — a transcript whose estimate is exactly known.
|
||||
fn user_entries(count: usize, chars: usize) -> Vec<Entry> {
|
||||
(0..count)
|
||||
.map(|_| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user("x".repeat(chars))),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
162
crates/luxe-core/src/compaction/prune.rs
Normal file
162
crates/luxe-core/src/compaction/prune.rs
Normal file
@@ -0,0 +1,162 @@
|
||||
//! Prune — eliding stale tool output from the outgoing copy, only under real window pressure.
|
||||
|
||||
use super::budget::{context_tokens, estimate_prefix_tokens, tool_result_tokens};
|
||||
use super::CompactionSettings;
|
||||
use crate::message::{ContentBlock, Message, ToolOutput};
|
||||
use crate::provider::ToolSchema;
|
||||
use crate::session::Entry;
|
||||
use tracing::debug;
|
||||
|
||||
/// The fullest this session's window has been since its last compaction.
|
||||
///
|
||||
/// A high-water mark rather than current occupancy, because the prune decision would otherwise read
|
||||
/// its own output: eliding drops occupancy back under the gate, the next request keeps everything,
|
||||
/// occupancy climbs again — and each flip rewrites the transcript and re-bills the conversation
|
||||
/// behind it. A mark only rises within an epoch, so the gate opens once and stays open.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq)]
|
||||
pub struct ContextPressure {
|
||||
/// The largest prompt the provider has measured this epoch, or the estimate until it has
|
||||
/// measured one.
|
||||
pub peak_tokens: u64,
|
||||
/// The usable input budget: the window minus the output reserve.
|
||||
pub usable_window: u64,
|
||||
}
|
||||
|
||||
impl ContextPressure {
|
||||
/// Measures how full this transcript has had the window, against the budget it has to fit in.
|
||||
///
|
||||
/// Prefers the provider's own figures: the char/4 estimate reads a prompt of code as about two
|
||||
/// thirds of its size, and a threshold hung off it fires late.
|
||||
pub fn measure(
|
||||
entries: &[Entry],
|
||||
system: Option<&str>,
|
||||
tools: &[ToolSchema],
|
||||
usable_window: u64,
|
||||
) -> Self {
|
||||
let peak_tokens = observed_context_peak(entries)
|
||||
.unwrap_or_else(|| context_tokens(entries) + estimate_prefix_tokens(system, tools));
|
||||
Self { peak_tokens, usable_window }
|
||||
}
|
||||
|
||||
/// The share of the budget the peak took, or `0.0` when the window is unknown — an unknown
|
||||
/// window is not evidence of pressure.
|
||||
pub fn fraction(self) -> f64 {
|
||||
if self.usable_window == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
self.peak_tokens as f64 / self.usable_window as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// The largest input the provider has reported since the newest compaction, if any.
|
||||
///
|
||||
/// Same reading rules as [`super::last_observed_context_tokens`] — stop at a compaction, because a
|
||||
/// measurement from before the cut describes a context that no longer exists, and count only
|
||||
/// single-call records, because a batched flush's summed prompt tokens are not an occupancy figure
|
||||
/// of anything.
|
||||
fn observed_context_peak(entries: &[Entry]) -> Option<u64> {
|
||||
let mut peak = None;
|
||||
for entry in entries.iter().rev() {
|
||||
match &entry.kind {
|
||||
crate::session::EntryKind::Compaction { .. } => break,
|
||||
crate::session::EntryKind::Usage { source, totals, .. }
|
||||
if *source == crate::usage::UsageSource::Agent && totals.calls == 1 =>
|
||||
{
|
||||
peak = peak.max(Some(totals.prompt_tokens));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
peak.filter(|tokens| *tokens > 0)
|
||||
}
|
||||
|
||||
/// Non-destructively shrink stale tool-result bodies in the model-facing message list.
|
||||
///
|
||||
/// Every tool result older than the newest `prune_protect_tokens` of tool output is replaced with a
|
||||
/// compact placeholder. Only the provider-facing copy is edited; the durable session keeps the full
|
||||
/// text. `call_id`/`is_error` are preserved so the tool_use↔tool_result pairing stays valid, and
|
||||
/// images go too (a stale screenshot rarely earns its place once it is past the protected tail).
|
||||
///
|
||||
/// # Why it is so reluctant
|
||||
///
|
||||
/// Eliding rewrites the *middle* of the transcript, and every provider that caches does it by exact
|
||||
/// prefix match — so one elision re-bills the entire conversation after it as a cache write, at
|
||||
/// 1.25x input rather than 0.1x. At Anthropic's rates that is break-even only after roughly
|
||||
/// `12.5 × conversation ÷ elided` further calls, typically hundreds. Eliding is
|
||||
/// therefore never a saving in itself: it buys *runway*, and it is worth paying for only when the
|
||||
/// window is genuinely running out.
|
||||
///
|
||||
/// Two rules keep it honest, and between them the outgoing transcript is append-only except at a
|
||||
/// handful of deliberate steps:
|
||||
///
|
||||
/// - **A pressure gate.** Nothing is touched below `prune_at_fraction` of the usable window, where
|
||||
/// eliding would spend most of a session's cost on runway it does not need.
|
||||
/// - **A quantized boundary.** The cut is snapped to a multiple of `prune_min_tokens` and measured
|
||||
/// from the *oldest* end, so it holds still between steps. Measured from the newest end it would
|
||||
/// slide by one result per turn, rewriting the transcript on every single call.
|
||||
pub fn prune_tool_outputs(
|
||||
messages: &mut [Message],
|
||||
settings: &CompactionSettings,
|
||||
pressure: ContextPressure,
|
||||
) {
|
||||
if !settings.enabled || pressure.fraction() < settings.prune_at_fraction {
|
||||
return;
|
||||
}
|
||||
let boundary = prune_boundary(messages, settings);
|
||||
if boundary == 0 {
|
||||
return;
|
||||
}
|
||||
debug!(
|
||||
boundary,
|
||||
peak = pressure.peak_tokens,
|
||||
window = pressure.usable_window,
|
||||
"eliding stale tool output"
|
||||
);
|
||||
elide_tool_output_before(messages, boundary);
|
||||
}
|
||||
|
||||
/// Tokens of tool output, counted from the oldest, that are stale enough to elide.
|
||||
///
|
||||
/// `0` means "leave the transcript alone": either the protected tail covers everything, or what is
|
||||
/// left over has not yet reached a full `prune_min_tokens` step.
|
||||
fn prune_boundary(messages: &[Message], settings: &CompactionSettings) -> u64 {
|
||||
let step = (settings.prune_min_tokens as u64).max(1);
|
||||
let total: u64 = tool_output_tokens(messages).map(|(_, _, tokens)| tokens).sum();
|
||||
total.saturating_sub(settings.prune_protect_tokens as u64) / step * step
|
||||
}
|
||||
|
||||
/// Replace every tool result that ends at or before `boundary` tokens of tool output with a
|
||||
/// placeholder naming what it cost.
|
||||
fn elide_tool_output_before(messages: &mut [Message], boundary: u64) {
|
||||
let stale: Vec<(usize, usize, u64)> = tool_output_tokens(messages)
|
||||
.scan(0u64, |seen, (mi, bi, tokens)| {
|
||||
*seen += tokens;
|
||||
Some((mi, bi, *seen, tokens))
|
||||
})
|
||||
.take_while(|(_, _, seen, _)| *seen <= boundary)
|
||||
.map(|(mi, bi, _, tokens)| (mi, bi, tokens))
|
||||
.collect();
|
||||
for (mi, bi, tokens) in stale {
|
||||
if let ContentBlock::ToolResult { content, .. } = &mut messages[mi].content[bi] {
|
||||
*content = vec![ToolOutput::Text(format!(
|
||||
"[{tokens} tokens of earlier tool output elided to save context]"
|
||||
))];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Every non-empty tool result in the list as `(message, block, tokens)`, oldest first.
|
||||
fn tool_output_tokens(messages: &[Message]) -> impl Iterator<Item = (usize, usize, u64)> + '_ {
|
||||
messages.iter().enumerate().flat_map(|(mi, msg)| {
|
||||
msg.content.iter().enumerate().filter_map(move |(bi, block)| match block {
|
||||
ContentBlock::ToolResult { content, .. } => match tool_result_tokens(content) {
|
||||
0 => None,
|
||||
tokens => Some((mi, bi, tokens)),
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
304
crates/luxe-core/src/compaction/prune/tests.rs
Normal file
304
crates/luxe-core/src/compaction/prune/tests.rs
Normal file
@@ -0,0 +1,304 @@
|
||||
use super::*;
|
||||
use crate::compaction::*;
|
||||
use crate::message::{ContentBlock, Message, Role};
|
||||
use crate::session::{Entry, EntryKind};
|
||||
|
||||
fn tool_msg(call_id: &str, text_tokens: usize, is_error: bool) -> Entry {
|
||||
Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message {
|
||||
role: Role::Tool,
|
||||
content: vec![ContentBlock::ToolResult {
|
||||
call_id: call_id.to_string(),
|
||||
content: vec![ToolOutput::Text("x".repeat(text_tokens * 4))],
|
||||
is_error,
|
||||
}],
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn msg_of(entry: &Entry) -> Message {
|
||||
match &entry.kind {
|
||||
EntryKind::Message(m) => m.clone(),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A window with no room left, so a prune test exercises pruning rather than the gate.
|
||||
fn under_pressure() -> ContextPressure {
|
||||
ContextPressure { peak_tokens: 100_000, usable_window: 100_000 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_protects_recent_and_shrinks_old_tool_output() {
|
||||
let settings =
|
||||
CompactionSettings { prune_protect_tokens: 40, prune_min_tokens: 20, ..Default::default() };
|
||||
// oldest -> newest. Walking back, the tail totalling ~protect (c + b) stays verbatim; a is
|
||||
// older than the protected window and gets pruned.
|
||||
let mut messages =
|
||||
[tool_msg("a", 1000, false), tool_msg("b", 1000, true), tool_msg("c", 30, false)]
|
||||
.iter()
|
||||
.map(msg_of)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
prune_tool_outputs(&mut messages, &settings, under_pressure());
|
||||
|
||||
match &messages[0].content[0] {
|
||||
ContentBlock::ToolResult { call_id, content, is_error } => {
|
||||
assert_eq!(call_id, "a", "pairing id preserved");
|
||||
assert!(!is_error);
|
||||
match &content[0] {
|
||||
ToolOutput::Text(t) => assert!(t.contains("elided to save context")),
|
||||
_ => panic!("expected placeholder text"),
|
||||
}
|
||||
}
|
||||
_ => panic!("expected tool result"),
|
||||
}
|
||||
// b (still within the protected tail) is untouched, error flag intact.
|
||||
match &messages[1].content[0] {
|
||||
ContentBlock::ToolResult { content, is_error, .. } => {
|
||||
assert!(is_error);
|
||||
match &content[0] {
|
||||
ToolOutput::Text(t) => assert_eq!(t.len(), 4000),
|
||||
_ => panic!(),
|
||||
}
|
||||
}
|
||||
_ => panic!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_skips_when_gain_is_below_minimum() {
|
||||
// Same shape, but the minimum is huge: not worth churning the prompt cache.
|
||||
let settings = CompactionSettings {
|
||||
prune_protect_tokens: 40,
|
||||
prune_min_tokens: 100_000,
|
||||
..Default::default()
|
||||
};
|
||||
let mut messages =
|
||||
[tool_msg("a", 1000, false), tool_msg("b", 1000, false), tool_msg("c", 30, false)]
|
||||
.iter()
|
||||
.map(msg_of)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
prune_tool_outputs(&mut messages, &settings, under_pressure());
|
||||
match &messages[0].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => match &content[0] {
|
||||
ToolOutput::Text(t) => assert_eq!(t.len(), 4000, "nothing pruned below minimum"),
|
||||
_ => panic!(),
|
||||
},
|
||||
_ => panic!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_elides_old_images_too() {
|
||||
let settings =
|
||||
CompactionSettings { prune_protect_tokens: 40, prune_min_tokens: 20, ..Default::default() };
|
||||
let old_image = Message {
|
||||
role: Role::Tool,
|
||||
content: vec![ContentBlock::ToolResult {
|
||||
call_id: "shot".to_string(),
|
||||
content: vec![ToolOutput::Image {
|
||||
data: "base64==".to_string(),
|
||||
mime: "image/png".to_string(),
|
||||
name: None,
|
||||
}],
|
||||
is_error: false,
|
||||
}],
|
||||
};
|
||||
// Oldest = an image; a big recent text result fills the protected tail.
|
||||
let mut messages = vec![old_image, msg_of(&tool_msg("recent", 1000, false))];
|
||||
|
||||
prune_tool_outputs(&mut messages, &settings, under_pressure());
|
||||
|
||||
match &messages[0].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => match &content[0] {
|
||||
ToolOutput::Text(t) => assert!(t.contains("elided"), "stale image elided to text"),
|
||||
_ => panic!("image should be replaced by a text placeholder"),
|
||||
},
|
||||
_ => panic!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_is_a_noop_when_compaction_disabled() {
|
||||
let settings = CompactionSettings {
|
||||
enabled: false,
|
||||
prune_protect_tokens: 40,
|
||||
prune_min_tokens: 20,
|
||||
..Default::default()
|
||||
};
|
||||
let mut messages = [tool_msg("a", 1000, false), tool_msg("b", 1000, false)]
|
||||
.iter()
|
||||
.map(msg_of)
|
||||
.collect::<Vec<_>>();
|
||||
prune_tool_outputs(&mut messages, &settings, under_pressure());
|
||||
match &messages[0].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => match &content[0] {
|
||||
ToolOutput::Text(t) => assert_eq!(t.len(), 4000),
|
||||
_ => panic!(),
|
||||
},
|
||||
_ => panic!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The defect this whole gate exists for: a session at 29% of a 1M window spent 71% of its
|
||||
/// cost eliding tool output it had all the room in the world for.
|
||||
#[test]
|
||||
fn a_roomy_window_is_never_pruned() {
|
||||
let settings =
|
||||
CompactionSettings { prune_protect_tokens: 40, prune_min_tokens: 20, ..Default::default() };
|
||||
let mut messages = [tool_msg("a", 1000, false), tool_msg("b", 1000, false)]
|
||||
.iter()
|
||||
.map(msg_of)
|
||||
.collect::<Vec<_>>();
|
||||
let roomy = ContextPressure { peak_tokens: 290_000, usable_window: 980_000 };
|
||||
assert!(roomy.fraction() < settings.prune_at_fraction);
|
||||
|
||||
prune_tool_outputs(&mut messages, &settings, roomy);
|
||||
|
||||
let elided = |messages: &[Message]| match &messages[0].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => match &content[0] {
|
||||
ToolOutput::Text(text) => text.contains("elided"),
|
||||
_ => panic!("expected text"),
|
||||
},
|
||||
_ => panic!("expected a tool result"),
|
||||
};
|
||||
assert!(!elided(&messages), "nothing is elided with room left");
|
||||
|
||||
// The gate is "at or above", so landing exactly on it prunes. Pinned because the other
|
||||
// reading costs a whole conversation's worth of cache writes one call later.
|
||||
let exactly = ContextPressure { peak_tokens: 80_000, usable_window: 100_000 };
|
||||
assert!((exactly.fraction() - settings.prune_at_fraction).abs() < f64::EPSILON);
|
||||
prune_tool_outputs(&mut messages, &settings, exactly);
|
||||
assert!(elided(&messages), "a window at the gate is a window worth reclaiming");
|
||||
}
|
||||
|
||||
/// The expensive half of the same defect, and the invariant that replaces it.
|
||||
///
|
||||
/// Every provider that caches does it by exact prefix match, so the transcript a session sends
|
||||
/// has to be an *append* to the one it sent last time. The old sliding window moved the cut by
|
||||
/// one result per turn, which rewrote the middle of the array on all 31 of the turns below and
|
||||
/// re-billed the whole conversation as a cache write each time — no error, no warning, just a
|
||||
/// four-fold invoice.
|
||||
#[test]
|
||||
fn the_outgoing_transcript_is_append_only_across_turns() {
|
||||
let settings = CompactionSettings {
|
||||
prune_protect_tokens: 40_000,
|
||||
prune_min_tokens: 20_000,
|
||||
..Default::default()
|
||||
};
|
||||
let mut history = vec![Message::user("start")];
|
||||
let mut previous: Vec<String> = Vec::new();
|
||||
let mut rewrites = 0;
|
||||
|
||||
for turn in 0..60 {
|
||||
history.push(msg_of(&tool_msg(&format!("call-{turn}"), 2_000, false)));
|
||||
let mut outgoing = history.clone();
|
||||
// Pressure rises with the transcript, and crosses the gate part-way through.
|
||||
let pressure =
|
||||
ContextPressure { peak_tokens: 2_000 * (turn as u64 + 1), usable_window: 100_000 };
|
||||
prune_tool_outputs(&mut outgoing, &settings, pressure);
|
||||
|
||||
let rendered: Vec<String> = outgoing.iter().map(|m| format!("{m:?}")).collect::<Vec<_>>();
|
||||
if previous.iter().zip(rendered.iter()).any(|(before, after)| before != after) {
|
||||
rewrites += 1;
|
||||
}
|
||||
previous = rendered;
|
||||
}
|
||||
|
||||
// 120k of tool output in all; the gate opens at 80k, and above it the cut moves one
|
||||
// 20k step at a time — the opening cut plus two steps. Not the 31 the sliding window took.
|
||||
assert_eq!(rewrites, 3, "the prefix was rewritten {rewrites} times in 60 turns");
|
||||
}
|
||||
|
||||
/// The gate reads a high-water mark, not current occupancy — otherwise eliding lowers the
|
||||
/// figure that authorized it, the next request keeps everything back, and the two states
|
||||
/// alternate, rewriting the transcript on every flip.
|
||||
#[test]
|
||||
fn pressure_is_the_fullest_the_window_has_been_this_epoch() {
|
||||
let usage = |prompt: u32| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Usage {
|
||||
source: crate::usage::UsageSource::Agent,
|
||||
model: "test".to_string(),
|
||||
totals: crate::usage::UsageTotals {
|
||||
calls: 1,
|
||||
prompt_tokens: u64::from(prompt),
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
};
|
||||
let tagged = |prompt: u32, source, calls| {
|
||||
let mut entry = usage(prompt);
|
||||
if let EntryKind::Usage { source: tag, totals, .. } = &mut entry.kind {
|
||||
*tag = source;
|
||||
totals.calls = calls;
|
||||
}
|
||||
entry
|
||||
};
|
||||
let measure =
|
||||
|entries: &[Entry]| ContextPressure::measure(entries, None, &[], 100_000).peak_tokens;
|
||||
|
||||
// A later, smaller measurement is what a prune looks like; the mark holds.
|
||||
assert_eq!(measure(&[usage(40_000), usage(90_000), usage(52_000)]), 90_000);
|
||||
// Only the agent's own single-call records are occupancy: a review pass measures a prompt
|
||||
// of its own, and a batched flush sums several. Neither describes this conversation.
|
||||
assert_eq!(
|
||||
measure(&[
|
||||
usage(40_000),
|
||||
tagged(800_000, crate::usage::UsageSource::Review, 1),
|
||||
tagged(700_000, crate::usage::UsageSource::Agent, 4),
|
||||
]),
|
||||
40_000,
|
||||
);
|
||||
// A record of zero is not a measurement — taking it at its word would report an empty
|
||||
// window for a transcript that plainly is not one — so the estimate answers instead.
|
||||
let said = Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user("x".repeat(8_000))),
|
||||
};
|
||||
assert_eq!(measure(&[said, tagged(0, crate::usage::UsageSource::Agent, 1)]), 2_000);
|
||||
// A compaction starts a new epoch, and the window it emptied is not evidence of pressure.
|
||||
let compacted = vec![
|
||||
usage(90_000),
|
||||
Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: "s".repeat(400),
|
||||
first_kept: EntryId::new(),
|
||||
tokens_before: 90_000,
|
||||
},
|
||||
},
|
||||
usage(12_000),
|
||||
];
|
||||
assert_eq!(measure(&compacted), 12_000);
|
||||
}
|
||||
|
||||
/// The cut is measured from the oldest end so a given result's verdict never flips; measured
|
||||
/// from the newest it slides every time anything is appended.
|
||||
#[test]
|
||||
fn the_elision_boundary_moves_in_whole_steps() {
|
||||
let settings = CompactionSettings {
|
||||
prune_protect_tokens: 1_000,
|
||||
prune_min_tokens: 500,
|
||||
..Default::default()
|
||||
};
|
||||
let boundary = |tool_tokens: usize| {
|
||||
let messages = vec![msg_of(&tool_msg("one", tool_tokens, false))];
|
||||
prune_boundary(&messages, &settings)
|
||||
};
|
||||
assert_eq!(boundary(1_200), 0, "a part-step over the protected tail is not worth a rewrite");
|
||||
assert_eq!(boundary(1_600), 500);
|
||||
assert_eq!(boundary(1_999), 500, "and it holds there until the next whole step");
|
||||
assert_eq!(boundary(2_100), 1_000);
|
||||
}
|
||||
259
crates/luxe-core/src/compaction/summary.rs
Normal file
259
crates/luxe-core/src/compaction/summary.rs
Normal file
@@ -0,0 +1,259 @@
|
||||
//! Summary — what the summarizer reads, and the shape its reply is filed in.
|
||||
|
||||
use crate::message::{ContentBlock, ToolOutput};
|
||||
use crate::session::Entry;
|
||||
use std::collections::HashSet;
|
||||
|
||||
/// Characters of real prose a summary has to carry to be allowed to stand for a transcript.
|
||||
///
|
||||
/// Deliberately near the floor: judging a summary's *quality* from here would throw away terse but
|
||||
/// perfectly good ones, and the failure this exists for is not a thin summary but an absent one.
|
||||
const SUMMARY_MUST_CARRY_CHARS: usize = 40;
|
||||
|
||||
/// Whether a formatted summary carries enough to replace what it summarizes.
|
||||
///
|
||||
/// The headings are added by [`format_structured_summary`] whether or not the summarizer wrote
|
||||
/// them, and the file list is derived from the transcript rather than from the model — so a reply
|
||||
/// that came back empty still formats into a full-looking document of empty sections, and would
|
||||
/// replace the session's whole memory with it.
|
||||
///
|
||||
/// Counted over the prose only: headings and the file list are structure, not content.
|
||||
pub fn summary_carries_content(formatted: &str) -> bool {
|
||||
let mut in_files = false;
|
||||
let mut prose = 0usize;
|
||||
for line in formatted.lines() {
|
||||
if let Some(heading) = line.trim_end().strip_prefix("## ") {
|
||||
in_files = FILE_SECTION_ALIASES.contains(&heading.trim());
|
||||
continue;
|
||||
}
|
||||
if !in_files {
|
||||
prose += line.trim().chars().count();
|
||||
}
|
||||
}
|
||||
prose >= SUMMARY_MUST_CARRY_CHARS
|
||||
}
|
||||
|
||||
/// The heading the summary lists touched files under.
|
||||
const FILES_SECTION: &str = "Files";
|
||||
|
||||
/// Headings a file list may be read back from.
|
||||
///
|
||||
/// [`FILES_SECTION`] was two sections, `Read Files` and `Modified Files`, in older builds. A
|
||||
/// session compacted then still has them on disk, and dropping its list on resume would lose
|
||||
/// exactly the history the summary exists to keep.
|
||||
const FILE_SECTION_ALIASES: [&str; 3] = [FILES_SECTION, "Read Files", "Modified Files"];
|
||||
|
||||
/// Tools whose `path` argument names a file the session actually touched.
|
||||
///
|
||||
/// Deliberately not every tool with a path: `glob` and `grep` take one as a *scope*, and a search
|
||||
/// that swept the repository has not touched the repository.
|
||||
const FILE_TOOLS: [&str; 3] = ["read", "write", "edit"];
|
||||
|
||||
/// The files a previous summary listed, so a chain of compactions keeps the record.
|
||||
///
|
||||
/// Scoped to the file headings: over the whole document, any bullet in `Progress` or `Next Steps`
|
||||
/// with a dot in it would be filed as a filename, one compaction at a time.
|
||||
pub(super) fn files_listed_in(summary: &str) -> HashSet<String> {
|
||||
let mut files = HashSet::new();
|
||||
let mut inside = false;
|
||||
for line in summary.lines() {
|
||||
let line = line.trim();
|
||||
if let Some(heading) = line.strip_prefix("## ") {
|
||||
inside = FILE_SECTION_ALIASES.contains(&heading.trim());
|
||||
continue;
|
||||
}
|
||||
let listed =
|
||||
inside.then(|| line.strip_prefix("- ").or_else(|| line.strip_prefix("* "))).flatten();
|
||||
if let Some(file) = listed.map(str::trim).filter(|file| !file.is_empty()) {
|
||||
files.insert(file.to_string());
|
||||
}
|
||||
}
|
||||
files
|
||||
}
|
||||
|
||||
/// The files a stretch of conversation opened or changed, read off the calls that named them.
|
||||
///
|
||||
/// The transcript is the only honest source: the summarizer is asked to list files and writes
|
||||
/// prose, and the previous summary can only ever repeat what some earlier segment found. Reading
|
||||
/// the tool calls is what makes the list accumulate at all.
|
||||
pub(super) fn files_touched(entries: &[Entry]) -> HashSet<String> {
|
||||
let mut files = HashSet::new();
|
||||
for entry in entries {
|
||||
let crate::session::EntryKind::Message(message) = &entry.kind else { continue };
|
||||
for block in &message.content {
|
||||
let ContentBlock::ToolCall { name, arguments, .. } = block else { continue };
|
||||
if !FILE_TOOLS.contains(&name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if let Some(path) = arguments.get("path").and_then(serde_json::Value::as_str) {
|
||||
let path = path.trim();
|
||||
if !path.is_empty() {
|
||||
files.insert(path.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
files
|
||||
}
|
||||
|
||||
/// How much of any one block the summarizer is shown.
|
||||
///
|
||||
/// Its input is a whole context window by construction, so one `cargo build` log left whole would
|
||||
/// spend the entire summary budget on a build that succeeded.
|
||||
const SUMMARY_BLOCK_CHARS: usize = 2_000;
|
||||
|
||||
/// Render a stretch of transcript as the text the summarizer reads.
|
||||
///
|
||||
/// Tool activity goes in, because in a coding session it *is* the session: a summary written from
|
||||
/// the prose alone cannot say what was read, what changed, or what the tests did. Reasoning stays
|
||||
/// out — it is the model talking to itself on the way to the answer, and the answer is already
|
||||
/// here.
|
||||
pub fn serialize_for_summary(entries: &[Entry]) -> String {
|
||||
let mut output = String::new();
|
||||
for entry in entries {
|
||||
match &entry.kind {
|
||||
crate::session::EntryKind::Message(message) => {
|
||||
let role = match message.role {
|
||||
crate::message::Role::User => "User",
|
||||
crate::message::Role::Assistant => "Assistant",
|
||||
crate::message::Role::Tool => "Tool",
|
||||
crate::message::Role::System => "System",
|
||||
};
|
||||
for block in &message.content {
|
||||
if let Some(line) = summary_line(block) {
|
||||
output.push_str(&format!("[{role}]: {line}\n"));
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::session::EntryKind::Compaction { summary, .. } => {
|
||||
output.push_str(&format!("[Compaction]: {summary}\n"));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
/// Whether an entry puts anything into [`serialize_for_summary`]'s output.
|
||||
///
|
||||
/// The mirror of that function's `match`, and the reason it is a predicate rather than a length
|
||||
/// check on the serialized string: the decision to compact is made before the input is built.
|
||||
pub(super) fn contributes_to_summary(entry: &Entry) -> bool {
|
||||
match &entry.kind {
|
||||
crate::session::EntryKind::Message(message) => {
|
||||
message.content.iter().any(|block| summary_line(block).is_some())
|
||||
}
|
||||
crate::session::EntryKind::Compaction { .. } => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// One content block as a line of summarizer input, or `None` when it has nothing to contribute.
|
||||
fn summary_line(block: &ContentBlock) -> Option<String> {
|
||||
match block {
|
||||
ContentBlock::Text(text) => Some(clip_for_summary(text)),
|
||||
ContentBlock::ToolCall { name, arguments, .. } => {
|
||||
Some(format!("called {name} {}", clip_for_summary(&arguments.to_string())))
|
||||
}
|
||||
ContentBlock::ToolResult { content, is_error, .. } => {
|
||||
let body = content
|
||||
.iter()
|
||||
.map(|output| match output {
|
||||
ToolOutput::Text(text) => text.as_str(),
|
||||
ToolOutput::Image { .. } => "<image>",
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let label = if *is_error { "failed" } else { "returned" };
|
||||
Some(format!("{label} {}", clip_for_summary(&body)))
|
||||
}
|
||||
ContentBlock::Thinking { .. }
|
||||
| ContentBlock::RedactedThinking { .. }
|
||||
| ContentBlock::Image { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cap one block at [`SUMMARY_BLOCK_CHARS`], counted in characters.
|
||||
///
|
||||
/// Counted in characters rather than bytes because slicing a byte offset panics the moment it lands
|
||||
/// inside a multi-byte one — which for a transcript carrying an accent or an emoji is a crash in
|
||||
/// the middle of a compaction, on a turn the user never asked for.
|
||||
fn clip_for_summary(text: &str) -> String {
|
||||
let mut kept: String = text.chars().take(SUMMARY_BLOCK_CHARS).collect();
|
||||
if kept.chars().count() == SUMMARY_BLOCK_CHARS
|
||||
&& text.chars().nth(SUMMARY_BLOCK_CHARS).is_some()
|
||||
{
|
||||
kept.push_str("...");
|
||||
}
|
||||
kept
|
||||
}
|
||||
|
||||
/// The sections every summary carries, whether or not the summarizer wrote them.
|
||||
///
|
||||
/// An empty heading is deliberate: it tells the next model that the question was asked and had no
|
||||
/// answer, which reads very differently from the section having been forgotten.
|
||||
const REQUIRED_SECTIONS: [&str; 6] = [
|
||||
"Goal",
|
||||
"Constraints & Preferences",
|
||||
"Progress",
|
||||
"Key Decisions",
|
||||
"Next Steps",
|
||||
"Critical Context",
|
||||
];
|
||||
|
||||
/// Shape the summarizer's prose into the sections a resumed session expects, and append the file
|
||||
/// record it cannot write for itself.
|
||||
///
|
||||
/// `files` is the running list from [`super::CompactionPlan::file_ops`] — already the union of what
|
||||
/// the last summary recorded and what this segment touched — so this only has to render it.
|
||||
pub fn format_structured_summary(summarizer_output: &str, files: &HashSet<String>) -> String {
|
||||
let mut sections: Vec<(String, String)> = Vec::new();
|
||||
let mut current: Option<(String, String)> = None;
|
||||
|
||||
for line in summarizer_output.lines() {
|
||||
match line.trim_end().strip_prefix("## ") {
|
||||
Some(heading) => {
|
||||
sections.extend(current.take());
|
||||
current = Some((heading.trim().to_string(), String::new()));
|
||||
}
|
||||
None => {
|
||||
if let Some((_, content)) = current.as_mut() {
|
||||
content.push_str(line);
|
||||
content.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sections.extend(current);
|
||||
|
||||
for required in REQUIRED_SECTIONS {
|
||||
if !sections.iter().any(|(heading, _)| heading == required) {
|
||||
sections.push((required.to_string(), String::new()));
|
||||
}
|
||||
}
|
||||
// Replaced rather than appended, so a summarizer that wrote its own file list does not leave
|
||||
// the summary carrying two of them that disagree.
|
||||
sections.retain(|(heading, _)| !FILE_SECTION_ALIASES.contains(&heading.as_str()));
|
||||
sections.push((FILES_SECTION.to_string(), format_files(files)));
|
||||
|
||||
let mut output = String::new();
|
||||
for (heading, content) in §ions {
|
||||
output.push_str(&format!("## {heading}\n"));
|
||||
let content = content.trim();
|
||||
if !content.is_empty() {
|
||||
output.push_str(&format!("\n{content}\n"));
|
||||
}
|
||||
output.push('\n');
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
/// Format a list of files, sorted so two summaries of the same work read the same.
|
||||
fn format_files(files: &HashSet<String>) -> String {
|
||||
let mut sorted: Vec<&String> = files.iter().collect();
|
||||
sorted.sort();
|
||||
sorted.iter().map(|file| format!("- {file}")).collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
299
crates/luxe-core/src/compaction/summary/tests.rs
Normal file
299
crates/luxe-core/src/compaction/summary/tests.rs
Normal file
@@ -0,0 +1,299 @@
|
||||
use super::*;
|
||||
use crate::compaction::*;
|
||||
use crate::message::{ContentBlock, Message, Role};
|
||||
use crate::session::{Entry, EntryKind};
|
||||
|
||||
/// A span with no conversation in it is not compacted, however many entries it holds.
|
||||
///
|
||||
/// The break this catches, measured on a loop run: the planner's guard asked only whether the
|
||||
/// span was non-*empty*, and a span of cost records is not empty — so the summarizer was sent
|
||||
/// "Summarize the following conversation:" with nothing under it. Three paid calls, each
|
||||
/// answering, correctly, that no conversation had been provided.
|
||||
#[test]
|
||||
fn a_span_of_cost_records_is_not_a_conversation() {
|
||||
use crate::usage::{UsageSource, UsageTotals};
|
||||
|
||||
let usage = || Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Usage {
|
||||
source: UsageSource::Agent,
|
||||
model: "test-model".into(),
|
||||
totals: UsageTotals { calls: 1, ..UsageTotals::default() },
|
||||
},
|
||||
};
|
||||
let said = |body: &str| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message::user(body)),
|
||||
};
|
||||
|
||||
assert!(!contributes_to_summary(&usage()));
|
||||
assert!(contributes_to_summary(&said("something worth keeping")));
|
||||
assert!(
|
||||
serialize_for_summary(&[usage(), usage()]).trim().is_empty(),
|
||||
"which is exactly what would have been sent"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_for_summary() {
|
||||
let entries = vec![
|
||||
Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message {
|
||||
role: Role::User,
|
||||
content: vec![ContentBlock::Text("What is Rust?".to_string())],
|
||||
}),
|
||||
},
|
||||
Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message {
|
||||
role: Role::Assistant,
|
||||
content: vec![ContentBlock::Text(
|
||||
"Rust is a systems programming language.".to_string(),
|
||||
)],
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
let serialized = serialize_for_summary(&entries);
|
||||
assert!(serialized.contains("[User]: What is Rust?"));
|
||||
assert!(serialized.contains("[Assistant]: Rust is a systems programming language."));
|
||||
}
|
||||
|
||||
fn files<const N: usize>(names: [&str; N]) -> HashSet<String> {
|
||||
names.iter().map(|name| (*name).to_string()).collect()
|
||||
}
|
||||
|
||||
fn message_entry(role: Role, content: Vec<ContentBlock>) -> Entry {
|
||||
Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message { role, content }),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the summarizer is shown is what the summary can possibly say. Tool activity is most of
|
||||
/// a coding session, and it used to be dropped entirely — the function truncated "long tool
|
||||
/// results" it never actually matched.
|
||||
#[test]
|
||||
fn the_summarizer_is_shown_the_tool_work_not_only_the_prose() {
|
||||
let serialized = serialize_for_summary(&[
|
||||
message_entry(
|
||||
Role::Assistant,
|
||||
vec![
|
||||
ContentBlock::Thinking {
|
||||
text: "let me think".into(),
|
||||
signature: None,
|
||||
id: None,
|
||||
encrypted: None,
|
||||
opaque: None,
|
||||
},
|
||||
ContentBlock::ToolCall {
|
||||
id: "c1".into(),
|
||||
name: "edit".into(),
|
||||
arguments: serde_json::json!({ "path": "src/lib.rs" }),
|
||||
},
|
||||
],
|
||||
),
|
||||
message_entry(
|
||||
Role::Tool,
|
||||
vec![ContentBlock::ToolResult {
|
||||
call_id: "c1".into(),
|
||||
content: vec![ToolOutput::Text("no such file".into())],
|
||||
is_error: true,
|
||||
}],
|
||||
),
|
||||
]);
|
||||
|
||||
assert!(serialized.contains("[Assistant]: called edit"), "{serialized}");
|
||||
assert!(serialized.contains("src/lib.rs"), "{serialized}");
|
||||
assert!(serialized.contains("[Tool]: failed no such file"), "{serialized}");
|
||||
assert!(!serialized.contains("let me think"), "reasoning is not the record: {serialized}");
|
||||
}
|
||||
|
||||
/// A compaction summary is the history everything before it was replaced by, so a later pass
|
||||
/// summarizing across one has to be shown it.
|
||||
#[test]
|
||||
fn an_earlier_compaction_summary_reaches_the_summarizer() {
|
||||
let serialized = serialize_for_summary(&[Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Compaction {
|
||||
summary: "## Goal\n\nShip the cache fix.".to_string(),
|
||||
first_kept: EntryId::new(),
|
||||
tokens_before: 90_000,
|
||||
},
|
||||
}]);
|
||||
assert!(serialized.starts_with("[Compaction]: ## Goal"), "{serialized}");
|
||||
}
|
||||
|
||||
/// The crash this cap used to be: slicing byte 1997 of a block whose 1997th byte is the middle
|
||||
/// of a character panics, in the middle of a compaction the user never asked for.
|
||||
#[test]
|
||||
fn an_over_long_block_is_cut_on_a_character_not_a_byte() {
|
||||
let straddles = format!("{}\u{e9}{}", "a".repeat(1_996), "b".repeat(100));
|
||||
let serialized =
|
||||
serialize_for_summary(&[message_entry(Role::User, vec![ContentBlock::Text(straddles)])]);
|
||||
assert!(serialized.ends_with("...\n"), "the cut is marked: {}", &serialized[..40]);
|
||||
assert_eq!(serialized.chars().count(), "[User]: ".len() + SUMMARY_BLOCK_CHARS + 4);
|
||||
|
||||
// And a block that fits is passed through whole, with no ellipsis invented for it.
|
||||
let exact = "x".repeat(SUMMARY_BLOCK_CHARS);
|
||||
let fits = serialize_for_summary(&[message_entry(Role::User, vec![ContentBlock::Text(exact)])]);
|
||||
assert!(!fits.contains("..."), "nothing was cut");
|
||||
}
|
||||
|
||||
/// The whole shape, pinned: what the summarizer wrote, every heading it skipped, and one file
|
||||
/// list — sorted, so two summaries of the same work read the same.
|
||||
#[test]
|
||||
fn a_summary_carries_the_prose_the_missing_headings_and_one_file_list() {
|
||||
let formatted = format_structured_summary(
|
||||
"## Goal\nBuild a web server.\n\n## Next Steps\nDeploy to production.",
|
||||
&files(["src/main.rs", "Cargo.toml"]),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
formatted,
|
||||
"## Goal\n\nBuild a web server.\n\n\
|
||||
## Next Steps\n\nDeploy to production.\n\n\
|
||||
## Constraints & Preferences\n\n\
|
||||
## Progress\n\n\
|
||||
## Key Decisions\n\n\
|
||||
## Critical Context\n\n\
|
||||
## Files\n\n- Cargo.toml\n- src/main.rs\n\n"
|
||||
);
|
||||
}
|
||||
|
||||
/// A summary of empty sections is not a summary, whatever it looks like.
|
||||
///
|
||||
/// The break this catches, measured on a live loop with a local summarizer: the model returned
|
||||
/// nothing usable, the formatter added the six required headings and the running file list
|
||||
/// anyway, and a 20,677-token transcript was replaced by that — reported as a successful
|
||||
/// compaction. The session lost everything it knew and nothing said so.
|
||||
#[test]
|
||||
fn a_summary_of_empty_headings_does_not_get_to_stand_for_a_transcript() {
|
||||
let empty = format_structured_summary("", &files(["wordfreq.py", "test_wordfreq.py"]));
|
||||
assert!(empty.contains("## Goal"), "it still looks like a summary: {empty}");
|
||||
assert!(
|
||||
!summary_carries_content(&empty),
|
||||
"...and that is exactly why looking is not enough: {empty}"
|
||||
);
|
||||
|
||||
let headings_only = format_structured_summary(
|
||||
"## Goal\n\n## Progress\n\n## Next Steps\n",
|
||||
&files(["wordfreq.py"]),
|
||||
);
|
||||
assert!(!summary_carries_content(&headings_only), "{headings_only}");
|
||||
|
||||
let real = format_structured_summary(
|
||||
"## Goal\nBuild wordfreq, a standard-library word counter with a small argparse CLI.\n\n\
|
||||
## Progress\nPhases 0 and 1 are done and the suite is green at 20 tests.\n",
|
||||
&files(["wordfreq.py"]),
|
||||
);
|
||||
assert!(summary_carries_content(&real), "{real}");
|
||||
}
|
||||
|
||||
/// A long file list is structure, not content, and cannot carry an empty summary past the check.
|
||||
#[test]
|
||||
fn a_file_list_is_not_a_summary() {
|
||||
let listed = format_structured_summary(
|
||||
"",
|
||||
&files([
|
||||
"src/one/very/long/path/to/a/file.rs",
|
||||
"src/another/quite/long/path/second.rs",
|
||||
"src/a/third/long/path/third.rs",
|
||||
"src/a/fourth/long/path/fourth.rs",
|
||||
]),
|
||||
);
|
||||
assert!(!summary_carries_content(&listed), "{listed}");
|
||||
}
|
||||
|
||||
/// Prose the summarizer wrote before any heading is not a section, and must not be silently
|
||||
/// swallowed into one — or filed under a heading it does not belong to.
|
||||
#[test]
|
||||
fn a_preamble_before_the_first_heading_is_dropped_rather_than_misfiled() {
|
||||
let formatted =
|
||||
format_structured_summary("Here is my summary!\n\n## Goal\nShip it.", &HashSet::new());
|
||||
assert!(!formatted.contains("Here is my summary"), "{formatted}");
|
||||
assert!(formatted.starts_with("## Goal\n\nShip it.\n"), "{formatted}");
|
||||
assert!(formatted.ends_with("## Files\n\n"), "an empty list still asks: {formatted}");
|
||||
}
|
||||
|
||||
/// A summarizer that writes its own file list must not leave the summary carrying two that
|
||||
/// disagree — the running record is the one the transcript can prove.
|
||||
#[test]
|
||||
fn the_running_file_list_replaces_one_the_summarizer_invented() {
|
||||
let formatted = format_structured_summary(
|
||||
"## Goal\nShip it.\n\n## Files\n- imaginary.rs\n\n## Read Files\n- also-imaginary.rs",
|
||||
&files(["src/real.rs", "src/also-real.rs"]),
|
||||
);
|
||||
assert!(!formatted.contains("imaginary.rs"), "{formatted}");
|
||||
assert_eq!(formatted.matches("## Files").count(), 1, "{formatted}");
|
||||
assert!(!formatted.contains("## Read Files"), "the retired heading is gone: {formatted}");
|
||||
assert!(formatted.contains("- src/real.rs"), "{formatted}");
|
||||
}
|
||||
|
||||
/// The defect the file list existed to have and did not: it only ever repeated what an earlier
|
||||
/// summary said, so it started empty and stayed that way. The transcript is what makes it grow.
|
||||
#[test]
|
||||
fn the_file_list_comes_from_the_calls_that_named_the_files() {
|
||||
let call = |tool: &str, path: &str| Entry {
|
||||
id: EntryId::new(),
|
||||
parent_id: None,
|
||||
ts: 0,
|
||||
kind: EntryKind::Message(Message {
|
||||
role: Role::Assistant,
|
||||
content: vec![ContentBlock::ToolCall {
|
||||
id: EntryId::new().to_string(),
|
||||
name: tool.to_string(),
|
||||
arguments: serde_json::json!({ "path": path }),
|
||||
}],
|
||||
}),
|
||||
};
|
||||
let touched = files_touched(&[
|
||||
call("read", "src/lib.rs"),
|
||||
call("edit", "src/main.rs"),
|
||||
call("write", "README.md"),
|
||||
// A search scope is not a file this session touched.
|
||||
call("grep", "crates/"),
|
||||
call("read", " "),
|
||||
]);
|
||||
|
||||
let mut expected = files(["src/lib.rs", "src/main.rs"]);
|
||||
expected.insert("README.md".to_string());
|
||||
assert_eq!(touched, expected);
|
||||
}
|
||||
|
||||
/// A summary is read back the way it was written, and the headings it used to be written under
|
||||
/// still count — a session compacted before the rename must not lose its record on resume.
|
||||
#[test]
|
||||
fn a_file_list_survives_the_round_trip_and_the_rename() {
|
||||
let written = format_structured_summary("## Goal\nShip it.", &files(["a/b.rs", "c.toml"]));
|
||||
assert_eq!(files_listed_in(&written), files(["a/b.rs", "c.toml"]));
|
||||
|
||||
let legacy = "## Progress\n\n- Rewrote the parser. See notes.md for why.\n\n\
|
||||
## Read Files\n\n- src/old.rs\n\n## Modified Files\n\n- src/older.rs\n";
|
||||
assert_eq!(files_listed_in(legacy), files(["src/old.rs", "src/older.rs"]));
|
||||
}
|
||||
|
||||
/// A bullet in `Progress` is a sentence, not a filename. Parsed over the whole document — as it
|
||||
/// was — every one of them that contained a dot was filed as a file, forever.
|
||||
#[test]
|
||||
fn prose_outside_the_file_section_is_never_mistaken_for_a_file() {
|
||||
let summary = "## Progress\n\n- Fixed the cache in compaction.rs and shipped it.\n\n\
|
||||
## Files\n\n- crates/luxe-core/src/compaction.rs\n";
|
||||
assert_eq!(
|
||||
files_listed_in(summary),
|
||||
["crates/luxe-core/src/compaction.rs".to_string()].into_iter().collect::<HashSet<_>>()
|
||||
);
|
||||
}
|
||||
38
crates/luxe-core/src/compaction/tests.rs
Normal file
38
crates/luxe-core/src/compaction/tests.rs
Normal file
@@ -0,0 +1,38 @@
|
||||
use super::*;
|
||||
use crate::provider::ModelRef;
|
||||
|
||||
#[test]
|
||||
fn circuit_breaker_trips_after_the_failure_cap() {
|
||||
assert!(!compaction_circuit_open(0));
|
||||
assert!(!compaction_circuit_open(MAX_COMPACTION_FAILURES - 1));
|
||||
assert!(compaction_circuit_open(MAX_COMPACTION_FAILURES));
|
||||
assert!(compaction_circuit_open(MAX_COMPACTION_FAILURES + 5));
|
||||
}
|
||||
|
||||
/// The backstop behind the arithmetic: a summary that freed almost nothing has spent a provider
|
||||
/// call to leave occupancy exactly where it was, and the next iteration will ask again.
|
||||
#[test]
|
||||
fn a_compaction_that_freed_almost_nothing_does_not_count_as_working() {
|
||||
assert!(compaction_was_effective(1_000, 800), "a fifth freed is the bar");
|
||||
assert!(compaction_was_effective(1_000, 100), "and most of it, comfortably");
|
||||
assert!(!compaction_was_effective(1_000, 801), "just under it is not");
|
||||
assert!(!compaction_was_effective(1_000, 900), "nor a tenth");
|
||||
// A transcript that somehow grew has not freed anything, rather than a negative share.
|
||||
assert!(!compaction_was_effective(1_000, 1_200));
|
||||
// Nothing to free is not a failure to free it — the alternative reads as a broken
|
||||
// summarizer and trips a breaker over an empty session.
|
||||
assert!(compaction_was_effective(0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarizer_prefers_the_override_else_active() {
|
||||
let active = ModelInfo { r#ref: ModelRef::new("openai", "gpt-primary"), ..Default::default() };
|
||||
// No override -> the active model writes the summary.
|
||||
assert_eq!(summarizer_model(None, &active).r#ref.id, "gpt-primary");
|
||||
// Override -> that ref (provider + id), so a cheap cross-provider summarizer is honored.
|
||||
let cheap =
|
||||
ModelInfo { r#ref: ModelRef::new("anthropic", "claude-haiku"), ..Default::default() };
|
||||
let picked = summarizer_model(Some(&cheap), &active);
|
||||
assert_eq!(picked.r#ref.provider, "anthropic");
|
||||
assert_eq!(picked.r#ref.id, "claude-haiku");
|
||||
}
|
||||
Reference in New Issue
Block a user