优化: 缓存命中字段解析与提示词前缀稳定化
- openai 兼容口径补解析嵌套 cached_tokens - cache 显示区分无数据与真零,估算轮不渲染 - 轨迹展开折叠空过程记录行降噪 - token 统计行恒完整可见不截断 - openai_compat 纳入 system 稳定易变拆分 - 系统警示与主题标记改尾部注入不位移前缀 - 工具结果视图压缩按入库时刻冻结 - 会话内工具集冻结仅增不减 - anthropic 缓存补消息级断点,设置页加开关 - 补 intent 测试对 Agent domain 覆盖
This commit is contained in:
@@ -43,8 +43,9 @@ use crate::anthropic_helpers::{
|
||||
/// API 网关——**cache_control 字段可能被非官方网关拒收**，故必须默认关、用户对支持的端点显式开启。
|
||||
///
|
||||
/// 开关开：`convert_request` 把 system 包成 blocks 数组并附加 `cache_control:{type:ephemeral}`
|
||||
/// （配合上层把易变段挪出 system，让 system 稳定段真正命中缓存）。
|
||||
/// 开关关：system 保持纯字符串（现状行为，零风险兼容网关）。
|
||||
/// （配合上层把易变段挪出 system，让 system 稳定段真正命中缓存），
|
||||
/// 并在历史段稳定边界补消息级断点（使 system 之后的对话历史也可命中）。
|
||||
/// 开关关：system 保持纯字符串、消息不打断点（现状行为，零风险兼容网关）。
|
||||
///
|
||||
/// 进程级全局而非 per-provider：上层(AppState settings)按「用户对当前启用的 anthropic 端点
|
||||
/// 是否支持缓存」一次性开启，所有 anthropic 请求一致生效。默认关。
|
||||
@@ -239,6 +240,12 @@ impl AnthropicCompatProvider {
|
||||
// (正常会话首条本就是 user),远优于丢弃上下文。
|
||||
Self::ensure_leading_user(&mut messages);
|
||||
|
||||
// 消息级缓存断点(开关开):除 system 断点外在历史段尾部再打一个 cache_control,
|
||||
// 使 system 之后的对话历史也可命中(否则只缓存 system,长会话历史段全价重算)。
|
||||
if anthropic_cache_enabled() {
|
||||
Self::apply_message_cache_breakpoint(&mut messages);
|
||||
}
|
||||
|
||||
let tools = req.tools.map(|defs| {
|
||||
defs.into_iter()
|
||||
.map(|d| AnthropicToolDef {
|
||||
@@ -316,6 +323,45 @@ impl AnthropicCompatProvider {
|
||||
messages.push(serde_json::json!({ "role": "user", "content": blocks }));
|
||||
}
|
||||
|
||||
/// 消息级缓存断点:在「历史段稳定边界」消息的末个 content block 上打 cache_control。
|
||||
///
|
||||
/// 断点落点取**末条 user 消息之前的相邻前一条**(末条非 user 收尾的异常序列则取末条):
|
||||
/// 末条 user 是每轮请求的漂移边界——缓存拆分模式下它是请求末尾追加的易变尾段
|
||||
/// (日期/清单/augmentation,每次重组),非拆分模式下是本轮新问题,二者都不属于
|
||||
/// 只增历史;打在它身上断点位置随请求漂移,永远无法命中。标其前一条(assistant /
|
||||
/// tool_result 归并的 user)则断点始终落在只增历史的末尾,历史只增不改位:
|
||||
/// 上一轮写入的缓存前缀是本轮请求前缀的逐字节前缀 → 命中后增量续写,滚动前进。
|
||||
///
|
||||
/// content 形态:string(简写)→ 升级为单 text block 数组再标;array(blocks)→ 标末块。
|
||||
/// 断点预算:Anthropic 单请求最多 4 个 cache_control,system 1 个 + 消息级 1 个,余量充足。
|
||||
fn apply_message_cache_breakpoint(messages: &mut [serde_json::Value]) {
|
||||
let n = messages.len();
|
||||
if n == 0 {
|
||||
return;
|
||||
}
|
||||
let last_is_user =
|
||||
messages[n - 1].get("role").and_then(|r| r.as_str()) == Some("user");
|
||||
let idx = if last_is_user && n >= 2 { n - 2 } else { n - 1 };
|
||||
let msg = &mut messages[idx];
|
||||
let cache_control = serde_json::json!({ "type": "ephemeral" });
|
||||
match msg.get("content").cloned() {
|
||||
Some(serde_json::Value::Array(mut blocks)) => {
|
||||
if let Some(last_block) = blocks.last_mut() {
|
||||
last_block["cache_control"] = cache_control;
|
||||
}
|
||||
msg["content"] = serde_json::Value::Array(blocks);
|
||||
}
|
||||
Some(serde_json::Value::String(text)) => {
|
||||
msg["content"] = serde_json::json!([{
|
||||
"type": "text",
|
||||
"text": text,
|
||||
"cache_control": cache_control,
|
||||
}]);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// 合并相邻 user 消息为一条(content 拼成 blocks 数组)。
|
||||
/// Anthropic 协议要求 user/assistant 严格交替,连续 user 触发 1214。
|
||||
/// 触发场景:flush_tool_results 把 tool_result 转 user 后紧跟新 user(drainQueue 续发,
|
||||
@@ -752,6 +798,14 @@ mod tests {
|
||||
use super::*;
|
||||
// apply_anthropic_event / TokenUsage 经 super::*（含 anthropic_helpers::apply_anthropic_event 的 use）可见。
|
||||
|
||||
/// convert_request 读取进程级缓存开关(全局 AtomicBool)。开关切换用例的「开」窗口内,
|
||||
/// 并行运行的普通转换用例会带上缓存形态(system blocks / 消息级断点)致断言互踩。
|
||||
/// 所有走 convert_request 的用例统一先取本锁串行化,消除进程级状态的用例间污染。
|
||||
static CONVERT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
fn convert_lock() -> std::sync::MutexGuard<'static, ()> {
|
||||
CONVERT_LOCK.lock().unwrap_or_else(|p| p.into_inner())
|
||||
}
|
||||
|
||||
/// 辅助：构造 message_start 事件
|
||||
fn message_start(input_tokens: u32) -> String {
|
||||
format!(
|
||||
@@ -945,6 +999,7 @@ mod tests {
|
||||
/// 含图 user 消息 → content blocks（text + image source.base64）
|
||||
#[test]
|
||||
fn anthropic_convert_multimodal_user_blocks() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -980,6 +1035,7 @@ mod tests {
|
||||
/// 纯文本 user 消息 → content 仍是字符串简写（无图不数组化）
|
||||
#[test]
|
||||
fn anthropic_convert_text_only_user_remains_string() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -1001,6 +1057,62 @@ mod tests {
|
||||
assert_eq!(user_msg.get("content").and_then(|c| c.as_str()), Some("hello"));
|
||||
}
|
||||
|
||||
// ---------- 消息级缓存断点(apply_message_cache_breakpoint,纯函数直测) ----------
|
||||
// 注:不开全局 ANTHROPIC_CACHE_ENABLED(convert_request 层开关默认关,已有断言不受扰),
|
||||
// 直接测纯函数落点,避免进程级开关与并行测试互踩。
|
||||
|
||||
/// 末条 user(漂移边界:易变尾段/本轮新问题)不打断点,标其前一条稳定历史消息。
|
||||
#[test]
|
||||
fn cache_breakpoint_marks_message_before_trailing_user() {
|
||||
let mut messages = vec![
|
||||
serde_json::json!({ "role": "user", "content": "q1" }),
|
||||
serde_json::json!({ "role": "assistant", "content": [{"type": "text", "text": "a1"}] }),
|
||||
serde_json::json!({ "role": "user", "content": "q2" }),
|
||||
];
|
||||
AnthropicCompatProvider::apply_message_cache_breakpoint(&mut messages);
|
||||
// 前一条 assistant 的末块带 cache_control
|
||||
let a1 = &messages[1];
|
||||
let blocks = a1["content"].as_array().unwrap();
|
||||
assert_eq!(blocks.last().unwrap()["cache_control"]["type"], "ephemeral");
|
||||
// 末条 user 保持字符串简写,未被打点
|
||||
assert_eq!(messages[2]["content"].as_str(), Some("q2"));
|
||||
}
|
||||
|
||||
/// string content 升级为单 text block 数组并附带 cache_control。
|
||||
#[test]
|
||||
fn cache_breakpoint_upgrades_string_content_to_blocks() {
|
||||
let mut messages = vec![
|
||||
serde_json::json!({ "role": "user", "content": "q1" }),
|
||||
serde_json::json!({ "role": "assistant", "content": "a1" }),
|
||||
serde_json::json!({ "role": "user", "content": "q2" }),
|
||||
];
|
||||
AnthropicCompatProvider::apply_message_cache_breakpoint(&mut messages);
|
||||
let blocks = messages[1]["content"].as_array().expect("string 应升级为 blocks 数组");
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0]["type"], "text");
|
||||
assert_eq!(blocks[0]["text"], "a1");
|
||||
assert_eq!(blocks[0]["cache_control"]["type"], "ephemeral");
|
||||
}
|
||||
|
||||
/// 单条消息(无前一条可标)→ 标自身,不 panic。
|
||||
#[test]
|
||||
fn cache_breakpoint_single_message_marks_itself() {
|
||||
let mut messages = vec![
|
||||
serde_json::json!({ "role": "user", "content": "only" }),
|
||||
];
|
||||
AnthropicCompatProvider::apply_message_cache_breakpoint(&mut messages);
|
||||
let blocks = messages[0]["content"].as_array().expect("单条应升级为 blocks");
|
||||
assert_eq!(blocks[0]["cache_control"]["type"], "ephemeral");
|
||||
}
|
||||
|
||||
/// 空 messages 不 panic。
|
||||
#[test]
|
||||
fn cache_breakpoint_empty_messages_noop() {
|
||||
let mut messages: Vec<serde_json::Value> = vec![];
|
||||
AnthropicCompatProvider::apply_message_cache_breakpoint(&mut messages);
|
||||
assert!(messages.is_empty());
|
||||
}
|
||||
|
||||
// ---------- ensure_leading_user（首条 assistant → 补 user 占位,保留上下文）----------
|
||||
|
||||
/// 辅助:构造 assistant(tool_use) 消息
|
||||
@@ -1016,6 +1128,7 @@ mod tests {
|
||||
/// (原"砍"策略会把每对三元组砍掉,多轮砍到空,丢失全部工具调用历史——"补"策略零丢失。)
|
||||
#[test]
|
||||
fn anthropic_ensure_leading_user_tool_use_chain_preserves_context() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -1054,6 +1167,7 @@ mod tests {
|
||||
/// 首条 assistant 无 tool_use → 补占位,首条 user,原上下文保留。
|
||||
#[test]
|
||||
fn anthropic_ensure_leading_user_plain_assistant() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -1080,6 +1194,7 @@ mod tests {
|
||||
/// 正常序列(user 开头)不补占位——零回归验证。
|
||||
#[test]
|
||||
fn anthropic_ensure_leading_user_normal_sequence_unchanged() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -1107,6 +1222,7 @@ mod tests {
|
||||
/// 补一个 user 占位后全部保留,验证多轮链不丢数据、precheck 通过(原"砍"策略此场景砍到空)。
|
||||
#[test]
|
||||
fn anthropic_ensure_leading_user_three_round_chain() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -1144,6 +1260,7 @@ mod tests {
|
||||
/// 避免发空 messages 触发 precheck "messages 为空"(降级让会话能继续)。
|
||||
#[test]
|
||||
fn anthropic_ensure_leading_user_empty_messages_gets_placeholder() {
|
||||
let _convert = convert_lock();
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
let req = CompletionRequest {
|
||||
model: "claude-3-5-sonnet".into(),
|
||||
@@ -1173,6 +1290,7 @@ mod tests {
|
||||
/// (两态共用进程级全局开关,拆两个用例并行跑会互相污染——单测内顺序切换断言,末尾复位。)
|
||||
#[test]
|
||||
fn anthropic_cache_switch_controls_system_shape() {
|
||||
let _convert = convert_lock();
|
||||
set_anthropic_cache_enabled(false); // 兜底复位,防污染其他并行用例
|
||||
|
||||
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||
|
||||
Reference in New Issue
Block a user