优化: 缓存命中字段解析与提示词前缀稳定化
- openai 兼容口径补解析嵌套 cached_tokens - cache 显示区分无数据与真零,估算轮不渲染 - 轨迹展开折叠空过程记录行降噪 - token 统计行恒完整可见不截断 - openai_compat 纳入 system 稳定易变拆分 - 系统警示与主题标记改尾部注入不位移前缀 - 工具结果视图压缩按入库时刻冻结 - 会话内工具集冻结仅增不减 - anthropic 缓存补消息级断点,设置页加开关 - 补 intent 测试对 Agent domain 覆盖
This commit is contained in:
@@ -97,25 +97,78 @@ pub(crate) struct OpenAiFunctionResp {
|
||||
pub arguments: String,
|
||||
}
|
||||
|
||||
/// OpenAI 标准嵌套缓存口径形态(usage.prompt_tokens_details):
|
||||
/// GLM/Qwen/OpenAI 官方在嵌套结构报缓存命中,DeepSeek 用扁平 prompt_cache_hit_tokens。
|
||||
/// cached_tokens 用 Option 容错给空 details 对象的网关。
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct PromptTokensDetails {
|
||||
#[serde(default)]
|
||||
pub cached_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
/// usage 反序列化中间载体:与 OpenAiUsage 字段同构,经 `From` 做缓存口径归一后产出
|
||||
/// OpenAiUsage(见 impl From)。流式 SSE 与非流式 JSON 两条解析路径因此都自动获得
|
||||
/// 嵌套 cached_tokens 回填,消费方一律读归一后的扁平字段。
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct OpenAiUsageRaw {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
/// DeepSeek 扩展:缓存命中 token(低价,deepseek-chat/reasoner)。无此字段 → default 0。
|
||||
#[serde(default)]
|
||||
pub prompt_cache_hit_tokens: u32,
|
||||
/// DeepSeek 扩展:未命中 token(全价真实输入,prompt_cache_miss_tokens)。无此字段 → default 0。
|
||||
#[serde(default)]
|
||||
pub prompt_cache_miss_tokens: u32,
|
||||
/// DeepSeek-reasoner / OpenAI o1 扩展:思考 token(隐藏输出,reasoning_tokens)。
|
||||
/// 非 reasoning 模型无此字段 → default 0。
|
||||
#[serde(default)]
|
||||
pub reasoning_tokens: u32,
|
||||
/// OpenAI 标准嵌套缓存口径(GLM/Qwen/OpenAI 官方)。
|
||||
#[serde(default)]
|
||||
pub prompt_tokens_details: Option<PromptTokensDetails>,
|
||||
}
|
||||
|
||||
/// OpenAI usage(缓存口径已归一)
|
||||
///
|
||||
/// 归一发生在反序列化时(`#[serde(from)]`):扁平 hit 有值优先(DeepSeek 自报 hit+miss);
|
||||
/// 扁平 hit 未报而嵌套 cached_tokens > 0 时回填 hit,miss 推断为 prompt - hit(嵌套口径
|
||||
/// 缺全价分项);两种口径都没有则保持 0。消费方(apply_openai_sse / openai_compat 非流式)
|
||||
/// 直接读扁平字段,无需感知 provider 差异。
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(from = "OpenAiUsageRaw")]
|
||||
pub(crate) struct OpenAiUsage {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
/// DeepSeek 扩展:缓存命中 token(低价,deepseek-chat/reasoner prompt_cache_hit_tokens)。
|
||||
/// OpenAI 官方(o1 等)无此字段 → serde default 0。其他 OpenAI 兼容网关若支持 cache 也用此名。
|
||||
#[serde(default)]
|
||||
/// 缓存命中 token(低价;扁平优先,缺失时回填自嵌套 cached_tokens)
|
||||
pub prompt_cache_hit_tokens: u32,
|
||||
/// DeepSeek 扩展:未命中 token(全价真实输入,prompt_cache_miss_tokens)。
|
||||
/// OpenAI 官方无此字段 → serde default 0。
|
||||
#[serde(default)]
|
||||
/// 缓存未命中 token(全价真实输入;嵌套口径推断时 = prompt - hit)
|
||||
pub prompt_cache_miss_tokens: u32,
|
||||
/// DeepSeek-reasoner / OpenAI o1 扩展:思考 token(隐藏输出,reasoning_tokens)。
|
||||
/// 非 reasoning 模型无此字段 → serde default 0。
|
||||
#[serde(default)]
|
||||
/// 思考 token(隐藏输出)
|
||||
pub reasoning_tokens: u32,
|
||||
}
|
||||
|
||||
impl From<OpenAiUsageRaw> for OpenAiUsage {
|
||||
fn from(raw: OpenAiUsageRaw) -> Self {
|
||||
let (hit, miss) = match raw.prompt_tokens_details.and_then(|d| d.cached_tokens) {
|
||||
Some(nested) if nested > 0 && raw.prompt_cache_hit_tokens == 0 => {
|
||||
let hit = u32::try_from(nested).unwrap_or(u32::MAX);
|
||||
(hit, raw.prompt_tokens.saturating_sub(hit))
|
||||
}
|
||||
_ => (raw.prompt_cache_hit_tokens, raw.prompt_cache_miss_tokens),
|
||||
};
|
||||
Self {
|
||||
prompt_tokens: raw.prompt_tokens,
|
||||
completion_tokens: raw.completion_tokens,
|
||||
total_tokens: raw.total_tokens,
|
||||
prompt_cache_hit_tokens: hit,
|
||||
prompt_cache_miss_tokens: miss,
|
||||
reasoning_tokens: raw.reasoning_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SSE 流式响应 chunk
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct OpenAiStreamChunk {
|
||||
@@ -287,3 +340,56 @@ pub(crate) fn apply_openai_sse(data: &str, usage_accum: &mut Option<TokenUsage>)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// DeepSeek 扁平口径:hit/miss 各取其值,不参与嵌套回填
|
||||
#[test]
|
||||
fn usage_deepseek_flat_cache() {
|
||||
let u: OpenAiUsage = serde_json::from_str(
|
||||
r#"{"prompt_tokens":100,"completion_tokens":10,"total_tokens":110,
|
||||
"prompt_cache_hit_tokens":64,"prompt_cache_miss_tokens":36}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(u.prompt_tokens, 100);
|
||||
assert_eq!(u.prompt_cache_hit_tokens, 64);
|
||||
assert_eq!(u.prompt_cache_miss_tokens, 36);
|
||||
}
|
||||
|
||||
/// OpenAI/GLM 嵌套口径:扁平未报,hit 回填自 cached_tokens,miss 推断为 prompt - hit
|
||||
#[test]
|
||||
fn usage_nested_cached_tokens_backfill() {
|
||||
let u: OpenAiUsage = serde_json::from_str(
|
||||
r#"{"prompt_tokens":100,"completion_tokens":10,"total_tokens":110,
|
||||
"prompt_tokens_details":{"cached_tokens":64}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(u.prompt_cache_hit_tokens, 64);
|
||||
assert_eq!(u.prompt_cache_miss_tokens, 36);
|
||||
}
|
||||
|
||||
/// 两种口径都没有:hit/miss 保持 0(前端不渲染 cache 段)
|
||||
#[test]
|
||||
fn usage_no_cache_fields() {
|
||||
let u: OpenAiUsage = serde_json::from_str(
|
||||
r#"{"prompt_tokens":100,"completion_tokens":10,"total_tokens":110}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(u.prompt_cache_hit_tokens, 0);
|
||||
assert_eq!(u.prompt_cache_miss_tokens, 0);
|
||||
}
|
||||
|
||||
/// 端到端:usage-only chunk 经 apply_openai_sse 累加路径带出归一后的缓存分项
|
||||
#[test]
|
||||
fn sse_usage_chunk_nested_cache_reaches_token_usage() {
|
||||
let data = r#"{"choices":[],"usage":{"prompt_tokens":100,"completion_tokens":10,
|
||||
"total_tokens":110,"prompt_tokens_details":{"cached_tokens":64}}}"#;
|
||||
let mut accum = None;
|
||||
let chunk = apply_openai_sse(data, &mut accum);
|
||||
let usage = chunk.usage.expect("usage chunk 应携带 usage");
|
||||
assert_eq!(usage.prompt_cache_hit_tokens, 64);
|
||||
assert_eq!(usage.prompt_cache_miss_tokens, 36);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user