优化: 上下文管理与提示词链路批
- T1 usage 口径: anthropic usage 两层容错+cache 三口径归一(prompt=input+read+creation) - T2 压缩水位判据: compress_should_trigger 双模式+ratio 钳位,开关关逐字节等价 - T3 跨会话检索: history_search 工具(两跳式)+message_repo 关键词检索+prompt 指引 - T4 代语义闭环: status 终局补写+飞书停靠/换代推卡+交付引导+watcher 兜底补漏 - 段序缓存重排: build_runtime_prompt 恒静段上移/工作上下文压尾(前缀缓存失效面收窄) - 智谱缓存解析: anthropic 臂补 prompt_tokens_details.cached_tokens 嵌套回退 - 工时禁令: 提示词双语加「禁止预估工时,AI Coding 工时无意义」 - 拖拽文件落盘: 非图片附件存引用行,防整段文本灌入输入框 - 事实修正: 提示词「40+ 工具」→「80+」(基线 81);tool_failure_memory 测试串行锁除竞态 - UI 拆分: TopBar/ToolResultBody 壳层化(top-bar/tool-result-body 子件目录)
This commit is contained in:
@@ -806,12 +806,21 @@ impl LlmProvider for AnthropicCompatProvider {
|
||||
other => warn!(block_type = other, "Anthropic 响应含未知 content 块类型，已忽略"),
|
||||
}
|
||||
}
|
||||
// 口径归一(G1,P0-A/u-claw 三口径,与 anthropic_helpers message_start 同表):
|
||||
// anthropic input_tokens 只含未缓存输入,不含 cache_read/creation 两段;
|
||||
// 落库 prompt_tokens 对齐 openai/deepseek 全量输入语义(hit/miss ⊆ prompt),
|
||||
// 故 prompt = input + read + creation,total = prompt + output。
|
||||
// cache 字段:creation(写入)= miss 全价语义,read(命中)= hit 低价。
|
||||
let prompt_tokens = resp
|
||||
.usage
|
||||
.input_tokens
|
||||
.saturating_add(resp.usage.cache_read())
|
||||
.saturating_add(resp.usage.cache_creation_input_tokens);
|
||||
let usage = TokenUsage {
|
||||
prompt_tokens: resp.usage.input_tokens,
|
||||
prompt_tokens,
|
||||
completion_tokens: resp.usage.output_tokens,
|
||||
total_tokens: resp.usage.input_tokens + resp.usage.output_tokens,
|
||||
// anthropic cache 字段:creation(写入)= miss 全价语义,read(命中)= hit 低价
|
||||
prompt_cache_hit_tokens: resp.usage.cache_read_input_tokens,
|
||||
total_tokens: prompt_tokens.saturating_add(resp.usage.output_tokens),
|
||||
prompt_cache_hit_tokens: resp.usage.cache_read(),
|
||||
prompt_cache_miss_tokens: resp.usage.cache_creation_input_tokens,
|
||||
reasoning_tokens: 0,
|
||||
};
|
||||
@@ -1016,6 +1025,47 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
/// 辅助：构造带缓存分项的 message_start 事件（cache_read/cache_creation，P0-A 三口径）
|
||||
fn message_start_with_cache(input_tokens: u32, cache_read: u32, cache_creation: u32) -> String {
|
||||
format!(
|
||||
r#"{{"type":"message_start","message":{{"usage":{{"input_tokens":{},"cache_read_input_tokens":{},"cache_creation_input_tokens":{},"output_tokens":0}}}}}}"#,
|
||||
input_tokens, cache_read, cache_creation
|
||||
)
|
||||
}
|
||||
|
||||
/// 辅助：构造智谱 anthropic 兼容端点形态的 message_start（隐式缓存命中走嵌套
|
||||
/// prompt_tokens_details.cached_tokens，无顶层 cache 字段——官方缓存文档口径）
|
||||
fn message_start_zhipu_nested(input_tokens: u32, cached_tokens: u32) -> String {
|
||||
format!(
|
||||
r#"{{"type":"message_start","message":{{"usage":{{"input_tokens":{},"prompt_tokens_details":{{"cached_tokens":{}}},"output_tokens":0}}}}}}"#,
|
||||
input_tokens, cached_tokens
|
||||
)
|
||||
}
|
||||
|
||||
/// 智谱嵌套形态回退：顶层 cache 字段缺席时从 prompt_tokens_details.cached_tokens
|
||||
/// 取命中量，prompt=input+cached（隐式缓存无写入分项，miss=0）
|
||||
#[test]
|
||||
fn anthropic_zhipu_nested_cache_tokens_fallback() {
|
||||
let mut acc: Option<TokenUsage> = None;
|
||||
let c = apply_anthropic_event(&message_start_zhipu_nested(500, 4000), &mut acc);
|
||||
let u = c.usage.expect("usage 应随 message_start 带出");
|
||||
assert_eq!(u.prompt_cache_hit_tokens, 4000);
|
||||
assert_eq!(u.prompt_cache_miss_tokens, 0);
|
||||
// 口径归一:prompt = input + cached
|
||||
assert_eq!(u.prompt_tokens, 4500);
|
||||
}
|
||||
|
||||
/// 顶层优先：官方 Anthropic 端点回顶层字段时嵌套值（即便异常并存）不覆盖顶层
|
||||
#[test]
|
||||
fn anthropic_top_level_cache_read_wins_over_nested() {
|
||||
let mut acc: Option<TokenUsage> = None;
|
||||
let frame = r#"{"type":"message_start","message":{"usage":{"input_tokens":100,"cache_read_input_tokens":200,"prompt_tokens_details":{"cached_tokens":999},"output_tokens":0}}}"#;
|
||||
let c = apply_anthropic_event(frame, &mut acc);
|
||||
let u = c.usage.expect("usage 应随 message_start 带出");
|
||||
assert_eq!(u.prompt_cache_hit_tokens, 200);
|
||||
assert_eq!(u.prompt_tokens, 300);
|
||||
}
|
||||
|
||||
/// 辅助：构造 message_delta 事件（output_tokens 为累计值）
|
||||
fn message_delta(output_tokens: u32) -> String {
|
||||
format!(
|
||||
@@ -1135,6 +1185,48 @@ mod tests {
|
||||
assert!(c.usage.is_none(), "无 usage 时 message_stop usage 为 None");
|
||||
}
|
||||
|
||||
/// 三口径映射锁定(G1,P0-A):anthropic input_tokens 不含缓存两段,落库 prompt_tokens
|
||||
/// 对齐 openai/deepseek 全量输入语义(hit/miss ⊆ prompt)→ prompt = input + read +
|
||||
/// creation;message_delta 的 total 重算基于累加器 prompt(已含缓存段)自动继承新口径。
|
||||
#[test]
|
||||
fn anthropic_message_start_cache_three_caliber_prompt_sums_cache_segments() {
|
||||
let mut acc: Option<TokenUsage> = None;
|
||||
let c = apply_anthropic_event(&message_start_with_cache(100, 64, 36), &mut acc);
|
||||
let u0 = c.usage.expect("message_start 应带出初始 usage 快照");
|
||||
assert_eq!(u0.prompt_tokens, 200, "prompt = input(100)+read(64)+creation(36)");
|
||||
assert_eq!(u0.prompt_cache_hit_tokens, 64, "cache_read → hit 原样映射");
|
||||
assert_eq!(u0.prompt_cache_miss_tokens, 36, "cache_creation → miss 原样映射");
|
||||
assert_eq!(u0.total_tokens, 200, "初始 total 同口径");
|
||||
|
||||
// message_delta 累计 output 后 total 重算:继承含缓存段 prompt 基数(200+30)
|
||||
let _ = apply_anthropic_event(&message_delta(30), &mut acc);
|
||||
let a = acc.as_ref().unwrap();
|
||||
assert_eq!(a.prompt_tokens, 200, "delta 不改 prompt");
|
||||
assert_eq!(a.total_tokens, 230, "total = prompt(含缓存两段)+output");
|
||||
}
|
||||
|
||||
/// 缺失容错(G2,P0-A):端点 200 但 usage 键整体缺失/usage 对象缺计数字段时,
|
||||
/// 不再整帧反序列化失败硬炸(三路走查 #3「有效文本回复整体丢弃」),全零兜底
|
||||
/// 下行走既有估算轨(round_usage_effective prompt==0 → 估算打标),对齐 OpenAI 臂。
|
||||
#[test]
|
||||
fn anthropic_response_missing_or_partial_usage_deserializes_not_fatal() {
|
||||
// usage 键整体缺失
|
||||
let resp: AnthropicResponse = serde_json::from_str(
|
||||
r#"{"id":"msg_1","model":"claude-x","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn"}"#,
|
||||
)
|
||||
.expect("usage 键缺失不应炸帧");
|
||||
assert_eq!(resp.usage.input_tokens, 0, "缺 usage → 全零兜底走估算轨");
|
||||
assert_eq!(resp.usage.output_tokens, 0);
|
||||
|
||||
// usage 对象存在但缺计数字段(部分网关只回 cache 分项)
|
||||
let resp2: AnthropicResponse = serde_json::from_str(
|
||||
r#"{"id":"msg_2","model":"claude-x","content":[],"usage":{"cache_read_input_tokens":7}}"#,
|
||||
)
|
||||
.expect("usage 缺 input/output 字段不应炸帧");
|
||||
assert_eq!(resp2.usage.input_tokens, 0);
|
||||
assert_eq!(resp2.usage.cache_read_input_tokens, 7, "已有分项不丢");
|
||||
}
|
||||
|
||||
/// content_block_start (tool_use) 带 id+name
|
||||
#[test]
|
||||
fn anthropic_content_block_start_tool_use() {
|
||||
|
||||
@@ -69,6 +69,10 @@ pub(crate) struct AnthropicResponse {
|
||||
// Anthropic 响应反序列化字段,保留以备调试/未来消费(如日志记录调用终止原因),标注意图消除 dead_code warning
|
||||
#[allow(dead_code)]
|
||||
stop_reason: Option<String>,
|
||||
/// usage 整段 default(G2):端点不回 usage 键时不再 missing-field 炸帧,全零兜底
|
||||
/// 下行走既有估算轨(round_usage_effective)。字段级容错在 AnthropicUsage 侧,
|
||||
/// 此处兜「usage 键整体缺失」——两层合起来才覆盖「端点不回 usage」全形态。
|
||||
#[serde(default)]
|
||||
pub usage: AnthropicUsage,
|
||||
}
|
||||
|
||||
@@ -88,9 +92,15 @@ pub(crate) struct AnthropicContentBlock {
|
||||
pub input: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct AnthropicUsage {
|
||||
/// 计数字段全部 default 容错(G2,P0-A 口径统一):部分网关 200 但 usage 缺失/缺字段,
|
||||
/// 必填会让整帧反序列化失败 → 有效回复整体丢弃(三路走查 #3/#5)。缺省 0 后全零
|
||||
/// TokenUsage 下行走既有估算兜底轨(round_usage_effective,prompt==0 → 估算打标),
|
||||
/// 与 OpenAI 臂的 serde default 容错(openai_helpers.rs:139-157)对齐。
|
||||
#[serde(default)]
|
||||
pub input_tokens: u32,
|
||||
#[serde(default)]
|
||||
pub output_tokens: u32,
|
||||
/// Anthropic prompt caching 扩展:cache 写入 token(本次写入缓存,计费如输入但稍便宜)。
|
||||
/// 映射到 TokenUsage.prompt_cache_miss_tokens(全价输入语义)。
|
||||
@@ -102,6 +112,32 @@ pub(crate) struct AnthropicUsage {
|
||||
/// 非 cache 场景无此字段 → serde default 0。
|
||||
#[serde(default)]
|
||||
pub cache_read_input_tokens: u32,
|
||||
/// 智谱 anthropic 兼容端点的隐式缓存(自动生效,命中半价)不回顶层 cache 字段,
|
||||
/// 命中量走 OpenAI 风格嵌套 prompt_tokens_details.cached_tokens(智谱官方缓存文档
|
||||
/// 口径)。serde 容错读入,消费侧经 [`Self::cache_read`] 回退取用——缺此层时
|
||||
/// GLM 链路账本/命中率恒 0(实测 54 条观测全零的根因)。
|
||||
#[serde(default)]
|
||||
pub prompt_tokens_details: Option<PromptTokensDetails>,
|
||||
}
|
||||
|
||||
/// OpenAI 风格嵌套的缓存命中分项(智谱 anthropic 兼容端点口径)。
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct PromptTokensDetails {
|
||||
#[serde(default)]
|
||||
pub cached_tokens: u32,
|
||||
}
|
||||
|
||||
impl AnthropicUsage {
|
||||
/// 缓存命中读取量:顶层 cache_read_input_tokens 非零优先(官方 Anthropic 端点),
|
||||
/// 智谱嵌套形态(顶层缺省 0)回退;两处都不存在时 0。
|
||||
pub fn cache_read(&self) -> u32 {
|
||||
if self.cache_read_input_tokens > 0 {
|
||||
return self.cache_read_input_tokens;
|
||||
}
|
||||
self.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map_or(0, |d| d.cached_tokens)
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
@@ -155,15 +191,30 @@ pub(crate) fn apply_anthropic_event(data: &str, usage_accum: &mut Option<TokenUs
|
||||
let cache_read = u_obj
|
||||
.and_then(|u| u.get("cache_read_input_tokens"))
|
||||
.and_then(|t| t.as_u64())
|
||||
// 智谱 anthropic 兼容端点隐式缓存走嵌套 prompt_tokens_details.cached_tokens,
|
||||
// 顶层字段缺席时回退读取(与非流式 AnthropicUsage::cache_read 同表)。
|
||||
.or_else(|| {
|
||||
u_obj
|
||||
.and_then(|u| u.get("prompt_tokens_details"))
|
||||
.and_then(|d| d.get("cached_tokens"))
|
||||
.and_then(|t| t.as_u64())
|
||||
})
|
||||
.unwrap_or(0) as u32;
|
||||
let cache_creation = u_obj
|
||||
.and_then(|u| u.get("cache_creation_input_tokens"))
|
||||
.and_then(|t| t.as_u64())
|
||||
.unwrap_or(0) as u32;
|
||||
// 口径归一(G1,P0-A/u-claw 三口径): anthropic input_tokens 只含未缓存输入,
|
||||
// 不含 cache_read/cache_creation 两段;落库 prompt_tokens 对齐 openai/deepseek
|
||||
// 全量输入语义(prompt=全量输入,hit/miss 为其子集),故 prompt = input + read
|
||||
// + creation。total 同口径(prompt + 当前 output)。
|
||||
let prompt = (inp as u32)
|
||||
.saturating_add(cache_read)
|
||||
.saturating_add(cache_creation);
|
||||
*usage_accum = Some(TokenUsage {
|
||||
prompt_tokens: inp as u32,
|
||||
prompt_tokens: prompt,
|
||||
completion_tokens: 0,
|
||||
total_tokens: inp as u32,
|
||||
total_tokens: prompt,
|
||||
prompt_cache_hit_tokens: cache_read,
|
||||
prompt_cache_miss_tokens: cache_creation,
|
||||
reasoning_tokens: 0,
|
||||
|
||||
@@ -705,8 +705,16 @@ impl AiConversationRepo {
|
||||
///
|
||||
/// A2-B9(G3.2 clearChat 裁决):原 `clear_messages` + `delete_range` 两条独立 DB 写非原子,
|
||||
/// DB 失败会致 messages JSON 列与 ai_messages 表不一致(如仅一条成功)。本方法一次 transaction
|
||||
/// 覆盖两条写(① UPDATE ai_conversations 置空消息 + 清零 token;② DELETE ai_messages 该 conv
|
||||
/// 全部行),成功全成功 / 失败回滚全失败。供 `ai_chat_clear` 先停 loop 再单事务清空。
|
||||
/// 覆盖两条写(① UPDATE ai_conversations 置空消息 + 清零 token + 生命周期标记归位;
|
||||
/// ② DELETE ai_messages 该 conv 全部行),成功全成功 / 失败回滚全失败。供 `ai_chat_clear`
|
||||
/// 先停 loop 再单事务清空。
|
||||
///
|
||||
/// 生命周期标记归位(P1-B 方案甲代史收尾,2026-10-21):硬清空=被清代的终局,对话壳虽
|
||||
/// 保留但属全新一代,旧代终局语义必须随事务归位防泄漏给同壳新任务——loop_state→NULL
|
||||
/// (被清的 parked 会话不得再进 list_parked 可恢复清单)、status→'active'(旧 completed/
|
||||
/// incomplete/error 不再代表新代)、boundary_json→NULL(V45「随会话同亡」:旧任务边界
|
||||
/// 不得约束同壳新任务;内存侧 per_conv.boundary 由调用方同步清,防下轮 save 随行覆写
|
||||
/// 复活 DB 侧刚清的快照)。
|
||||
///
|
||||
/// 对话壳保留(侧栏仍可见,可继续在该对话内聊);返回 Ok(())——调用方只关心成功与否
|
||||
/// (对齐 replace_conversation 语义,不返回受影响行数)。
|
||||
@@ -718,9 +726,9 @@ impl AiConversationRepo {
|
||||
let mut guard = conn.blocking_lock();
|
||||
let tx = guard.transaction().map_err(storage_err)?;
|
||||
{
|
||||
// ① ai_conversations.messages 置空 + token 清零(对话壳保留)
|
||||
// ① ai_conversations.messages 置空 + token 清零 + 生命周期标记归位(对话壳保留)
|
||||
tx.execute(
|
||||
"UPDATE ai_conversations SET messages = '[]', prompt_tokens = 0, completion_tokens = 0, updated_at = ?1 WHERE id = ?2",
|
||||
"UPDATE ai_conversations SET messages = '[]', prompt_tokens = 0, completion_tokens = 0, loop_state = NULL, status = 'active', boundary_json = NULL, updated_at = ?1 WHERE id = ?2",
|
||||
params![now, id],
|
||||
)
|
||||
.map_err(storage_err)?;
|
||||
@@ -1244,6 +1252,47 @@ mod tests {
|
||||
assert_eq!(got2.status, "incomplete", "写 incomplete 应可读回");
|
||||
}
|
||||
|
||||
/// P1-B 方案甲代史收尾契约(2026-10-21):硬清空=被清代的终局,生命周期标记必须随
|
||||
/// 同一事务归位——loop_state→NULL(被清的 parked 会话不得再进可恢复清单)、status→
|
||||
/// 'active'(旧终局态不得泄漏给同壳新任务)、boundary_json→NULL(旧任务边界随代同亡,
|
||||
/// 不得约束同壳新任务)。锁死 clear_conversation_atomic 的收尾语义(调用方 ai_chat_clear)。
|
||||
#[tokio::test]
|
||||
async fn clear_conversation_atomic_resets_lifecycle_markers() {
|
||||
let db = Database::open_in_memory().await.expect("open_in_memory");
|
||||
let repo = AiConversationRepo::new(&db);
|
||||
// 前置态:带旧边界 + parked + incomplete 的非空会话(清空前的典型停靠残留)
|
||||
repo.insert(AiConversationRecord {
|
||||
id: "c1".into(),
|
||||
title: None,
|
||||
messages: r#"[{"role":"user","content":"旧任务"}]"#.into(),
|
||||
provider_id: None,
|
||||
model: None,
|
||||
models: None,
|
||||
archived: false,
|
||||
pinned: false,
|
||||
prompt_tokens: Some(120),
|
||||
completion_tokens: Some(80),
|
||||
pinned_goals: None,
|
||||
pending_approvals: None,
|
||||
created_at: "0".into(),
|
||||
updated_at: "0".into(),
|
||||
origin: None,
|
||||
boundary_json: Some(r#"{"risk_ceiling":"Medium"}"#.into()),
|
||||
loop_state: Some("parked".into()),
|
||||
status: "incomplete".into(),
|
||||
working_context_json: None,
|
||||
})
|
||||
.await
|
||||
.expect("insert");
|
||||
|
||||
repo.clear_conversation_atomic("c1").await.expect("clear");
|
||||
let got = repo.get_by_id("c1").await.expect("get").expect("row");
|
||||
assert_eq!(got.loop_state, None, "清空后 loop_state 应归位 NULL(停靠残留不得进可恢复清单)");
|
||||
assert_eq!(got.status, "active", "清空后 status 应归位 active(旧代终局不得泄漏)");
|
||||
assert_eq!(got.boundary_json, None, "清空后 boundary_json 应随代同亡");
|
||||
assert_eq!(got.messages, "[]", "消息内容照旧清空");
|
||||
}
|
||||
|
||||
/// V53 契约:working_context_json 经 insert → get_by_id → update_full 全程原样存取
|
||||
/// (存储层按不透明 TEXT,结构解析归调用方);老库行(缺列插入)读回 None 不炸。
|
||||
#[tokio::test]
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
//! P0 阶段(本文件):仅建 Repo + CRUD 方法,不接 ContextManager 读写路径
|
||||
//! (切读策略待用户决策,P2)。ContextManager 仍走旧 messages JSON 列双写期未启动。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rusqlite::{params, Connection, Row};
|
||||
@@ -20,6 +21,7 @@ use df_types::error::Result;
|
||||
use crate::db::Database;
|
||||
use crate::models::AiMessageRecord;
|
||||
|
||||
use super::idea_repo::escape_like;
|
||||
use super::storage_err;
|
||||
|
||||
// ============================================================
|
||||
@@ -51,6 +53,72 @@ fn ai_message_from_row(row: &Row<'_>) -> std::result::Result<AiMessageRecord, ru
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 跨会话关键词检索(P1-A,对标 u-claw #5/#6)
|
||||
// ============================================================
|
||||
|
||||
/// LIKE 检索不做通配模糊,关键词为分词产物(调用方 src-tauri 层 extract_query_keywords
|
||||
/// 分好传入,本层不做分词——df-storage 在依赖下层,反向依赖 src-tauri 会成环)。
|
||||
///
|
||||
/// **FTS5 升级触发条件(2026-10-21 拍板留档)**:ai_messages 行数 > 50 万,或 role 过滤后
|
||||
/// LIKE 检索 P95 实测 > 500ms 时,再评估升级 FTS5 虚表(V54+ 建虚表 + 一次性回填,
|
||||
/// 检索接口签名不变)。当前生产库实测(32,349 行 / content 39.5MB):role 过滤后单关键词
|
||||
/// LIKE 113ms、双关键词 127ms,桌面场景百毫秒级无体感;且中文 unicode61 按字切分召回
|
||||
/// 质量反不如子串 LIKE,仓内 idea_repo/conversation_repo 两处既有「不上 FTS5」决策先例。
|
||||
|
||||
/// 关键词检索单条命中。
|
||||
///
|
||||
/// `snippet` 为命中片段:content 中首个关键词命中点前后各约 200 字,整体截 500 字
|
||||
/// (截断处以 … 标记),避免全文回灌撑爆调用方上下文。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MessageSearchHit {
|
||||
pub message_id: String,
|
||||
pub conversation_id: String,
|
||||
pub seq: i64,
|
||||
pub role: String,
|
||||
pub timestamp: Option<i64>,
|
||||
pub snippet: String,
|
||||
}
|
||||
|
||||
/// 会话摘要降级链数据源(无参列会话时逐会话取用)。
|
||||
///
|
||||
/// `last_system` = 该会话最近一条 role=system 消息(压缩摘要/关键词锚点的载体,
|
||||
/// 四段式「## 意图/## 决策…」);`first_user` = 首条 user 消息。调用方按
|
||||
/// 「最近 system 摘要 > 首条 user」降级链取会话概要,两级皆空(空会话)则无摘要。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ConversationSummarySource {
|
||||
pub last_system: Option<String>,
|
||||
pub first_user: Option<String>,
|
||||
}
|
||||
|
||||
/// 命中片段窗口:命中点前保留上下文与片段总长(按字符计,中文一字一符)。
|
||||
const SNIPPET_CONTEXT_CHARS: usize = 200;
|
||||
const SNIPPET_MAX_CHARS: usize = 500;
|
||||
|
||||
/// 从 content 提取命中片段:首个关键词命中点前后各约 200 字,整体截 500 字。
|
||||
///
|
||||
/// 大小写不敏感定位(与 SQLite LIKE 的 ASCII 大小写不敏感对齐);分词产物在 LIKE 层
|
||||
/// 已命中,理论必有命中点,兜底未找到时退回首部截断。全程按字符迭代操作,无字节
|
||||
/// 索引切边界的乱码风险。
|
||||
fn search_snippet(content: &str, keywords: &[String]) -> String {
|
||||
let ellipsis = |s: &str, head_cut: bool, tail_cut: bool| {
|
||||
format!("{}{}{}", if head_cut { "…" } else { "" }, s, if tail_cut { "…" } else { "" })
|
||||
};
|
||||
let lower = content.to_lowercase();
|
||||
let hit_char = keywords.iter().find_map(|kw| lower.find(&kw.to_lowercase())).map(|byte| {
|
||||
// 命中字节位 → 字符位(to_lowercase 极端 Unicode 下字符数可能漂移,
|
||||
// 但仅影响窗口中心偏移,chars().skip 的迭代本身无越界风险)
|
||||
lower[..byte].chars().count()
|
||||
});
|
||||
let Some(hit_char) = hit_char else {
|
||||
let head: String = content.chars().take(SNIPPET_MAX_CHARS).collect();
|
||||
return ellipsis(&head, false, content.chars().count() > SNIPPET_MAX_CHARS);
|
||||
};
|
||||
let start = hit_char.saturating_sub(SNIPPET_CONTEXT_CHARS);
|
||||
let body: String = content.chars().skip(start).take(SNIPPET_MAX_CHARS).collect();
|
||||
ellipsis(&body, start > 0, content.chars().count() > start + SNIPPET_MAX_CHARS)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// AiMessageRepo
|
||||
// ============================================================
|
||||
@@ -375,6 +443,156 @@ impl AiMessageRepo {
|
||||
.await
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 跨会话关键词检索(P1-A):LIKE OR 逐词命中任一即入候选,timestamp DESC,limit/offset 分页。
|
||||
///
|
||||
/// - 分词在调用方(src-tauri 层 extract_query_keywords)完成,本层只收关键词数组,
|
||||
/// 不反向依赖 src-tauri(依赖方向铁律,对齐 KnowledgeRepo.search_keywords_scoped 先例);
|
||||
/// - role 恒过滤 user+assistant+system(拍板④:tool 结果占内容量 86% 且噪音大,默认排除);
|
||||
/// - `conversation_id` 传入时限定单会话(两跳式深查第二跳);
|
||||
/// - LIKE 转义与 `ESCAPE '|'` 复用 [`super::idea_repo::escape_like`] 单一实现,
|
||||
/// 关键词含 %/_/| 时按字面匹配;
|
||||
/// - 返回带命中片段(500 字截断,见 [`search_snippet`]),全文不出存储层;
|
||||
/// - status 不过滤:compressed 消息原文完整保留(ai_messages = 全量账本),照常可检索。
|
||||
///
|
||||
/// FTS5 升级触发条件见模块顶部注释(留档,>50 万行或 P95>500ms 再评估)。
|
||||
pub async fn search_keywords(
|
||||
&self,
|
||||
keywords: &[String],
|
||||
conversation_id: Option<&str>,
|
||||
limit: usize,
|
||||
offset: usize,
|
||||
) -> Result<Vec<MessageSearchHit>> {
|
||||
if keywords.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.clone();
|
||||
let conv_id = conversation_id.map(|s| s.to_owned());
|
||||
// 每个关键词占一个编号占位符;conversation_id 占 ?(n+1),LIMIT/OFFSET 依次后推。
|
||||
// 动态 WHERE 拼接模式对齐 AiToolExecutionRepo::list_by_query 先例。
|
||||
let mut patterns: Vec<String> = Vec::with_capacity(keywords.len());
|
||||
let mut where_parts: Vec<String> = Vec::with_capacity(keywords.len());
|
||||
for (i, kw) in keywords.iter().enumerate() {
|
||||
let n = i + 1;
|
||||
patterns.push(format!("%{}%", escape_like(kw)));
|
||||
where_parts.push(format!("content LIKE ?{n} ESCAPE '|'"));
|
||||
}
|
||||
let mut extra: Vec<String> = Vec::new();
|
||||
if let Some(c) = &conv_id {
|
||||
extra.push(c.clone());
|
||||
}
|
||||
// 编号必须连续(SQLite 参数计数取最大 ?N,跳号会产生未绑定空洞报错):
|
||||
// 无 conv 条件时 LIMIT/OFFSET 紧跟关键词占位符之后。
|
||||
let conv_n = patterns.len() + 1;
|
||||
let (limit_n, offset_n) = if conv_id.is_some() {
|
||||
(conv_n + 1, conv_n + 2)
|
||||
} else {
|
||||
(conv_n, conv_n + 1)
|
||||
};
|
||||
let limit_i = limit as i64;
|
||||
let offset_i = offset as i64;
|
||||
let conv_clause = if conv_id.is_some() {
|
||||
format!(" AND conversation_id = ?{conv_n}")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let sql = format!(
|
||||
"SELECT id, conversation_id, seq, role, timestamp, content FROM ai_messages \
|
||||
WHERE role IN ('user','assistant','system'){conv_clause} AND ({}) \
|
||||
ORDER BY timestamp DESC LIMIT ?{limit_n} OFFSET ?{offset_n}",
|
||||
where_parts.join(" OR "),
|
||||
);
|
||||
let keywords_owned: Vec<String> = keywords.to_vec();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = conn.blocking_lock();
|
||||
let mut stmt = guard.prepare(&sql).map_err(storage_err)?;
|
||||
// 参数序列:关键词 pattern × n [+ conv_id] + limit + offset
|
||||
let mut param_refs: Vec<&dyn rusqlite::ToSql> =
|
||||
patterns.iter().map(|p| p as &dyn rusqlite::ToSql).collect();
|
||||
for e in &extra {
|
||||
param_refs.push(e);
|
||||
}
|
||||
param_refs.push(&limit_i);
|
||||
param_refs.push(&offset_i);
|
||||
let rows = stmt
|
||||
.query_map(param_refs.as_slice(), |row| {
|
||||
let content: String = row.get("content")?;
|
||||
Ok(MessageSearchHit {
|
||||
message_id: row.get("id")?,
|
||||
conversation_id: row.get("conversation_id")?,
|
||||
seq: row.get("seq")?,
|
||||
role: row.get("role")?,
|
||||
timestamp: row.get("timestamp")?,
|
||||
snippet: search_snippet(&content, &keywords_owned),
|
||||
})
|
||||
})
|
||||
.map_err(storage_err)?;
|
||||
let mut results = Vec::new();
|
||||
for r in rows {
|
||||
results.push(r.map_err(storage_err)?);
|
||||
}
|
||||
Ok(results)
|
||||
})
|
||||
.await
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 会话摘要降级链数据源(无参列最近会话用):一批会话各取「最近一条 system 消息」与
|
||||
/// 「首条 user 消息」content,逐会话单行返回(无消息的会话无行,调用方按无摘要兜底)。
|
||||
///
|
||||
/// 关联子查询走 idx_ai_messages_conv(conversation_id, seq) 索引,批量一次查询替代
|
||||
/// 逐会话 list_by_conversation 全量物化(276 条/会话均值,10 会话省 ~340KB 物化)。
|
||||
pub async fn summary_sources(
|
||||
&self,
|
||||
conversation_ids: &[String],
|
||||
) -> Result<HashMap<String, ConversationSummarySource>> {
|
||||
if conversation_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
let conn = self.conn.clone();
|
||||
// IN 占位符逐个编号(?1..?n)
|
||||
let placeholders: Vec<String> =
|
||||
(1..=conversation_ids.len()).map(|n| format!("?{n}")).collect();
|
||||
let sql = format!(
|
||||
"SELECT m.conversation_id, \
|
||||
(SELECT s.content FROM ai_messages s \
|
||||
WHERE s.conversation_id = m.conversation_id AND s.role = 'system' \
|
||||
ORDER BY s.seq DESC LIMIT 1) AS last_system, \
|
||||
(SELECT u.content FROM ai_messages u \
|
||||
WHERE u.conversation_id = m.conversation_id AND u.role = 'user' \
|
||||
ORDER BY u.seq ASC LIMIT 1) AS first_user \
|
||||
FROM ai_messages m \
|
||||
WHERE m.conversation_id IN ({}) \
|
||||
GROUP BY m.conversation_id",
|
||||
placeholders.join(","),
|
||||
);
|
||||
let ids: Vec<String> = conversation_ids.to_vec();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = conn.blocking_lock();
|
||||
let mut stmt = guard.prepare(&sql).map_err(storage_err)?;
|
||||
let param_refs: Vec<&dyn rusqlite::ToSql> =
|
||||
ids.iter().map(|s| s as &dyn rusqlite::ToSql).collect();
|
||||
let rows = stmt
|
||||
.query_map(param_refs.as_slice(), |row| {
|
||||
Ok((
|
||||
row.get::<_, String>("conversation_id")?,
|
||||
ConversationSummarySource {
|
||||
last_system: row.get("last_system")?,
|
||||
first_user: row.get("first_user")?,
|
||||
},
|
||||
))
|
||||
})
|
||||
.map_err(storage_err)?;
|
||||
let mut out = HashMap::new();
|
||||
for r in rows {
|
||||
let (id, source) = r.map_err(storage_err)?;
|
||||
out.insert(id, source);
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
.await
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
@@ -801,4 +1019,137 @@ mod tests {
|
||||
assert_eq!(got3.len(), 1);
|
||||
assert_eq!(got3[0].seq, 1, "limit=1 应取 seq 最大(最新)一条");
|
||||
}
|
||||
|
||||
// ---------- search_keywords(P1-A 跨会话检索) ----------
|
||||
|
||||
/// role 过滤(tool 默认排除)+ timestamp DESC 排序 + conversation_id 限定深查。
|
||||
#[tokio::test]
|
||||
async fn search_keywords_filters_role_and_orders_desc() {
|
||||
let db = Database::open_in_memory().await.expect("open_in_memory");
|
||||
let repo = AiMessageRepo::new(&db);
|
||||
let mut u0 = mk_msg("u0", "conv_a", 0, "user", "怎么部署服务");
|
||||
u0.timestamp = Some(1000);
|
||||
let mut a0 = mk_msg("a0", "conv_a", 1, "assistant", "部署 分三步走");
|
||||
a0.timestamp = Some(2000);
|
||||
let mut hit_tool = mk_msg("t0", "conv_a", 2, "tool", "部署流程的 tool 结果");
|
||||
hit_tool.timestamp = Some(3000);
|
||||
let mut u1 = mk_msg("u1", "conv_b", 0, "user", "旧会话也提过部署");
|
||||
u1.timestamp = Some(500);
|
||||
repo.insert_batch(vec![u0, a0, hit_tool, u1])
|
||||
.await
|
||||
.expect("insert");
|
||||
|
||||
// tool role 默认排除:命中 3 条(两 conv 的 user/assistant),tool 结果不入检索面
|
||||
let hits = repo
|
||||
.search_keywords(&["部署".to_string()], None, 20, 0)
|
||||
.await
|
||||
.expect("search");
|
||||
assert_eq!(hits.len(), 3, "tool role 应被排除: {:?}", hits.iter().map(|h| &h.role).collect::<Vec<_>>());
|
||||
// timestamp DESC:conv_a(3000 之外的 2000/1000 应在 conv_b 100 之前)
|
||||
assert_eq!(hits[0].message_id, "a0");
|
||||
assert_eq!(hits[1].message_id, "u0");
|
||||
|
||||
// conversation_id 限定:只回该会话命中(两跳式深查第二跳)
|
||||
let scoped = repo
|
||||
.search_keywords(&["部署".to_string()], Some("conv_b"), 20, 0)
|
||||
.await
|
||||
.expect("search scoped");
|
||||
assert_eq!(scoped.len(), 1);
|
||||
assert_eq!(scoped[0].conversation_id, "conv_b");
|
||||
|
||||
// offset 分页:跳过第一条
|
||||
let paged = repo
|
||||
.search_keywords(&["部署".to_string()], None, 20, 1)
|
||||
.await
|
||||
.expect("search paged");
|
||||
assert_eq!(paged.len(), 2);
|
||||
assert_eq!(paged[0].message_id, "u0");
|
||||
}
|
||||
|
||||
/// LIKE 通配符按字面匹配(%/_ 不扩大命中面)+ 命中片段以关键词为中心截 500 字。
|
||||
#[tokio::test]
|
||||
async fn search_keywords_escapes_wildcards_and_centers_snippet() {
|
||||
let db = Database::open_in_memory().await.expect("open_in_memory");
|
||||
let repo = AiMessageRepo::new(&db);
|
||||
let long_prefix = "前".repeat(600); // 命中点前 600 字:窗口应前推 200 字且带 …
|
||||
let long_tail = "后".repeat(600);
|
||||
repo.insert_batch(vec![
|
||||
mk_msg("m0", "c", 0, "user", &format!("{long_prefix}配额100%封顶{long_tail}")),
|
||||
mk_msg("m1", "c", 1, "user", "配额100x封顶,百分号不应误命中"),
|
||||
mk_msg("m2", "c", 2, "user", "变量名 quota_limit 带下划线"),
|
||||
mk_msg("m3", "c", 3, "user", "quotaXlimit 下划线不该当通配命中"),
|
||||
])
|
||||
.await
|
||||
.expect("insert");
|
||||
|
||||
// % 按字面:只命中 m0,不命中 100x
|
||||
let pct = repo
|
||||
.search_keywords(&["100%".to_string()], None, 20, 0)
|
||||
.await
|
||||
.expect("search %");
|
||||
assert_eq!(pct.len(), 1, "100% 不应通配命中 100x: {:?}", pct.iter().map(|h| &h.message_id).collect::<Vec<_>>());
|
||||
assert_eq!(pct[0].message_id, "m0");
|
||||
|
||||
// _ 按字面:只命中 quota_limit,不命中 quotaXlimit
|
||||
let under = repo
|
||||
.search_keywords(&["quota_limit".to_string()], None, 20, 0)
|
||||
.await
|
||||
.expect("search _");
|
||||
assert_eq!(under.len(), 1);
|
||||
assert_eq!(under[0].message_id, "m2");
|
||||
|
||||
// 片段:命中点前约 200 字,总长 500+省略号,两端截断带 …
|
||||
let snip = &pct[0].snippet;
|
||||
assert!(snip.contains('…'), "两端截断应带省略号: {snip}");
|
||||
assert!(snip.contains("100%封顶"), "片段应以命中点为中心: {snip}");
|
||||
let body = snip.trim_matches('…');
|
||||
assert!(
|
||||
body.chars().count() <= 500,
|
||||
"片段正文应截 500 字,实际 {}",
|
||||
body.chars().count()
|
||||
);
|
||||
assert!(snip.starts_with('…'), "命中点前 600 字,片段应带前省略号");
|
||||
}
|
||||
|
||||
/// 空关键词防御:直返空集不发 SQL(LIKE %% 恒真会全表命中)。
|
||||
#[tokio::test]
|
||||
async fn search_keywords_empty_keywords_returns_empty() {
|
||||
let db = Database::open_in_memory().await.expect("open_in_memory");
|
||||
let repo = AiMessageRepo::new(&db);
|
||||
repo.insert_batch(vec![mk_msg("m0", "c", 0, "user", "任意内容")])
|
||||
.await
|
||||
.expect("insert");
|
||||
let hits = repo.search_keywords(&[], None, 20, 0).await.expect("search empty");
|
||||
assert!(hits.is_empty(), "空关键词应返空集");
|
||||
}
|
||||
|
||||
// ---------- summary_sources(摘要降级链数据源) ----------
|
||||
|
||||
/// 每会话取「最近一条 system」与「首条 user」;无消息会话无行;批量一次查询。
|
||||
#[tokio::test]
|
||||
async fn summary_sources_takes_last_system_and_first_user() {
|
||||
let db = Database::open_in_memory().await.expect("open_in_memory");
|
||||
let repo = AiMessageRepo::new(&db);
|
||||
repo.insert_batch(vec![
|
||||
mk_msg("u0", "c1", 0, "user", "首个问题"),
|
||||
mk_msg("a0", "c1", 1, "assistant", "回复"),
|
||||
mk_msg("s0", "c1", 2, "system", "旧锚点"),
|
||||
mk_msg("s1", "c1", 3, "system", "## 意图\n- 最新压缩摘要"),
|
||||
mk_msg("u1", "c2", 0, "user", "c2 的唯一用户消息"),
|
||||
])
|
||||
.await
|
||||
.expect("insert");
|
||||
|
||||
let out = repo
|
||||
.summary_sources(&["c1".to_string(), "c2".to_string(), "c3".to_string()])
|
||||
.await
|
||||
.expect("summary_sources");
|
||||
assert_eq!(out.len(), 2, "无消息的 c3 不应有行");
|
||||
let c1 = &out["c1"];
|
||||
assert_eq!(c1.last_system.as_deref(), Some("## 意图\n- 最新压缩摘要"), "应取 seq 最大的 system");
|
||||
assert_eq!(c1.first_user.as_deref(), Some("首个问题"));
|
||||
let c2 = &out["c2"];
|
||||
assert!(c2.last_system.is_none(), "无 system 消息应返 None(降级链走首条 user)");
|
||||
assert_eq!(c2.first_user.as_deref(), Some("c2 的唯一用户消息"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user