新增: 知识库检索分词化与晋升配置化及对话工具
整句 LIKE 命中率趋零,注入检索改 CJK 2-gram+ASCII 词分词 OR 匹配,标题命中计分排序 停用词剥离前置到切分前,滤掉跨词组合噪音 gram;ASCII 等频精确词优先 candidate 纳入检索面(include_candidates 默认开,配开关可回仅 published) 复用即信任自动晋升配置化:默认关+阈值 5,等晋升把关策略拍板后再开,防未审核产物经误命中固化污染 晋升留痕进时间线(auto/agent 来源区分);补 candidate 复用达标自动转正机制 agent 对话侧新增 knowledge_search/insert/promote 三工具,insert 固定 candidate,转正仅限 candidate 单向 知识配置结构新增两字段带 serde 默认,旧配置 JSON 缺字段不破坏启动恢复
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
//! 知识库集成 — 注入 + 提炼(嵌入生成 / 混合检索 / 上下文构建 / 对话提炼)
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
|
||||
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
@@ -24,6 +25,168 @@ use crate::commands::err_str;
|
||||
|
||||
use super::{AiSession};
|
||||
|
||||
// ============================================================
|
||||
// 知识注入/提炼参数(集中收口,不散魔法数)
|
||||
// ============================================================
|
||||
|
||||
/// 注入检索 query 最多保留的关键词个数(按出现频次取前 N)
|
||||
///
|
||||
/// 多词会放大 SQL 命中面并稀释「标题命中 > 内容命中」的排序精度,6 个足以覆盖
|
||||
/// 一句提问的信息量(ASCII 专有名词 + 核心汉字 2-gram)。
|
||||
const KNOWLEDGE_QUERY_MAX_KEYWORDS: usize = 6;
|
||||
/// ASCII 词最短长度(单字符词如 a/x/i 全表乱命中,过滤)
|
||||
const KNOWLEDGE_QUERY_MIN_ASCII_LEN: usize = 2;
|
||||
/// 自动提炼窗口:取最近 N 条 user/assistant 消息送 LLM 提炼
|
||||
///
|
||||
/// 原 6 条窗口过窄,长会话后半程的经验不在提炼视野内;40 条覆盖近期完整上下文,
|
||||
/// 同时不撑爆提炼调用的 token 预算(max_tokens 2048 的输出侧不受影响,输入侧由
|
||||
/// 提炼模型上下文窗口兜底)。
|
||||
const KNOWLEDGE_EXTRACT_WINDOW: usize = 40;
|
||||
/// 自动提炼频次:距上次提炼至少新增 N 条用户消息才再提炼
|
||||
///
|
||||
/// 取代原「每会话仅提炼一次」的 bool 硬去重——长会话后半程的新经验此前永不提炼。
|
||||
const KNOWLEDGE_EXTRACT_MIN_NEW_USER_MSGS: usize = 10;
|
||||
|
||||
/// 自动提炼频次水位(conv_id → 上次提炼进度),见 [`EXTRACT_WATERMARKS`]。
|
||||
#[derive(Clone, Copy)]
|
||||
enum ExtractWatermark {
|
||||
/// 提炼进行中(spawn 前置位,完成/失败后改写,防并发两路同时触发)
|
||||
InFlight,
|
||||
/// 上次成功提炼完成时的用户消息累计条数(含已压缩消息,保证单调递增)
|
||||
Done(usize),
|
||||
}
|
||||
|
||||
/// 提炼频次水位表(进程内,不落库)。
|
||||
///
|
||||
/// 原「每会话仅提炼一次」的 `knowledge_extracted` bool 去重使长会话后半程的新经验
|
||||
/// 永不提炼(知识库走查 2026-09-29);改为水位制:距上次提炼累计新增
|
||||
/// [`KNOWLEDGE_EXTRACT_MIN_NEW_USER_MSGS`] 条用户消息可再提炼。
|
||||
/// 放本模块静态而非 PerConvState:后者归属会话状态层(本次改动边界外),且水位随进程
|
||||
/// 生命周期存续即可——重启丢失后最多多提炼一轮,无正确性风险。
|
||||
static EXTRACT_WATERMARKS: LazyLock<StdMutex<HashMap<String, ExtractWatermark>>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
|
||||
/// 统计会话的用户消息累计条数(提炼频次水位的计量单位)
|
||||
fn count_user_rounds<'a>(msgs: impl Iterator<Item = &'a ChatMessage>) -> usize {
|
||||
msgs.filter(|m| matches!(m.role, MessageRole::User)).count()
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 检索 query 分词(整句 LIKE 命中率≈0 的根治,见 search_keywords)
|
||||
// ============================================================
|
||||
|
||||
/// 中文判断(CJK 统一表意文字区,覆盖日常中文)
|
||||
fn is_cjk_char(c: char) -> bool {
|
||||
matches!(c, '\u{4E00}'..='\u{9FFF}')
|
||||
}
|
||||
|
||||
/// ASCII 词字符(字母数字下划线)
|
||||
fn is_ascii_word_char(c: char) -> bool {
|
||||
c.is_ascii_alphanumeric() || c == '_'
|
||||
}
|
||||
|
||||
/// 高频虚词停用表(2-gram 粒度):中文疑问/指代/客套组合 + 英文虚词。
|
||||
/// ASCII 词抽取时已统一小写,英文停用词按小写收录;命中即弃,不参与检索。
|
||||
const QUERY_STOPWORDS: &[&str] = &[
|
||||
"怎么", "如何", "什么", "为何", "帮我", "看看", "这个", "那个", "一下", "可以",
|
||||
"应该", "需要", "是否", "如果", "但是", "所以", "因为", "还是", "就是", "没有",
|
||||
"一个", "这些", "那些", "然后", "现在", "已经", "我们", "你们", "他们", "自己",
|
||||
"时候", "我看", "请把", "觉得", "感觉", "好像", "对于", "关于",
|
||||
"the", "and", "for", "with", "this", "that", "from", "have", "what", "how",
|
||||
"can", "you", "are", "was", "not", "but", "about", "into", "use", "please",
|
||||
"help", "want", "need", "should", "would", "could", "there", "here", "when",
|
||||
"where", "which", "your",
|
||||
];
|
||||
|
||||
/// 从用户消息抽取检索关键词:ASCII 词(≥2 字符,统一小写)+ CJK 2-gram,按频次取前 N
|
||||
///
|
||||
/// 背景(知识库走查 2026-09-29):原把整条用户消息当 LIKE pattern,中文长句几乎不可能
|
||||
/// 作为连续子串命中知识正文,注入检索近乎失效。不上分词器:提问多为短句,
|
||||
/// 2-gram + 停用词已够;不依赖 df-ai 的同思路实现(df-storage 在依赖链下层,反向依赖会成环),
|
||||
/// 故此处自实现最小分词。停用词过滤同时压掉客套话与指代词的噪音 2-gram。
|
||||
///
|
||||
/// 返回按出现频次降序的前 [`KNOWLEDGE_QUERY_MAX_KEYWORDS`] 个,同频次 ASCII 词优先于
|
||||
/// CJK 2-gram(2-gram 是无词典近似噪音多,等频抢占容量时精确词让位),组内按首次
|
||||
/// 出现顺序稳定排序(检索 query 确定,行为可测)。
|
||||
pub(crate) fn extract_query_keywords(text: &str) -> Vec<String> {
|
||||
let mut tokens: Vec<String> = Vec::new();
|
||||
let mut ascii_buf = String::new();
|
||||
let mut cjk_buf: Vec<char> = Vec::new();
|
||||
|
||||
for c in text.chars() {
|
||||
if is_ascii_word_char(c) {
|
||||
// ASCII 词打断了汉字连续段:先结算汉字 2-gram 再收集 ASCII
|
||||
flush_cjk_bigrams(&mut cjk_buf, &mut tokens);
|
||||
ascii_buf.push(c.to_ascii_lowercase());
|
||||
} else {
|
||||
if !ascii_buf.is_empty() {
|
||||
tokens.push(std::mem::take(&mut ascii_buf));
|
||||
}
|
||||
if is_cjk_char(c) {
|
||||
cjk_buf.push(c);
|
||||
} else {
|
||||
// 空白/标点等分隔符:结算当前汉字段
|
||||
flush_cjk_bigrams(&mut cjk_buf, &mut tokens);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ascii_buf.is_empty() {
|
||||
tokens.push(ascii_buf);
|
||||
}
|
||||
flush_cjk_bigrams(&mut cjk_buf, &mut tokens);
|
||||
|
||||
// 过滤(过短/停用词)后按频次降序取前 N;同频次 ASCII 词优先于 CJK 2-gram,
|
||||
// 组内按首现顺序稳定排序
|
||||
let ascii_word = |t: &str| t.starts_with(is_ascii_word_char);
|
||||
let mut freq: Vec<(String, usize, usize)> = Vec::new();
|
||||
for (idx, t) in tokens.into_iter().enumerate() {
|
||||
if t.chars().count() < KNOWLEDGE_QUERY_MIN_ASCII_LEN
|
||||
|| QUERY_STOPWORDS.contains(&t.as_str())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if let Some(entry) = freq.iter_mut().find(|(tok, _, _)| tok == &t) {
|
||||
entry.1 += 1;
|
||||
} else {
|
||||
freq.push((t, 1, idx));
|
||||
}
|
||||
}
|
||||
freq.sort_by(|a, b| {
|
||||
b.1.cmp(&a.1)
|
||||
.then_with(|| ascii_word(&b.0).cmp(&ascii_word(&a.0)))
|
||||
.then_with(|| a.2.cmp(&b.2))
|
||||
});
|
||||
freq
|
||||
.into_iter()
|
||||
.take(KNOWLEDGE_QUERY_MAX_KEYWORDS)
|
||||
.map(|(t, _, _)| t)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 结算 CJK 连续段:剥离段内停用词后两两切 2-gram。
|
||||
///
|
||||
/// 中文无空格分词,2-gram 是无词典近似的最小可靠粒度;孤单汉字不产出
|
||||
/// (单字命中率低噪音高)。段长 <2 时 windows(2) 自然产出空,无需特判。
|
||||
///
|
||||
/// 停用词须在切 gram 前替换为分隔符打断连续段:切完再整词过滤滤不掉
|
||||
/// 停用词与邻字的跨词组合 gram("帮我看看这个怎么修复"会产生"看这/个怎"噪音)。
|
||||
/// 表内 CJK 停用词均为 2 字词,ASCII 停用词不会命中纯汉字段,统一替换无副作用;
|
||||
/// 剥离后单字残留不成 gram 自然丢弃,与孤单汉字语义一致。
|
||||
fn flush_cjk_bigrams(buf: &mut Vec<char>, out: &mut Vec<String>) {
|
||||
let mut segment: String = buf.iter().collect();
|
||||
for w in QUERY_STOPWORDS {
|
||||
if segment.contains(w) {
|
||||
segment = segment.replace(w, "\u{0}");
|
||||
}
|
||||
}
|
||||
for part in segment.split('\u{0}') {
|
||||
for w in part.chars().collect::<Vec<_>>().windows(2) {
|
||||
out.push(w.iter().collect());
|
||||
}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
/// 按配置构建 embedding provider + model。None = 配置缺失/provider 不存在。
|
||||
///
|
||||
/// F-01 阶段5: model 选择路由优先 — 在 embedding provider 的 model_configs 池中按
|
||||
@@ -232,13 +395,20 @@ pub async fn retry_failed_embeddings(state: &AppState) -> anyhow::Result<usize>
|
||||
///
|
||||
/// 双信号(两路都命中)排最前,LIKE 单信号次之,向量单信号第三。
|
||||
/// vector_enabled 关闭或 embed 失败时纯 LIKE(零外部依赖降级)。
|
||||
/// LIKE 路 query 先经 [`extract_query_keywords`] 分词(整句 LIKE 命中率≈0);
|
||||
/// 向量路 query 同步改用关键词拼接(虚词稀释 embedding 语义),分词为空时回退原句。
|
||||
async fn hybrid_search(
|
||||
state: &AppState,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
config: &crate::state::KnowledgeConfig,
|
||||
) -> Vec<df_storage::models::KnowledgeRecord> {
|
||||
let keyword_results = match state.knowledge.search(query, None, limit).await {
|
||||
let keywords = extract_query_keywords(query);
|
||||
let keyword_results = match state
|
||||
.knowledge
|
||||
.search_keywords(&keywords, config.include_candidates, limit)
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "[ai] 知识关键词检索失败,降级空结果");
|
||||
@@ -249,7 +419,12 @@ async fn hybrid_search(
|
||||
if !config.vector_enabled {
|
||||
return keyword_results;
|
||||
}
|
||||
let query_vec = match generate_embedding(state, query, config).await {
|
||||
let embed_query = if keywords.is_empty() {
|
||||
query.to_string()
|
||||
} else {
|
||||
keywords.join("、")
|
||||
};
|
||||
let query_vec = match generate_embedding(state, &embed_query, config).await {
|
||||
Some(v) => v,
|
||||
None => return keyword_results, // embed 失败降级
|
||||
};
|
||||
@@ -322,19 +497,40 @@ pub(crate) async fn build_knowledge_context(
|
||||
if results.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
// 命中条目:复用计数 +1 + 记录引用事件(fire-and-forget,单个 spawn 任务批量处理)
|
||||
// 命中条目:复用计数 +1 + candidate 复用达标自动晋升 + 记录引用事件(fire-and-forget,单个 spawn 任务批量处理)
|
||||
let db = state.db.clone();
|
||||
let ids: Vec<String> = results.iter().map(|r| r.id.clone()).collect();
|
||||
// (id, 是否 candidate):晋升检查只对 candidate 发起,published 免一次注定空跑的 UPDATE
|
||||
let hits: Vec<(String, bool)> = results
|
||||
.iter()
|
||||
.map(|r| (r.id.clone(), r.status == "candidate"))
|
||||
.collect();
|
||||
let conv_id = conv_id.to_string();
|
||||
let query_clone = query.to_string();
|
||||
let user_message_id = user_message_id.map(|s| s.to_string());
|
||||
// 晋升开关与阈值取配置快照值拷贝进闭包(决策单未拍板前默认保守,开关默认关)
|
||||
let auto_publish_enabled = config.auto_publish_enabled;
|
||||
let auto_publish_threshold = config.auto_publish_reuse_threshold;
|
||||
tauri::async_runtime::spawn(async move {
|
||||
let repo = KnowledgeRepo::new(&db);
|
||||
let timeline = crate::commands::knowledge_timeline::KnowledgeTimeline::new(&db);
|
||||
for id in &ids {
|
||||
for (id, is_candidate) in &hits {
|
||||
if let Err(e) = repo.increment_reuse_count(id).await {
|
||||
tracing::warn!("reuse_count +1 失败(非阻断): {}", e);
|
||||
}
|
||||
// 复用即信任:开关开启时 candidate 达到阈值自动晋升 published(SQL 侧原子判定,
|
||||
// 重复调用幂等)。默认关(决策卡 C3 未拍板前保守),开启后阈值取配置。
|
||||
if *is_candidate && auto_publish_enabled {
|
||||
match repo.auto_publish_if_reused(id, auto_publish_threshold).await {
|
||||
Ok(true) => {
|
||||
tracing::info!("[knowledge] candidate 复用达标({auto_publish_threshold} 次),自动晋升 published: {id}");
|
||||
// 晋升留痕:生命线记 status_changed(source_ref=auto_publish),
|
||||
// fire-and-forget 与上方 record_referenced 同风格,失败仅 warn 非阻断。
|
||||
timeline.record_auto_publish(id, auto_publish_threshold).await;
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(e) => tracing::warn!("candidate 自动晋升检查失败(非阻断): {e}"),
|
||||
}
|
||||
}
|
||||
timeline
|
||||
.record_referenced(id, &conv_id, &query_clone, user_message_id.as_deref())
|
||||
.await;
|
||||
@@ -423,13 +619,15 @@ pub(crate) async fn inject_knowledge_into_prompt(
|
||||
/// 判断是否应触发提炼,满足则后台 spawn 提炼 task(非阻断)
|
||||
///
|
||||
/// 守卫: auto_extract 开 + trigger_mode == OnComplete + 消息数 ≥ min_messages
|
||||
/// + 去重标志(P1 修复:本会话已成功提炼过则跳过,防重复触发刷 candidate)
|
||||
/// + 频次水位(提炼中 InFlight,或距上次提炼新增用户消息
|
||||
/// < [`KNOWLEDGE_EXTRACT_MIN_NEW_USER_MSGS`] 则跳过)。
|
||||
/// 原「每会话仅提炼一次」的 bool 硬去重已由水位制取代——长会话后半程的新经验
|
||||
/// 此前永不提炼(知识库走查 2026-09-29)。
|
||||
///
|
||||
/// TOCTOU 修复(🟡D):老实现 read(knowledge_extracted) → release lock → spawn,
|
||||
/// 并发窗口内另一路 maybe_spawn_extraction 也能读到 false 各自 spawn,致重复提炼刷 candidate。
|
||||
/// 现在判重 + 消息数 + **预置位**三步在**同一个锁临界区**内原子完成:置位即占用提炼槽位,
|
||||
/// 并发的后来者读到 true 直接跳过。spawn 后按结果修正:0 条 / Err 清位(允许下次重试),
|
||||
/// ≥1 条保持 true(已提炼,后续跳过)。
|
||||
/// TOCTOU 防护保持:守卫判定 + **预置位**(InFlight 水位 + knowledge_extracted 哨兵)
|
||||
/// 在**同一个锁临界区**内原子完成,并发窗口内的后来者读到占用即跳过,防重复提炼刷
|
||||
/// candidate。spawn 后按结果修正:0 条 / Err 清位回退水位(允许下次重试),
|
||||
/// ≥1 条记 Done 水位(此后须再攒够阈值条用户消息才再提炼)。
|
||||
pub(crate) async fn maybe_spawn_extraction(
|
||||
session_arc: &Arc<Mutex<AiSession>>,
|
||||
db: &Arc<Database>,
|
||||
@@ -444,46 +642,60 @@ pub(crate) async fn maybe_spawn_extraction(
|
||||
if config.trigger_mode != ExtractTrigger::OnComplete {
|
||||
return Ok(());
|
||||
}
|
||||
// P1 修复(提炼重复触发)+ 🟡D TOCTOU 修复:判重 + 消息数守卫 + 预置位同一锁内原子完成。
|
||||
// 背景:审批续跑 try_continue → 重 spawn run_agentic_loop → 正常完成块 →
|
||||
// maybe_spawn_extraction 二次触发,无去重致同知识点重复 candidate 刷屏。
|
||||
// 预置位语义:knowledge_extracted 在 spawn 前先置 true 占提炼槽位,并发后来者读到
|
||||
// true 即跳过(等价「提炼中」哨兵);spawn 后按 inserted 结果修正(见下)。
|
||||
// 清位:trigger_extraction_now(手动按钮)强制清位,允许用户手动重提炼。
|
||||
// 频次水位守卫 + 消息数守卫 + 预置位在同一锁临界区内原子完成(TOCTOU 防护与原 P1 修复同思路:
|
||||
// 审批续跑 try_continue → 重 spawn run_agentic_loop → 正常完成块会二次触发本函数,
|
||||
// 无占位致并发重复提炼刷 candidate)。
|
||||
// 清位/回退:trigger_extraction_now(手动按钮)强制清哨兵 + 水位,允许用户立即手动重提炼。
|
||||
{
|
||||
let __lock_t = std::time::Instant::now();
|
||||
let mut session = session_arc.lock().await;
|
||||
// 一次 conv_read 读两个字段(knowledge_extracted + messages.len()),map 后借用即结束,
|
||||
// 一次 conv_read 读两个字段(消息总数守卫 + 用户消息水位计量),map 后借用即结束,
|
||||
// 后续 session.conv()(&mut self)不再冲突。
|
||||
let (already, count) = session
|
||||
let (msg_count, user_rounds) = session
|
||||
.conv_read(conv_id)
|
||||
.map(|c| (c.knowledge_extracted, c.messages.len()))
|
||||
.unwrap_or((false, 0));
|
||||
// 守卫 1:已提炼过(或提炼中) → 跳过 + warn。
|
||||
if already {
|
||||
.map(|c| (c.messages.len(), count_user_rounds(c.messages.iter())))
|
||||
.unwrap_or((0, 0));
|
||||
// 守卫 1:频次水位(InFlight=提炼中防并发;Done=新增用户消息不足阈值)。
|
||||
// 替代原 bool 硬去重;跳过走 debug 留痕(改水位制后此路径高频,warn 会刷屏)。
|
||||
let blocked = {
|
||||
let wms = EXTRACT_WATERMARKS.lock().expect("提炼水位互斥锁中毒");
|
||||
match wms.get(conv_id) {
|
||||
Some(ExtractWatermark::InFlight) => true,
|
||||
Some(ExtractWatermark::Done(last)) => {
|
||||
user_rounds.saturating_sub(*last) < KNOWLEDGE_EXTRACT_MIN_NEW_USER_MSGS
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
};
|
||||
if blocked {
|
||||
let __hold = __lock_t.elapsed();
|
||||
if __hold > std::time::Duration::from_millis(30) {
|
||||
eprintln!("[LOCK-SLOW] maybe_spawn_extraction:437 持锁 {:?} (含 lock 等待)", __hold);
|
||||
eprintln!("[LOCK-SLOW] maybe_spawn_extraction 持锁 {:?} (含 lock 等待)", __hold);
|
||||
}
|
||||
tracing::warn!(
|
||||
tracing::debug!(
|
||||
conv_id = %conv_id,
|
||||
"[knowledge] 跳过自动提炼:本会话已提炼过/提炼中(去重标志置位,防重复刷 candidate);如需重提炼用手动按钮"
|
||||
"[knowledge] 跳过自动提炼:提炼中或距上次提炼新增用户消息不足 {} 条(如需立即重提炼用手动按钮)",
|
||||
KNOWLEDGE_EXTRACT_MIN_NEW_USER_MSGS
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
// 守卫 2:消息数。conv_read 未建返 0(< min_messages 自然跳过,语义=空对话不注入知识)。
|
||||
if (count as u32) < config.min_messages {
|
||||
if msg_count < config.min_messages as usize {
|
||||
let __hold = __lock_t.elapsed();
|
||||
if __hold > std::time::Duration::from_millis(30) {
|
||||
eprintln!("[LOCK-SLOW] maybe_spawn_extraction:437 持锁 {:?} (含 lock 等待)", __hold);
|
||||
eprintln!("[LOCK-SLOW] maybe_spawn_extraction 持锁 {:?} (含 lock 等待)", __hold);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
// 原子预置位(TOCTOU 核心):释放锁前先占提炼槽位,杜绝并发窗口内重复 spawn。
|
||||
session.conv(conv_id).knowledge_extracted = true;
|
||||
EXTRACT_WATERMARKS
|
||||
.lock()
|
||||
.expect("提炼水位互斥锁中毒")
|
||||
.insert(conv_id.to_string(), ExtractWatermark::InFlight);
|
||||
let __hold = __lock_t.elapsed();
|
||||
if __hold > std::time::Duration::from_millis(30) {
|
||||
eprintln!("[LOCK-SLOW] maybe_spawn_extraction:437 持锁 {:?} (含 lock 等待)", __hold);
|
||||
eprintln!("[LOCK-SLOW] maybe_spawn_extraction 持锁 {:?} (含 lock 等待)", __hold);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,26 +708,44 @@ pub(crate) async fn maybe_spawn_extraction(
|
||||
match extract_knowledge_from_conversation(&db, &conv_id, &provider_config, &llm_concurrency).await {
|
||||
Ok(inserted) => {
|
||||
if inserted > 0 {
|
||||
// ≥1 条:保持预置的 true(已占用槽位即最终状态),仅 log。
|
||||
// ≥1 条:哨兵保持置位,水位记为当前用户消息进度
|
||||
// (此后须再新增 ≥ 阈值条用户消息才会再提炼)。
|
||||
let session = session_arc.lock().await;
|
||||
let rounds = session
|
||||
.conv_read(&conv_id)
|
||||
.map(|c| count_user_rounds(c.messages.iter()))
|
||||
.unwrap_or(0);
|
||||
EXTRACT_WATERMARKS
|
||||
.lock()
|
||||
.expect("提炼水位互斥锁中毒")
|
||||
.insert(conv_id.clone(), ExtractWatermark::Done(rounds));
|
||||
tracing::info!(
|
||||
conv_id = %conv_id, inserted,
|
||||
"[knowledge] 提炼成功,去重标志保持置位,后续自动触发将被跳过"
|
||||
conv_id = %conv_id, inserted, rounds,
|
||||
"[knowledge] 提炼成功,水位记为当前进度,新增用户消息达阈值后才再提炼"
|
||||
);
|
||||
} else {
|
||||
// 0 条:清位回 false,允许下次自动触发重试(对话可能后续补充了有价值内容)。
|
||||
// 0 条:清哨兵 + 回退水位,允许下次自动触发重试(对话可能后续补充了有价值内容)。
|
||||
let mut session = session_arc.lock().await;
|
||||
session.conv(&conv_id).knowledge_extracted = false;
|
||||
EXTRACT_WATERMARKS
|
||||
.lock()
|
||||
.expect("提炼水位互斥锁中毒")
|
||||
.remove(&conv_id);
|
||||
tracing::info!(
|
||||
conv_id = %conv_id,
|
||||
"[knowledge] 提炼 0 条(无可提炼内容),清去重标志允许下次自动重试"
|
||||
"[knowledge] 提炼 0 条(无可提炼内容),清哨兵与水位允许下次自动重试"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// 失败:清位回 false,允许下次触发重试(预置位已在,须回滚否则会话被锁死)。
|
||||
// 失败:清哨兵 + 回退水位,允许下次触发重试(预置位已在,须回滚否则会话被锁死)。
|
||||
tracing::warn!("知识提取失败(非阻断): {}", e);
|
||||
let mut session = session_arc.lock().await;
|
||||
session.conv(&conv_id).knowledge_extracted = false;
|
||||
EXTRACT_WATERMARKS
|
||||
.lock()
|
||||
.expect("提炼水位互斥锁中毒")
|
||||
.remove(&conv_id);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -532,9 +762,13 @@ pub async fn trigger_extraction_now(state: &AppState) -> Result<bool, String> {
|
||||
let conv_id = {
|
||||
let mut session = state.ai_session.lock().await;
|
||||
let conv_id = session.active_conversation_id.clone();
|
||||
// 强制清去重标志:手动提炼是用户显式动作,允许对已提炼过的会话重提炼。
|
||||
// 强制清哨兵 + 频次水位:手动提炼是用户显式动作,绕过自动路径的间隔限制(语义=手动覆盖)。
|
||||
if let Some(cid) = conv_id.as_deref() {
|
||||
session.conv(cid).knowledge_extracted = false;
|
||||
EXTRACT_WATERMARKS
|
||||
.lock()
|
||||
.expect("提炼水位互斥锁中毒")
|
||||
.remove(cid);
|
||||
}
|
||||
conv_id
|
||||
};
|
||||
@@ -546,11 +780,19 @@ pub async fn trigger_extraction_now(state: &AppState) -> Result<bool, String> {
|
||||
tauri::async_runtime::spawn(async move {
|
||||
match extract_knowledge_from_conversation(&db, &conv_id, &provider_config, &llm_concurrency).await {
|
||||
Ok(inserted) => {
|
||||
// 手动提炼成功(≥1 条)后也置去重标志,保持与自动路径一致:
|
||||
// 手动提炼成功(≥1 条)后与自动路径同语义收口:置哨兵 + 记水位,
|
||||
// 避免手动提炼后再触发自动提炼重复。用户再次手动按按钮会再次清位,行为自洽。
|
||||
if inserted > 0 {
|
||||
let mut session = session_arc.lock().await;
|
||||
session.conv(&conv_id).knowledge_extracted = true;
|
||||
let rounds = session
|
||||
.conv_read(&conv_id)
|
||||
.map(|c| count_user_rounds(c.messages.iter()))
|
||||
.unwrap_or(0);
|
||||
EXTRACT_WATERMARKS
|
||||
.lock()
|
||||
.expect("提炼水位互斥锁中毒")
|
||||
.insert(conv_id.clone(), ExtractWatermark::Done(rounds));
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!("手动提炼失败(非阻断): {}", e),
|
||||
@@ -574,7 +816,8 @@ const EXTRACTION_SYSTEM_PROMPT: &str = "你是知识提炼引擎,从 AI 对话
|
||||
|
||||
/// 从对话中提炼知识,产出 candidate 写入知识库
|
||||
///
|
||||
/// 流程: 读对话消息 → 过滤 user/assistant 取最后 6 条 → LLM 提炼(强制 JSON) → 解析 → 批量插入 candidate
|
||||
/// 流程: 读对话消息 → 过滤 user/assistant 取最近 [`KNOWLEDGE_EXTRACT_WINDOW`] 条 →
|
||||
/// LLM 提炼(强制 JSON) → 解析 → 批量插入 candidate
|
||||
///
|
||||
/// 返回值: 成功插入的 candidate 条数(inserted)。调用方(maybe_spawn_extraction)据此置
|
||||
/// `knowledge_extracted` 去重标志(≥1 才置位,0 条不置位—允许下次自动触发重试)。
|
||||
@@ -636,12 +879,13 @@ async fn extract_knowledge_from_conversation(
|
||||
}
|
||||
}
|
||||
};
|
||||
// 过滤 user/assistant,取最后 6 条
|
||||
// 过滤 user/assistant,取最近 KNOWLEDGE_EXTRACT_WINDOW 条
|
||||
// (原 6 条窗口过窄:长会话后半程的经验不在提炼视野内;放宽后单次提炼可覆盖近期完整上下文)
|
||||
let recent: Vec<&ChatMessage> = messages
|
||||
.iter()
|
||||
.filter(|m| matches!(m.role, MessageRole::User | MessageRole::Assistant))
|
||||
.rev()
|
||||
.take(6)
|
||||
.take(KNOWLEDGE_EXTRACT_WINDOW)
|
||||
.collect();
|
||||
if recent.len() < 4 {
|
||||
return Ok(0); // 太短,不值得提炼(0 条,不置去重标志)
|
||||
@@ -947,4 +1191,74 @@ mod tests {
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].id, "kw1");
|
||||
}
|
||||
|
||||
// ---------- 检索 query 分词(extract_query_keywords) ----------
|
||||
|
||||
#[test]
|
||||
fn keywords_ascii_words_lowercased() {
|
||||
let out = extract_query_keywords("修复 KnowledgeRepo 的 Search_Bug");
|
||||
assert!(out.contains(&"knowledgerepo".to_string()), "ASCII 词应小写: {:?}", out);
|
||||
assert!(out.contains(&"search_bug".to_string()), "下划线应保留在词内: {:?}", out);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_cjk_bigrams_extracted() {
|
||||
let out = extract_query_keywords("修复知识库检索");
|
||||
// 连续汉字两两切:修复/复知/知识/识库/检索
|
||||
assert!(out.contains(&"修复".to_string()));
|
||||
assert!(out.contains(&"知识".to_string()));
|
||||
assert!(out.contains(&"检索".to_string()));
|
||||
assert!(out.contains(&"复知".to_string()), "跨词 2-gram 允许存在(无词典近似的已知代价)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_stopwords_filtered() {
|
||||
let out = extract_query_keywords("帮我看看这个怎么修复");
|
||||
// 帮我/看看/这个/怎么全为停用词,只剩修复
|
||||
assert_eq!(out, vec!["修复".to_string()], "停用词应被过滤: {:?}", out);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_short_ascii_dropped() {
|
||||
let out = extract_query_keywords("use a x axis_config");
|
||||
// 单字符 a/x 被过滤(use 为英文停用词),保留 axis_config
|
||||
assert_eq!(out, vec!["axis_config".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_single_cjk_char_dropped() {
|
||||
// 孤单汉字不成 2-gram,不产出
|
||||
let out = extract_query_keywords("好 的");
|
||||
assert!(out.is_empty(), "单字与停用词应全过滤: {:?}", out);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_frequency_ranking_and_cap() {
|
||||
let text = "tokio 出现多次 tokio 然后 tokio 还有 rust 与 rust 以及 postgres";
|
||||
let out = extract_query_keywords(text);
|
||||
// 频次降序:tokio(3) > rust(2) > postgres(1);「然后」为停用词
|
||||
assert_eq!(out[0], "tokio");
|
||||
assert_eq!(out[1], "rust");
|
||||
assert!(out.contains(&"postgres".to_string()));
|
||||
// 同频次按首现顺序稳定:构造 8 个互异词验证上限 6
|
||||
let many = extract_query_keywords("alpha beta gamma delta epsilon zeta eta theta");
|
||||
assert_eq!(many.len(), 6, "应截断到前 6 个: {:?}", many);
|
||||
assert_eq!(many[0], "alpha", "同频次按首现顺序");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_empty_or_noise_returns_empty() {
|
||||
assert!(extract_query_keywords("").is_empty());
|
||||
assert!(extract_query_keywords(" ,。!?").is_empty(), "纯标点应产出空");
|
||||
assert!(extract_query_keywords("帮我看看一下这个").is_empty(), "全停用词应产出空");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_mixed_ascii_cjk_segmented() {
|
||||
let out = extract_query_keywords("用cargo编译失败error E0499");
|
||||
assert!(out.contains(&"cargo".to_string()));
|
||||
assert!(out.contains(&"e0499".to_string()), "字母数字混合词应整体保留: {:?}", out);
|
||||
assert!(out.contains(&"编译".to_string()));
|
||||
assert!(out.contains(&"失败".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user