新增: 知识库检索分词化与晋升配置化及对话工具
整句 LIKE 命中率趋零,注入检索改 CJK 2-gram+ASCII 词分词 OR 匹配,标题命中计分排序 停用词剥离前置到切分前,滤掉跨词组合噪音 gram;ASCII 等频精确词优先 candidate 纳入检索面(include_candidates 默认开,配开关可回仅 published) 复用即信任自动晋升配置化:默认关+阈值 5,等晋升把关策略拍板后再开,防未审核产物经误命中固化污染 晋升留痕进时间线(auto/agent 来源区分);补 candidate 复用达标自动转正机制 agent 对话侧新增 knowledge_search/insert/promote 三工具,insert 固定 candidate,转正仅限 candidate 单向 知识配置结构新增两字段带 serde 默认,旧配置 JSON 缺字段不破坏启动恢复
This commit is contained in:
@@ -85,6 +85,14 @@ fn parse_json_id_array(raw: &Option<String>) -> Vec<String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// LIKE 通配符转义(与 SQL 侧 `ESCAPE '|'` 配套)。
|
||||
///
|
||||
/// 检索词里的 `%`/`_`/`|` 会被当成通配/转义符,用户输入与分词产物都可能含这些字符,
|
||||
/// 统一在此转义,避免关键词意外扩大命中面。
|
||||
fn escape_like(s: &str) -> String {
|
||||
s.replace('|', "||").replace('%', "|%").replace('_', "|_")
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// from_row 辅助函数
|
||||
// ============================================================
|
||||
@@ -634,13 +642,13 @@ impl IdeaRepo {
|
||||
}
|
||||
|
||||
impl KnowledgeRepo {
|
||||
/// 检索知识: title/content LIKE 匹配,可选 kind 过滤,按 reuse_count 降序,top-N
|
||||
/// 检索知识(整句 LIKE,兼容路径): title/content LIKE 匹配,可选 kind 过滤,按 reuse_count 降序,top-N
|
||||
///
|
||||
/// 克制检索: top-N≤3(由调用方 limit 控制),精确匹配优先(语义模糊后做)。
|
||||
/// 保留整句匹配语义,服务 UI 知识面板搜索与 df-mcp `search_knowledge`(调用方传的就是
|
||||
/// 用户显式输入的搜索串,整句即意图)。AI 注入路径的分词检索走 [`KnowledgeRepo::search_keywords`]。
|
||||
pub async fn search(&self, query: &str, kind: Option<&str>, limit: usize) -> Result<Vec<KnowledgeRecord>> {
|
||||
let conn = self.conn.clone();
|
||||
let escaped = query.replace('|', "||").replace('%', "|%").replace('_', "|_");
|
||||
let pattern = format!("%{escaped}%");
|
||||
let pattern = format!("%{}%", escape_like(query));
|
||||
let kind = kind.map(|s| s.to_owned());
|
||||
let limit_i = limit as i64;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
@@ -673,6 +681,77 @@ impl KnowledgeRepo {
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 分词关键词检索(AI 注入路径专用): 多关键词 OR 命中 title/content
|
||||
///
|
||||
/// 排序:标题命中词数 > 内容命中 > reuse_count 降序(SQL 内 CASE 计分,同分再看复用次数)。
|
||||
///
|
||||
/// 背景(知识库走查 2026-09-29):注入路径原复用 [`KnowledgeRepo::search`] 把**整句用户消息**
|
||||
/// 当 LIKE pattern,`%怎么修复知识库检索命中率%` 这类长串几乎不可能作为连续子串出现在
|
||||
/// 任何知识正文里,命中率≈0。本方法由 caller 分词后逐词匹配,任一词命中即入候选,
|
||||
/// 多词同时命中者排前。关键词条数为 WHERE 的 OR 分支数,通常 ≤6,全表扫描量级可控。
|
||||
///
|
||||
/// `include_candidates`:是否把 status='candidate'(AI 自动提炼产物)纳入检索面。
|
||||
/// 提炼产物曾长期全部滞留 candidate 而本路径只认 published,存量知识近乎零生效;
|
||||
/// 默认纳入后靠「复用即信任」([`KnowledgeRepo::auto_publish_if_reused`])渐进晋升,
|
||||
/// 人工审核降级为可选修正路径。
|
||||
///
|
||||
/// 关键词为空 → 返回空(不回退整句 LIKE,避免退化回原缺陷)。
|
||||
pub async fn search_keywords(
|
||||
&self,
|
||||
keywords: &[String],
|
||||
include_candidates: bool,
|
||||
limit: usize,
|
||||
) -> Result<Vec<KnowledgeRecord>> {
|
||||
if keywords.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.clone();
|
||||
// 每个关键词占一个编号占位符;WHERE 与 ORDER BY 的 CASE 复用同号(SQLite 允许 ?N 复用),
|
||||
// 参数列表只装一份 pattern,末尾追加 LIMIT 占位符。
|
||||
let mut patterns: Vec<String> = Vec::with_capacity(keywords.len());
|
||||
let mut where_parts: Vec<String> = Vec::with_capacity(keywords.len());
|
||||
let mut title_score_terms: Vec<String> = Vec::with_capacity(keywords.len());
|
||||
for (i, kw) in keywords.iter().enumerate() {
|
||||
let n = i + 1;
|
||||
where_parts.push(format!(
|
||||
"(title LIKE ?{n} ESCAPE '|' OR content LIKE ?{n} ESCAPE '|')"
|
||||
));
|
||||
title_score_terms.push(format!("CASE WHEN title LIKE ?{n} ESCAPE '|' THEN 1 ELSE 0 END"));
|
||||
patterns.push(format!("%{}%", escape_like(kw)));
|
||||
}
|
||||
let status_clause = if include_candidates {
|
||||
"status IN ('published','candidate')"
|
||||
} else {
|
||||
"status = 'published'"
|
||||
};
|
||||
let limit_i = limit as i64;
|
||||
let sql = format!(
|
||||
"SELECT {KNOWLEDGE_COLS} FROM knowledges \
|
||||
WHERE {status_clause} AND ({}) \
|
||||
ORDER BY ({}) DESC, reuse_count DESC, created_at DESC LIMIT ?{}",
|
||||
where_parts.join(" OR "),
|
||||
title_score_terms.join(" + "),
|
||||
patterns.len() + 1,
|
||||
);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = conn.blocking_lock();
|
||||
let mut param_refs: Vec<&dyn rusqlite::ToSql> =
|
||||
patterns.iter().map(|p| p as &dyn rusqlite::ToSql).collect();
|
||||
param_refs.push(&limit_i);
|
||||
let mut stmt = guard.prepare(&sql).map_err(storage_err)?;
|
||||
let rows = stmt
|
||||
.query_map(param_refs.as_slice(), |row| knowledge_from_row(row))
|
||||
.map_err(storage_err)?;
|
||||
let mut results = Vec::new();
|
||||
for r in rows {
|
||||
results.push(r.map_err(storage_err)?);
|
||||
}
|
||||
Ok(results)
|
||||
})
|
||||
.await
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 按状态列出(审核收件箱用): 按 confidence 语义排序(high>medium>low),次按 created_at
|
||||
///
|
||||
/// 用 CASE WHEN 替代纯 TEXT 排序(字典序 high>low>medium 非预期语义)。
|
||||
@@ -724,6 +803,59 @@ impl KnowledgeRepo {
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 复用即信任:candidate 的 reuse_count 达到阈值后自动晋升 published
|
||||
///
|
||||
/// 背景(知识库走查 2026-09-29):AI 提炼产物长期滞留 candidate,人工审核入口堆积无人处理,
|
||||
/// 存量知识近乎零生效。「被真实注入命中 N 次」比「被人工翻到」信号更强,故以复用达标为
|
||||
/// 主晋升路径;人工审核保留为可选修正(仍可在面板驳回/归档)。
|
||||
///
|
||||
/// SQL 原子判定(`status='candidate' AND reuse_count >= threshold`):published/archived
|
||||
/// 条目不受影响,幂等可重复调用。返回 true = 本次发生了晋升(调用方据此打留痕日志)。
|
||||
pub async fn auto_publish_if_reused(&self, id: &str, threshold: i64) -> Result<bool> {
|
||||
let conn = self.conn.clone();
|
||||
let id = id.to_owned();
|
||||
let now = now_millis_str();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = conn.blocking_lock();
|
||||
let affected = guard
|
||||
.execute(
|
||||
"UPDATE knowledges SET status = 'published', updated_at = ?1 \
|
||||
WHERE id = ?2 AND status = 'candidate' AND reuse_count >= ?3",
|
||||
params![now, id, threshold],
|
||||
)
|
||||
.map_err(storage_err)?;
|
||||
Ok(affected > 0)
|
||||
})
|
||||
.await
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 显式转正:candidate 单向晋升 published(对话内 knowledge_promote 工具 / 人工放行通道)
|
||||
///
|
||||
/// 与 [`KnowledgeRepo::auto_publish_if_reused`] 同款 SQL 原子条件写,区别在晋升条件:
|
||||
/// 本方法不做复用计数判定,仅 `status='candidate'` 一个门槛——显式转正是「人/agent
|
||||
/// 拍板」语义,不复用即信任的阈值逻辑。SQL WHERE 收口边界:published/archived 条目
|
||||
/// 不命中(affected=0 → false),调用方据此回「仅 candidate 可转正」,天然幂等且
|
||||
/// 禁止降级/归档/改内容(只写 status + updated_at 两列)。
|
||||
pub async fn promote(&self, id: &str) -> Result<bool> {
|
||||
let conn = self.conn.clone();
|
||||
let id = id.to_owned();
|
||||
let now = now_millis_str();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = conn.blocking_lock();
|
||||
let affected = guard
|
||||
.execute(
|
||||
"UPDATE knowledges SET status = 'published', updated_at = ?1 \
|
||||
WHERE id = ?2 AND status = 'candidate'",
|
||||
params![now, id],
|
||||
)
|
||||
.map_err(storage_err)?;
|
||||
Ok(affected > 0)
|
||||
})
|
||||
.await
|
||||
.map_err(storage_err)?
|
||||
}
|
||||
|
||||
/// 写入向量嵌入(BLOB = Vec<f32> 小端字节序列化)
|
||||
///
|
||||
/// embedding 列不进 KnowledgeRecord(IPC 不需要传向量给前端),专用方法读写。
|
||||
@@ -1261,6 +1393,205 @@ mod tests {
|
||||
assert_eq!(hits[1].id, "k3");
|
||||
}
|
||||
|
||||
// ---------- 分词关键词检索(search_keywords,知识注入路径) ----------
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_keywords_empty_input_returns_empty() {
|
||||
let repo = setup_repo().await;
|
||||
repo.insert(krec("k1", "Rust 异步", "tokio", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
// 关键词为空 → 直接空结果,不回退整句 LIKE(避免退化回整句匹配缺陷)
|
||||
let hits = repo.search_keywords(&[], true, 10).await.unwrap();
|
||||
assert!(hits.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_keywords_or_matches_any_keyword() {
|
||||
let repo = setup_repo().await;
|
||||
// k1 命中 title 的「异步」;k2 命中 content 的「tokio」——OR 语义两路都收
|
||||
repo.insert(krec("k1", "Rust 异步并发", "无关内容", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
repo.insert(krec("k2", "运行时", "tokio 深入", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let kws = vec!["异步".to_string(), "tokio".to_string()];
|
||||
let hits = repo.search_keywords(&kws, false, 10).await.unwrap();
|
||||
let ids: Vec<_> = hits.iter().map(|h| h.id.as_str()).collect();
|
||||
assert_eq!(ids.len(), 2);
|
||||
assert!(ids.contains(&"k1"));
|
||||
assert!(ids.contains(&"k2"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_keywords_title_hit_ranks_above_content_hit() {
|
||||
let repo = setup_repo().await;
|
||||
// k2 内容命中且复用次数极高,k1 仅标题命中 → 标题权重压过 reuse_count
|
||||
repo.insert(krec("k1", "Rust 所有权", "无关内容", "lesson", "published", None, 0))
|
||||
.await
|
||||
.unwrap();
|
||||
repo.insert(krec("k2", "无关标题", "聊聊 Rust 所有权与借用", "lesson", "published", None, 99))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let kws = vec!["所有权".to_string()];
|
||||
let hits = repo.search_keywords(&kws, false, 10).await.unwrap();
|
||||
assert_eq!(hits.len(), 2);
|
||||
assert_eq!(hits[0].id, "k1", "标题命中应排在内容命中之前");
|
||||
assert_eq!(hits[1].id, "k2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_keywords_more_title_hits_rank_first() {
|
||||
let repo = setup_repo().await;
|
||||
// 两条都是标题命中:k1 命中 2 词,k2 命中 1 词 → k1 前
|
||||
repo.insert(krec("k1", "Rust 异步模型", "...", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
repo.insert(krec("k2", "异步进阶", "...", "lesson", "published", None, 50))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let kws = vec!["Rust".to_string(), "异步".to_string()];
|
||||
let hits = repo.search_keywords(&kws, false, 10).await.unwrap();
|
||||
assert_eq!(hits.len(), 2);
|
||||
assert_eq!(hits[0].id, "k1", "标题命中词数多者应排前(同档才看 reuse_count)");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_keywords_include_candidates_toggle() {
|
||||
let repo = setup_repo().await;
|
||||
repo.insert(krec("pub1", "Rust 规范", "...", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
repo.insert(krec("cand1", "Rust 提炼稿", "...", "lesson", "candidate", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
let kws = vec!["Rust".to_string()];
|
||||
|
||||
// 默认(纳入 candidate):published + candidate 都出
|
||||
let hits = repo.search_keywords(&kws, true, 10).await.unwrap();
|
||||
assert_eq!(hits.len(), 2, "include_candidates=true 应纳入 candidate");
|
||||
|
||||
// 关闭:回到仅 published(兼容旧行为)
|
||||
let hits = repo.search_keywords(&kws, false, 10).await.unwrap();
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].id, "pub1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_keywords_like_wildcards_escaped() {
|
||||
let repo = setup_repo().await;
|
||||
// 关键词含 % 时不应充当通配符("100%" 不该命中任意串)
|
||||
repo.insert(krec("k1", "进度 100% 完成", "普通内容", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
repo.insert(krec("k2", "完全不同", "...", "lesson", "published", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let kws = vec!["100%".to_string()];
|
||||
let hits = repo.search_keywords(&kws, false, 10).await.unwrap();
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].id, "k1");
|
||||
}
|
||||
|
||||
// ---------- 复用即信任(auto_publish_if_reused) ----------
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_publish_promotes_candidate_at_threshold() {
|
||||
let repo = setup_repo().await;
|
||||
repo.insert(krec("c1", "提炼知识", "...", "lesson", "candidate", None, 0))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 复用 1 次(< 阈值 2)→ 不晋升
|
||||
repo.increment_reuse_count("c1").await.unwrap();
|
||||
assert!(!repo.auto_publish_if_reused("c1", 2).await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("c1").await.unwrap().unwrap().status.as_str(),
|
||||
"candidate"
|
||||
);
|
||||
|
||||
// 复用第 2 次(达标)→ 晋升 published
|
||||
repo.increment_reuse_count("c1").await.unwrap();
|
||||
assert!(repo.auto_publish_if_reused("c1", 2).await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("c1").await.unwrap().unwrap().status.as_str(),
|
||||
"published"
|
||||
);
|
||||
|
||||
// 已 published 再调:幂等 false(不再重复晋升)
|
||||
assert!(!repo.auto_publish_if_reused("c1", 2).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_publish_ignores_published_and_missing() {
|
||||
let repo = setup_repo().await;
|
||||
repo.insert(krec("p1", "已发布", "...", "lesson", "published", None, 10))
|
||||
.await
|
||||
.unwrap();
|
||||
// published 即使 reuse_count 远超阈值也不动(晋升仅面向 candidate)
|
||||
assert!(!repo.auto_publish_if_reused("p1", 2).await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("p1").await.unwrap().unwrap().status.as_str(),
|
||||
"published"
|
||||
);
|
||||
// 不存在的 id → false
|
||||
assert!(!repo.auto_publish_if_reused("ghost", 2).await.unwrap());
|
||||
}
|
||||
|
||||
// ---------- 显式转正(promote,对话内 knowledge_promote 工具通道) ----------
|
||||
|
||||
#[tokio::test]
|
||||
async fn promote_candidates_to_published() {
|
||||
let repo = setup_repo().await;
|
||||
repo.insert(krec("c1", "候选知识", "...", "lesson", "candidate", None, 0))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// candidate → published: true,状态落库
|
||||
assert!(repo.promote("c1").await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("c1").await.unwrap().unwrap().status.as_str(),
|
||||
"published"
|
||||
);
|
||||
|
||||
// 已 published 再调:幂等 false(晋升仅面向 candidate,不降级不重复)
|
||||
assert!(!repo.promote("c1").await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("c1").await.unwrap().unwrap().status.as_str(),
|
||||
"published"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn promote_ignores_published_archived_and_missing() {
|
||||
let repo = setup_repo().await;
|
||||
repo.insert(krec("p1", "已发布", "...", "lesson", "published", None, 3))
|
||||
.await
|
||||
.unwrap();
|
||||
repo.insert(krec("a1", "已归档", "...", "lesson", "archived", None, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// published / archived 均 WHERE 不命中 → false 且状态不变
|
||||
assert!(!repo.promote("p1").await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("p1").await.unwrap().unwrap().status.as_str(),
|
||||
"published"
|
||||
);
|
||||
assert!(!repo.promote("a1").await.unwrap());
|
||||
assert_eq!(
|
||||
repo.get_by_id("a1").await.unwrap().unwrap().status.as_str(),
|
||||
"archived"
|
||||
);
|
||||
// 不存在的 id → false
|
||||
assert!(!repo.promote("ghost").await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_by_status_orders_by_confidence_semantics() {
|
||||
// confidence 字典序 high>low>medium 非预期,实现用 CASE 强制 high>medium>low
|
||||
|
||||
Reference in New Issue
Block a user