新增: 断点续跑与日志租约底座

This commit is contained in:
lxy
2026-10-05 23:41:18 +08:00
parent 8233ba095e
commit cb260aba4b
22 changed files with 1132 additions and 136 deletions
+281 -22
View File
@@ -1,6 +1,7 @@
//! 知识库集成 — 注入 + 提炼(嵌入生成 / 混合检索 / 上下文构建 / 对话提炼)
use std::collections::HashMap;
use std::panic::AssertUnwindSafe;
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
use tokio::sync::Mutex;
@@ -13,7 +14,7 @@ use df_ai::provider::{ChatMessage, CompletionRequest, LlmProvider, MessageRole};
use df_ai::router::{
is_likely_image_model, select_model_id, Modality, TaskRequirements,
};
use df_storage::crud::{AiConversationRepo, AiMessageRepo, AiProviderRepo, KnowledgeRepo};
use df_storage::crud::{AiConversationRepo, AiMessageRepo, AiProviderRepo, KnowledgeRepo, ProjectRepo};
use df_storage::db::Database;
use df_storage::models::{AiProviderRecord, KnowledgeRecord};
@@ -21,6 +22,8 @@ use df_types::types::new_id;
use crate::state::{AppState, ExtractTrigger, LlmConcurrency, log_slow_lock};
use super::boundary::TaskBoundary;
use crate::commands::ai::audit::truncate_chars_ellipsis;
use crate::commands::err_str;
@@ -411,16 +414,21 @@ pub async fn retry_failed_embeddings(state: &AppState) -> anyhow::Result<usize>
/// vector_enabled 关闭或 embed 失败时纯 LIKE(零外部依赖降级)。
/// LIKE 路 query 先经 [`extract_query_keywords`] 分词(整句 LIKE 命中率≈0);
/// 向量路 query 同步改用关键词拼接(虚词稀释 embedding 语义),分词为空时回退原句。
///
/// `project_id`(跨会话召回 D2):Some 时 LIKE 路走项目软排序(同项目知识排前,
/// limit 截断时优先保留);None 与旧路径逐字一致。向量路不做项目过滤(sqlite-vec
/// 无列过滤,软排序在 merge 阶段同样以 keyword 路排序为准)。
async fn hybrid_search(
state: &AppState,
query: &str,
limit: usize,
config: &crate::state::KnowledgeConfig,
project_id: Option<&str>,
) -> Vec<df_storage::models::KnowledgeRecord> {
let keywords = extract_query_keywords(query);
let keyword_results = match state
.knowledge
.search_keywords(&keywords, config.include_candidates, limit)
.search_keywords_scoped(&keywords, config.include_candidates, limit, project_id)
.await
{
Ok(v) => v,
@@ -507,7 +515,10 @@ pub(crate) async fn build_knowledge_context(
if !config.auto_inject {
return String::new();
}
let results = hybrid_search(state, query, 3, config).await;
// 跨会话召回(D2 软排序):当前会话归属项目 → 注入侧同项目知识优先。
// 弱推断失败(None)=全局会话,不过滤,与旧行为逐字一致(回归门)。
let current_project = resolve_source_project(&state.db, conv_id, &[]).await;
let results = hybrid_search(state, query, 3, config, current_project.as_deref()).await;
if results.is_empty() {
return String::new();
}
@@ -713,7 +724,8 @@ pub(crate) async fn maybe_spawn_extraction(
let conv_id = conv_id.to_string();
let provider_config = provider_config.clone();
let llm_concurrency = llm_concurrency.clone();
tauri::async_runtime::spawn(async move {
// 占位兜底同手动路径:panic 出口清 InFlight,防自动提炼永久卡死(见 spawn_with_inflight_guard)。
spawn_with_inflight_guard(conv_id.clone(), async move {
match extract_knowledge_from_conversation(&db, &conv_id, &provider_config, &llm_concurrency).await {
Ok(inserted) => {
if inserted > 0 {
@@ -781,44 +793,96 @@ pub(crate) async fn maybe_spawn_extraction(
///
/// P1 修复(提炼重复触发):手动路径强制清 knowledge_extracted 去重标志,
/// 允许用户手动重提炼(语义=手动覆盖自动去重)。
///
/// panic 兜底包装(治走查🟡):占位(ExtractWatermark::InFlight)置位后,若后续任务体
/// panic(tokio spawn 默认吞 panic 不清状态),水位会永久卡 InFlight,自动提炼从此全跳过
/// 且手动路径也被并发占位拒——无补救通道。AssertUnwindSafe + catch_unwind 保证任何
/// panic 出口都记 Done 清占位(与 Err 出口同语义,保节奏),next 提炼可恢复。提取流水线
/// 持 Arc<Database>/Arc<Mutex> 等跨 unwind 引用,故用 AssertUnwindSafe(非 UnwindSafe)。
fn spawn_with_inflight_guard<F>(conv_id: String, fut: F)
where
F: std::future::Future<Output = ()> + Send + 'static,
{
tauri::async_runtime::spawn(async move {
let cid = conv_id.clone();
let result = futures::future::FutureExt::catch_unwind(AssertUnwindSafe(fut)).await;
if result.is_err() {
tracing::error!(conv_id = %cid, "[knowledge] 提炼任务 panic,清 InFlight 占位恢复水位");
EXTRACT_WATERMARKS
.lock()
.expect("提炼水位互斥锁中毒")
.insert(cid, ExtractWatermark::Done(0));
}
});
}
pub async fn trigger_extraction_now(state: &AppState) -> Result<bool, String> {
let conv_id = {
let mut session = state.ai_session.lock().await;
let conv_id = session.active_conversation_id.clone();
// 强制清哨兵 + 频次水位:手动提炼是用户显式动作,绕过自动路径的间隔限制(语义=手动覆盖)。
// 注意只清 Done(自动节奏水位),InFlight 并发占位保留——清掉会让下方占位检查永假,
// 连点按钮即并发重复提炼(占位语义优先于手动覆盖)。
if let Some(cid) = conv_id.as_deref() {
session.conv(cid).knowledge_extracted = false;
EXTRACT_WATERMARKS
.lock()
.expect("提炼水位互斥锁中毒")
.remove(cid);
let mut wms = EXTRACT_WATERMARKS.lock().expect("提炼水位互斥锁中毒");
if matches!(wms.get(cid), Some(ExtractWatermark::Done(_))) {
wms.remove(cid);
}
}
conv_id
};
let conv_id = conv_id.ok_or_else(|| "当前无活跃对话".to_string())?;
// 并发占位(治走查🟡):连点按钮/双入口(ManualOnly+按钮)此前无占位会并发跑多个提炼任务
// 重复刷 candidate。已 InFlight → 有提炼在跑,返回 false(前端仅刷新列表,无副作用)。
{
let mut wms = EXTRACT_WATERMARKS.lock().expect("提炼水位互斥锁中毒");
if matches!(wms.get(&conv_id), Some(ExtractWatermark::InFlight)) {
tracing::info!(conv_id = %conv_id, "[knowledge] 手动提炼跳过:已有提炼任务进行中");
return Ok(false);
}
wms.insert(conv_id.clone(), ExtractWatermark::InFlight);
}
let provider_config = super::prompt::get_active_provider(state).await.map_err(err_str)?;
let db = state.db.clone();
let session_arc = state.ai_session.clone();
let llm_concurrency = state.llm_concurrency.clone();
tauri::async_runtime::spawn(async move {
spawn_with_inflight_guard(conv_id.clone(), async move {
match extract_knowledge_from_conversation(&db, &conv_id, &provider_config, &llm_concurrency).await {
Ok(inserted) => {
// 手动提炼成功(≥1 条)后与自动路径同语义收口:置哨兵 + 记水位,
// 避免手动提炼后再触发自动提炼重复。用户再次手动按按钮会再次清位,行为自洽。
// 手动提炼后与自动路径同语义收口:≥1 条置哨兵;所有出口记 Done 水位
// (占位已置,任何出口不清位会永久卡死后续自动提炼)。
let mut session = session_arc.lock().await;
let rounds = session
.conv_read(&conv_id)
.map(|c| count_user_rounds(c.messages.iter()))
.unwrap_or(0);
if inserted > 0 {
let mut session = session_arc.lock().await;
session.conv(&conv_id).knowledge_extracted = true;
let rounds = session
.conv_read(&conv_id)
.map(|c| count_user_rounds(c.messages.iter()))
.unwrap_or(0);
EXTRACT_WATERMARKS
.lock()
.expect("提炼水位互斥锁中毒")
.insert(conv_id.clone(), ExtractWatermark::Done(rounds));
}
EXTRACT_WATERMARKS
.lock()
.expect("提炼水位互斥锁中毒")
.insert(conv_id.clone(), ExtractWatermark::Done(rounds));
tracing::info!(
conv_id = %conv_id, inserted, rounds,
"[knowledge] 手动提炼完成,水位记为当前进度,新增用户消息达阈值后才再自动提炼"
);
}
Err(e) => {
tracing::warn!("手动提炼失败(非阻断): {}", e);
// 失败出口同样记 Done 清占位(防 InFlight 永久卡死),保节奏与自动路径一致。
// (只读 conv_read + 写水位表,无需 mut session——原 Err 分支因清哨兵才要 mut)
let session = session_arc.lock().await;
let rounds = session
.conv_read(&conv_id)
.map(|c| count_user_rounds(c.messages.iter()))
.unwrap_or(0);
EXTRACT_WATERMARKS
.lock()
.expect("提炼水位互斥锁中毒")
.insert(conv_id.clone(), ExtractWatermark::Done(rounds));
}
Err(e) => tracing::warn!("手动提炼失败(非阻断): {}", e),
}
});
Ok(true)
@@ -838,6 +902,97 @@ const EXTRACTION_SYSTEM_PROMPT: &str = "你是知识提炼引擎,从 AI 对话
4. 输出纯 JSON,无任何额外文字\n\
5. 最多提取 5 条,按复用价值降序排列(只保留高价值条目,宁缺毋滥)";
/// 项目归属弱推断(跨会话召回最小闭环,设计 D1:a→b 两级,皆无 → None=全局知识)。
///
/// 级 a(高置信):读 `ai_conversations.boundary_json`(V45 无人值守边界)的 `dir_allowlist`
/// 首条目录,反查 projects 表 path 前缀匹配(canonicalize 归一口径对齐 `dir_covered`)。
/// boundary 是显式任务边界声明,视为用户主动绑定项目上下文。
/// 级 b(中置信,B1 拍板:mention 唯一才启用):反向扫描最近消息找 `[项目:名称]` enrichment
/// 标记形态(chat.rs extract_pinned_goal 同款字面量),命中集合恰好唯一时按名称匹配
/// projects.name;零命中或多项目不猜,置 None 防张冠李戴。
///
/// 两级皆不命中 → None(全局知识,注入侧不过滤,行为与存量一致)。
/// DB 查询失败一律降级 None(warn 留痕,非阻断:归属推断失败不该杀死提炼主流程)。
async fn resolve_source_project(db: &Arc<Database>, conv_id: &str, messages: &[ChatMessage]) -> Option<String> {
let project_repo = ProjectRepo::new(db);
let conv_repo = AiConversationRepo::new(db);
// 级 a:boundary_json.dir_allowlist 首条 → projects.path 前缀匹配
if let Ok(Some(conv)) = conv_repo.get_by_id(conv_id).await {
if let Some(bj) = conv.boundary_json.as_deref().filter(|s| !s.is_empty()) {
match serde_json::from_str::<TaskBoundary>(bj) {
Ok(boundary) => {
if let Some(dir) = boundary.dir_allowlist.first().filter(|d| !d.is_empty()) {
match project_repo.list_active().await {
Ok(projects) => {
let dir_norm = normalize_dir_for_match(dir);
// path 前缀匹配:项目根目录 ∈ boundary 白名单目录即归属
// (白名单常填项目根本身或其子目录;取首条命中,白名单多目录
// 跨项目时本就不该猜,取首条是确定性选择)
if let Some(p) = projects.iter().find(|p| {
p.path.as_deref()
.map(|pp| normalize_dir_for_match(pp).starts_with(&dir_norm)
|| dir_norm.starts_with(&normalize_dir_for_match(pp)))
.unwrap_or(false)
}) {
return Some(p.id.clone());
}
}
Err(e) => tracing::warn!(error = %e, conv_id, "[KNOWLEDGE-EXTRACT] 项目列表查询失败,级 a 归属跳过"),
}
}
}
Err(e) => tracing::warn!(error = %e, conv_id, "[KNOWLEDGE-EXTRACT] boundary_json 解析失败,级 a 归属跳过"),
}
}
}
// 级 b(B1:mention 唯一才启用):最近消息扫 `[项目:名称]` 标记,唯一命中才归属
let mut mentioned: Vec<&str> = Vec::new();
for m in messages.iter().rev().take(20) {
if !matches!(m.role, MessageRole::User) {
continue;
}
let mut rest = m.content.as_str();
// 找全部 `[项目:xxx]` / `[项目:xxx]` 标记(半/全角冒号),收名称
while let Some(start) = rest.find("[项目:") {
let after = &rest[start + "[项目:".len()..];
if let Some(end) = after.find(']') {
let name = after[..end].trim();
if !name.is_empty() {
mentioned.push(name);
}
rest = &after[end + 1..];
} else {
break; // 未闭合(可能截断),停扫本条
}
}
}
// 去重后恰好唯一才猜(B1 拍板:多项目 mention 不猜)
mentioned.dedup();
if mentioned.len() == 1 {
let name = mentioned[0];
if let Ok(projects) = project_repo.list_active().await {
if let Some(p) = projects.iter().find(|p| p.name == name) {
return Some(p.id.clone());
}
}
}
None
}
/// 目录形态归一(归属匹配专用,轻量词法版):斜杠统一 + 去尾分隔符 + 小写(Windows 盘符
/// 大小写差异)。不做 canonicalize(提炼路径是后台 best-effort 推断,不付 IO 成本;
/// boundary.dir_covered 已在审批判定侧做过严格归一,此处宽松形态匹配足够)。
fn normalize_dir_for_match(dir: &str) -> String {
let mut s = dir.replace('\\', "/");
while s.len() > 1 && s.ends_with('/') {
s.pop();
}
s.to_lowercase()
}
/// 从对话中提炼知识,产出 candidate 写入知识库
///
/// 流程: 读对话消息 → 过滤 user/assistant 取最近 [`KNOWLEDGE_EXTRACT_WINDOW`] 条 →
@@ -1011,6 +1166,11 @@ async fn extract_knowledge_from_conversation(
let knowledge_repo = KnowledgeRepo::new(db);
let timeline = crate::commands::knowledge_timeline::KnowledgeTimeline::new(db);
// 跨会话召回(D1 弱推断):本轮产出知识归属项目。None=全局知识(行为与存量一致)。
let source_project = resolve_source_project(db, conv_id, &messages).await;
if let Some(pid) = &source_project {
tracing::info!(conv_id, project_id = %pid, "[KNOWLEDGE-EXTRACT] 本轮产出知识归属项目");
}
let mut inserted = 0;
for item in &items {
let kind = match item.get("kind").and_then(|v| v.as_str()) {
@@ -1038,7 +1198,8 @@ async fn extract_knowledge_from_conversation(
confidence,
reuse_count: 0,
verified: false,
source_project: None,
// 跨会话召回(D1):归属项目弱推断结果(级 a boundary/级 b 唯一 mention),None=全局
source_project: source_project.clone(),
// F-260619-04 P1 消息级溯源:conv_msg:{assistant_msg_id} 精确定位本轮 AI 产出。
// last_assistant_msg_id 为 None(老数据无 id)→ 降级 conv:{conv_id} 对话级,
// 展示侧双格式解析兼容。
@@ -1393,4 +1554,102 @@ mod salvage_tests {
assert!(salvage_truncated_json_array(r#"{"kind":"pitfall""#).is_none(), "无完整闭合元素");
assert!(salvage_truncated_json_array("[").is_none());
}
// ---------- 项目归属弱推断(D1 跨会话召回) ----------
use crate::commands::ai::boundary::TaskBoundary as BoundaryForTest;
use df_storage::models::AiConversationRecord;
/// 精简 ChatMessage 构造(user 消息,走 df-ai-core 正典构造器)
fn um(content: &str) -> ChatMessage {
ChatMessage::user(content)
}
/// 插入测试项目
async fn ins_project(db: &Arc<Database>, id: &str, name: &str, path: Option<&str>) {
df_storage::crud::ProjectRepo::new(db)
.insert(df_storage::models::ProjectRecord {
id: id.into(),
name: name.into(),
description: String::new(),
status: df_types::types::ProjectStatus::Planning,
idea_id: None,
path: path.map(|s| s.into()),
stack: None,
archived: false,
created_at: "2026-01-01".into(),
updated_at: "2026-01-01".into(),
})
.await
.unwrap();
}
/// 插入测试会话(带可选 boundary_json)
async fn ins_conv(db: &Arc<Database>, id: &str, boundary: Option<&BoundaryForTest>) {
df_storage::crud::AiConversationRepo::new(db)
.insert(AiConversationRecord {
id: id.into(),
title: Some("t".into()),
messages: "[]".into(),
provider_id: None,
model: None,
models: None,
archived: false,
pinned: false,
prompt_tokens: None,
completion_tokens: None,
pinned_goals: None,
pending_approvals: None,
created_at: "2026-01-01".into(),
updated_at: "2026-01-01".into(),
origin: None,
boundary_json: boundary.map(|b| serde_json::to_string(b).unwrap()),
loop_state: None,
})
.await
.unwrap();
}
/// 目录归一:反斜杠→斜杠 + 去尾分隔 + 小写
#[test]
fn normalize_dir_slash_and_case() {
assert_eq!(normalize_dir_for_match("E:\\wk-lab\\DevFlow\\"), "e:/wk-lab/devflow");
assert_eq!(normalize_dir_for_match("e:/wk-lab/devflow"), "e:/wk-lab/devflow");
assert_eq!(normalize_dir_for_match("/"), "/");
assert_eq!(normalize_dir_for_match(""), "");
}
/// 级 a:boundary_json.dir_allowlist 指向项目根本身 → 归属该项目
#[tokio::test]
async fn resolve_tier_a_boundary_dir_hits_project_root() {
let db = Arc::new(Database::open_in_memory().await.unwrap());
ins_project(&db, "p-boundary", "边界项目", Some("E:/wk-lab/boundary-proj")).await;
let boundary = BoundaryForTest {
risk_ceiling: df_ai::ai_tools::RiskLevel::Medium,
dir_allowlist: vec!["e:\\wk-lab\\boundary-proj\\".into()],
forbid_destructive: true,
budget: None,
};
ins_conv(&db, "conv-a", Some(&boundary)).await;
let got = resolve_source_project(&db, "conv-a", &[]).await;
assert_eq!(got.as_deref(), Some("p-boundary"), "级 a:boundary 白名单目录应归属项目");
}
/// 级 b:唯一 `[项目:名称]` mention 才归属;多项目/零 mention 不猜(B1 拍板)
#[tokio::test]
async fn resolve_tier_b_unique_mention_only() {
let db = Arc::new(Database::open_in_memory().await.unwrap());
ins_project(&db, "p-1", "项目甲", None).await;
ins_project(&db, "p-2", "项目乙", None).await;
// 唯一 mention → 命中
let msgs = vec![um("修一下 [项目: 项目甲] 的列表页")];
assert_eq!(resolve_source_project(&db, "conv-ghost", &msgs).await.as_deref(), Some("p-1"));
// 多项目 mention → 不猜(None)
let msgs = vec![um("[项目:项目甲] 和 [项目:项目乙] 谁对")];
assert_eq!(resolve_source_project(&db, "conv-ghost", &msgs).await, None);
// 零 mention → None
assert_eq!(resolve_source_project(&db, "conv-ghost", &[um("普通消息")]).await, None);
}
}