修复: DeepSeek 400 全量扫描 + 队列 per-conv 隔离

- openai_compat: 扫描所有 assistant 消息剥离 orphan tool_calls(原仅查末条)
- queue 加 conversationId 字段,按会话精准 drain
- regenerate/editMessage 只清本会话排队消息
- newConversation 保留旧会话排队消息
- AiError 只清出错会话的队列项
This commit is contained in:
2026-07-20 00:18:30 +08:00
parent 42efb31bbf
commit e9e3578d26
59 changed files with 2875 additions and 1330 deletions

View File

@@ -4,6 +4,7 @@ use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use tauri::{AppHandle, Emitter, Manager};
use tokio::sync::Mutex;
use df_ai::ai_tools::{AiToolRegistry, RiskLevel};
use df_ai::provider::ChatMessage;
@@ -62,10 +63,10 @@ pub use restore::restore_pending_approvals;
mod finalize;
pub(crate) use finalize::audit_finalize;
// cache(audit/cache.rs):F-260616-05 高危工具去重缓存。
// 第三批从本文件抽离行为零变更。
// cache(audit/cache.rs):F-260616-05 高危工具去重缓存 + BUG-2026-07-07/P0-2 只读工具缓存
// 第三批从本文件抽离,行为零变更(P0-2 为新增能力)
mod cache;
pub(super) use cache::pending_placeholder_for;
pub(super) use cache::{find_cached_readonly_result, pending_placeholder_for};
// data_change(audit/data_change.rs):AR-11 数据变更联动刷新。
// 第四批从本文件抽离行为零变更。pub(crate) use 保持 emit_data_changed 对 crate 内可见
@@ -136,11 +137,25 @@ async fn execute_with_heartbeat(
});
// RAIIexecute 无论 Ok/Err/panicunwind_guard drop 自动 stop+abort无心跳 task 泄漏。
let _guard = HeartbeatGuard { stop, handle: heartbeat };
tools.execute(name, args).await
// BUG-2026-07-19: tools.execute 无 timeout 时,卡死工具(run_command 长命令/read_file 大文件/同步
// 阻塞工具)永久挂起 → process_tool_calls 持 session lock 永久 → guard.reset 等 lock → AiCompleted
// 永不发 → 前端"回答完卡住/超时清空"。60s timeout 兜底:超时返错误 tool_result,锁释放,loop 续跑。
// 心跳 30s 续命前端 watchdog,60s timeout 覆盖绝大多数工具(run_command 已自带 10s 子超时)。
match tokio::time::timeout(Duration::from_secs(60), tools.execute(name, args)).await {
Ok(result) => result,
Err(_elapsed) => {
tracing::error!(
conv_id = %conv_id,
tool = %name,
"[ai] 工具执行超时(60s),返回错误 tool_result(防 process_tool_calls 持 session lock 永久卡死)"
);
Err(anyhow::anyhow!("工具执行超时(60s),已中止(防死锁)"))
}
}
}
pub(crate) async fn process_tool_calls(
session: &mut AiSession,
session_arc: &Arc<Mutex<AiSession>>,
tool_calls_acc: HashMap<u32, ToolCallDraft>,
tools_arc: &Arc<AiToolRegistry>,
db: &Arc<Database>,
@@ -164,9 +179,33 @@ pub(crate) async fn process_tool_calls(
// push 到 per_conv.messagesaudit/mod.rs:882此处取末条 assistant id 作为本轮工具
// 调用所属的溯源 message_id贯穿所有 audit_tool_call 写入。None 表示无 assistant 消息
// (异常路径/老数据无 idaudit 落 message_id=None展示侧兼容。
let current_message_id: Option<String> = session
.conv_read(conv_id)
.and_then(|c| c.messages.last_assistant_message_id());
//
// BUG-260624-03/P0 重构:短 lock 段读 current_message_id(纯读,无 await),clone 出来后续传参。
// 整个 process_tool_calls 不再要求调用方持锁,内部所有 session.xxx 访问均短 lock 段,
// 慢操作(execute_with_heartbeat/audit_tool_call/detect_retry_count/DB 查)全在锁外 await。
// BUG-2026-07-19: current_message_id 读加 200ms timeout(防 session lock 竞争卡死致
// process_tool_calls 进不去 emit AiToolCallStarted → 前端工具卡片不呈现 + 45s 看门狗超时)。
// 超时用 None(溯源降级,非致命)。治本:消除 session lock 长持有(save clone 段已优化)。
let current_message_id: Option<String> = match tokio::time::timeout(
std::time::Duration::from_millis(200),
async {
let __lock_t = std::time::Instant::now();
let session = session_arc.lock().await;
let __id = session.conv_read(conv_id)
.and_then(|c| c.messages.last_assistant_message_id());
let __hold = __lock_t.elapsed();
if __hold > std::time::Duration::from_millis(30) {
eprintln!("[LOCK-SLOW] process_tool_calls:192 持锁 {:?} (含 lock 等待)", __hold);
}
__id
},
).await {
Ok(id) => id,
Err(_) => {
tracing::warn!(conv_id = %conv_id, "[ai] process_tool_calls current_message_id lock 200ms 超时,溯源降级 None(session lock 竞争)");
None
}
};
let current_message_id = current_message_id.as_deref();
// 解析 args + 批量发 Started前端骨架按原始 index 顺序展示)
@@ -218,8 +257,22 @@ pub(crate) async fn process_tool_calls(
// Phase C: Denied 路径 → 硬拒push 错误 tool_result + emit Completed不挂起 loop。
// 工具返 Err 让 LLM 知路径被禁自行调整loop 继续下一轮(不暂停)。
for (draft, reason) in path_denied {
let err_msg = format!("路径授权拒绝: {}", reason);
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &err_msg));
// BUG-2026-07-07/P1-3:拒绝消息改结构化 JSON,防裸文本破坏 tool role 消息协议
let err_msg = serde_json::json!({
"status": "rejected",
"reason": "path_blacklist",
"message": format!("路径授权拒绝: {}", reason)
}).to_string();
// 短 lock 段:push tool_result(纯写,无 await)
{
let __lock_t = std::time::Instant::now();
let mut session = session_arc.lock().await;
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &err_msg));
let __hold = __lock_t.elapsed();
if __hold > std::time::Duration::from_millis(30) {
eprintln!("[LOCK-SLOW] process_tool_calls:262 持锁 {:?} (含 lock 等待)", __hold);
}
}
// L3 emit 双写:路径黑名单拒绝 emit Completed 双路发布。
let ev = AiChatEvent::AiToolCallCompleted {
id: draft.id.clone(),
@@ -240,6 +293,9 @@ pub(crate) async fn process_tool_calls(
// Phase B: NeedsAuth 路径 → 挂起 loopinsert pending + emit AiDirAuthRequired + push 占位 tool_result
// 复用审批挂起架构pending_approvals 以 tool_call_id 为键ai_authorize_dir IPC remove 后恢复。
// pending_count 计入(让 agentic loop 检测到挂起并暂停,等 ai_authorize_dir → try_continue 恢复)。
//
// BUG-260624-03/P0 重构:detect_retry_count(DB 读)/emit/audit_tool_call(DB 写)均在锁外,
// 仅 push tool_result + pending_approvals.insert 两处纯写改短 lock 段。
for (draft, args, req) in path_auth_pending {
pending_count += 1;
// L1 补丁req.dirs 含所有未授权父目录emit 用首个作主展示目录(前端弹窗主显)。
@@ -248,14 +304,23 @@ pub(crate) async fn process_tool_calls(
.map(|d| d.to_string_lossy().to_string())
.unwrap_or_default();
let path_str = req.raw_paths.first().cloned().unwrap_or_default();
// 阶段4容错/恢复,开关 df-ai-approval-retry同 tc_id 重试检测。
// 锁外:阶段4容错/恢复,开关 df-ai-approval-retry同 tc_id 重试检测。
let retry_count = detect_retry_count(&audit_repo, &draft.id).await;
if retry_count >= 1 {
let skip_msg = format!(
"已跳过重试(同 tool_call_id={} 此前已审批执行过,防 LLM 死循环重试同卡死工具)",
draft.id
);
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &skip_msg));
// 短 lock 段:push tool_result(纯写)
{
let __lock_t = std::time::Instant::now();
let mut session = session_arc.lock().await;
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &skip_msg));
let __hold = __lock_t.elapsed();
if __hold > std::time::Duration::from_millis(30) {
eprintln!("[LOCK-SLOW] process_tool_calls:305 持锁 {:?} (含 lock 等待)", __hold);
}
}
let ev = AiChatEvent::AiToolCallCompleted {
id: draft.id.clone(),
result: serde_json::Value::String(skip_msg.clone()),
@@ -271,22 +336,31 @@ pub(crate) async fn process_tool_calls(
).await;
continue;
}
session.pending_approvals.insert(
draft.id.clone(),
PendingApproval {
tool_call_id: draft.id.clone(),
tool_name: draft.name.clone(),
arguments: args.clone(),
conversation_id: Some(conv_id.to_string()),
recovered: false,
// 阶段3a路径授权挂起标 kind=Path(req)(下沉原 path_auth 字段)。
kind: ApprovalKind::Path(req),
retry_count,
created_at: Some(std::time::SystemTime::now()),
},
);
// 占位 tool_result与 RiskLevel 审批一致ai_authorize_dir 批准后替换为真实结果。
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &pending_placeholder_for(&draft.id)));
// 短 lock 段:pending_approvals.insert + 占位 tool_result push(两处纯写,无 await)
{
let __lock_t = std::time::Instant::now();
let mut session = session_arc.lock().await;
session.pending_approvals.insert(
draft.id.clone(),
PendingApproval {
tool_call_id: draft.id.clone(),
tool_name: draft.name.clone(),
arguments: args.clone(),
conversation_id: Some(conv_id.to_string()),
recovered: false,
// 阶段3a路径授权挂起标 kind=Path(req)(下沉原 path_auth 字段)。
kind: ApprovalKind::Path(req),
retry_count,
created_at: Some(std::time::SystemTime::now()),
},
);
// 占位 tool_result与 RiskLevel 审批一致ai_authorize_dir 批准后替换为真实结果。
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &pending_placeholder_for(&draft.id)));
let __hold = __lock_t.elapsed();
if __hold > std::time::Duration::from_millis(30) {
eprintln!("[LOCK-SLOW] process_tool_calls:325 持锁 {:?} (含 lock 等待)", __hold);
}
}
tracing::debug!(target: "ai_dirauth", conv = %conv_id, tool = %draft.name, tc_id = %draft.id, "emit AiDirAuthRequired路径授权挂起等 ai_authorize_dir 恢复)");
let ev = AiChatEvent::AiDirAuthRequired {
id: draft.id.clone(),
@@ -351,17 +425,17 @@ pub(crate) async fn process_tool_calls(
let risk_level = tools_arc.get(&draft.name).map(|t| t.risk_level).unwrap_or(RiskLevel::High);
handle_approval_tool(
draft, args, risk_level, &auto_exec_mode,
session, conv_id, tools_arc, &audit_repo,
session_arc, conv_id, tools_arc, &audit_repo,
app_handle, db, &mut low_risk, &mut trust_hits,
&mut pending_count, current_message_id,
).await;
}
// AE-04 trust-hit 并行执行execute + 即时 emit 在闭包内(闭包不访问 session,非"锁已释放"——
// session 锁仍由调用方 agentic/mod.rs 持有至 process_tool_calls 返回CR-53 审查纠正原"移锁外"误述),
// push tool_result / audit 在 join_all 后串行回填(持锁)。对齐 Low risk 并行模式。
// AE-04 trust-hit 并行执行execute + 即时 emit 在闭包内(闭包不访问 session,
// BUG-260624-03/P0 重构后调用方不再持锁,本段所有 await 完全在锁外),
// push tool_result / audit 在 join_all 后串行回填(短 lock push + 锁外 audit)。对齐 Low risk 并行模式。
// CR-51 修:原 inline .await execute 串行执行每个工具阻塞期间锁被持有run_command 慢命令
// 阻塞同会话 IPC改 join_all 并行多工具减少总阻塞时间(锁持有时长不变,并行化降阻塞
// 阻塞同会话 IPC改 join_all 并行多工具减少总阻塞时间。P0 重构后锁外并行,根本消除阻塞。
// join_all 保序——结果顺序 = trust_hits 输入顺序 = tc_list 原始 index 顺序,不额外 sort
if !trust_hits.is_empty() {
let results: Vec<(ToolCallDraft, RiskLevel, Result<String, String>)> =
@@ -403,23 +477,68 @@ pub(crate) async fn process_tool_calls(
}
})).await;
// 串行回填 tool_result + 审计(持 session 锁
// 串行回填 tool_result + 审计(短 lock push + 锁外 audit
for (draft, risk_level, outcome) in results {
let (status, content) = match outcome {
Ok(c) => ("completed", c),
Err(c) => ("failed", c),
};
// F-260616-09 B 批2写 per_conv.messages。
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, content.clone()));
// 审计trust 放行仍记一条decided_by=auto_trust留痕可追溯
// 短 lock 段:push tool_result(纯写,无 await)
{
let mut session = session_arc.lock().await;
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, content.clone()));
}
// 锁外:审计(trust 放行仍记一条,decided_by=auto_trust),留痕可追溯
audit_tool_call(&audit_repo, conv_id, &draft.id, &draft.name, &draft.args, status, risk_level, Some(content), Some("auto_trust"), current_message_id).await;
}
}
// Low 风险并行执行execute + 即时 emit 在闭包内(不持 session 锁),
// push tool_result / audit 在 join_all 后串行回填(持锁,与 Med/High 占位拼接)。
// push tool_result / audit 在 join_all 后串行回填(短 lock push + 锁外 audit,与 Med/High 占位拼接)。
// join_all 保序——结果顺序 = low_risk 输入顺序 = tc_list 原始 index 顺序,不额外 sort
if !low_risk.is_empty() {
// BUG-2026-07-07/P0-2:只读幂等工具去重缓存(治 LLM 死循环重调)。
// 实测 9357c27c 会话:LLM 对 read_file(application.xml) 连调 17 次、search_files 连调 13 次,
// 每次返回几乎相同结果却反复重调,是 prompt 64 万的直接元凶。此处先查会话内是否已对
// 同参只读工具成功执行过,命中则直接回填缓存结果跳过真执行,断 LLM 失忆死循环。
// 安全边界见 find_cached_readonly_result 文档(仅白名单只读工具 + 仅 completed 成功结果)。
//
// BUG-260624-03/P0 重构:find_cached_readonly_result 内部短 lock + 锁外 DB 查,本段不持锁。
let mut low_risk_uncached: Vec<(ToolCallDraft, serde_json::Value, RiskLevel)> = Vec::with_capacity(low_risk.len());
for (draft, args, risk_level) in low_risk {
let cached = find_cached_readonly_result(session_arc, conv_id, &audit_repo, &draft.name, &args).await;
if let Some(cached_content) = cached {
// 缓存命中:直接 push tool_result + 审计(decided_by=cache_hit 标记缓存来源),
// 不走真执行 + 不重emit Started/Completed(避免误导前端工具又执行了一次)。
// emit Completed 携带缓存结果供前端折叠卡片展示(与 find_cached_high_risk_result 一致)。
let ev = AiChatEvent::AiToolCallCompleted {
id: draft.id.clone(),
result: serde_json::Value::String(cached_content.clone()),
conversation_id: Some(conv_id.to_string()),
};
let _ = app_handle.emit("ai-chat-event", ev.clone());
let _ = app_handle.state::<crate::state::AppState>().ai_event_bus.publish_event(ev);
// 短 lock 段:push tool_result(纯写,无 await)
{
let mut session = session_arc.lock().await;
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, &cached_content));
}
audit_tool_call(
&audit_repo, conv_id, &draft.id, &draft.name, &draft.args,
"completed", risk_level, Some(cached_content), Some("cache_hit"),
current_message_id,
).await;
tracing::info!(
conv_id = %conv_id,
tool = %draft.name,
tc_id = %draft.id,
"[ai] 只读工具缓存命中,跳过真执行(治 LLM 死循环重调)"
);
} else {
low_risk_uncached.push((draft, args, risk_level));
}
}
let low_risk = low_risk_uncached;
// 携带 args + 原始 JSON resultcreate_idea source 补全需解析 result.id + args.source
let results: Vec<(ToolCallDraft, serde_json::Value, RiskLevel, Result<String, String>)> =
futures::future::join_all(low_risk.into_iter().map(|(draft, args, risk_level)| {
@@ -464,7 +583,7 @@ pub(crate) async fn process_tool_calls(
}
})).await;
// 串行回填 tool_result + 审计(持 session 锁
// 串行回填 tool_result + 审计(短 lock push + 锁外 audit
for (draft, raw_result, risk_level, outcome) in results {
let (status, content) = match outcome {
Ok(c) => ("completed", c),
@@ -477,8 +596,11 @@ pub(crate) async fn process_tool_calls(
let args_val = serde_json::from_str(&draft.args).unwrap_or(serde_json::Value::Null);
maybe_fill_idea_source(db, &draft.name, &args_val, &raw_result, current_message_id).await;
}
// F-260616-09 B 批2写 per_conv.messages
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, content.clone()));
// 短 lock 段:写 per_conv.messages(纯写,无 await)
{
let mut session = session_arc.lock().await;
session.conv(conv_id).messages.push(ChatMessage::tool_result(&draft.id, content.clone()));
}
// F-#97 审计留痕low_risk 向量中 risk_level != Low 即 mode 放行Med 仅 medium/all、High 仅 all 才进)。
// decided_by 区分接管来源,审计表可追溯 all 模式执行了多少高危命令(治 securityReview blocker:
// 原硬编码 RiskLevel::Low 致 Med/High 接管操作全标 Low/auto 失真,等于没记)。
@@ -492,12 +614,18 @@ pub(crate) async fn process_tool_calls(
}
// 阶段3a单表 pending_approvals 统计 pending 总数path/risk 合一kind 区分)。
let (path_count, risk_count) = session.pending_approvals.values()
.fold((0usize, 0usize), |(p, r), a| match a.kind {
ApprovalKind::Path(_) => (p + 1, r),
ApprovalKind::Risk { .. } => (p, r + 1),
});
tracing::debug!(target: "ai_dirauth", pending_count, pending_approvals_len = session.pending_approvals.len(), path_count, risk_count, "process_tool_calls 返回(路径/风险挂起分布)");
// 短 lock 段:读 pending_approvals 统计(纯读,无 await)
let (path_count, risk_count, pending_approvals_len) = {
let session = session_arc.lock().await;
let len = session.pending_approvals.len();
let (p, r) = session.pending_approvals.values()
.fold((0usize, 0usize), |(p, r), a| match a.kind {
ApprovalKind::Path(_) => (p + 1, r),
ApprovalKind::Risk { .. } => (p, r + 1),
});
(p, r, len)
};
tracing::debug!(target: "ai_dirauth", pending_count, pending_approvals_len, path_count, risk_count, "process_tool_calls 返回(路径/风险挂起分布)");
pending_count
}