修复: DeepSeek 400 全量扫描 + 队列 per-conv 隔离
- openai_compat: 扫描所有 assistant 消息剥离 orphan tool_calls（原仅查末条） - queue 加 conversationId 字段，按会话精准 drain - regenerate/editMessage 只清本会话排队消息 - newConversation 保留旧会话排队消息 - AiError 只清出错会话的队列项
This commit is contained in:
@@ -1,11 +1,11 @@
|
||||
//! 流式接收 LLM 响应
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Duration;
|
||||
|
||||
use tauri::{AppHandle, Emitter, Manager};
|
||||
use futures::StreamExt;
|
||||
use tracing::warn;
|
||||
|
||||
use df_ai::provider::{CompletionRequest, LlmProvider};
|
||||
@@ -21,39 +21,30 @@ use super::{AiChatEvent, ToolCallDraft};
|
||||
/// timeout/connect 关键词分类。
|
||||
///
|
||||
/// 返回 `(status_or_class, raw)`：status 取文本中首个三位数；否则按关键词给 timeout/connect/unknown。
|
||||
fn extract_error_diag(e: &anyhow::Error) -> (String, String) {
|
||||
let raw = e.to_string();
|
||||
|
||||
// 1) 抠 HTTP 状态码：匹配 provider bail 串里的 "错误 4xx/5xx" 或 reqwest 的 "HTTP status"。
|
||||
// R-P2-6:原实现按字节窗口 `&bytes[i..i+3]` 切片——若窗口恰好切在多字节 UTF-8 字符中间会 panic
|
||||
// (依赖中文恰好 3 字节、状态码恰为 ascii 的巧合)。改为按 char 迭代 + 前后非数字边界判断,
|
||||
// 既 UTF-8 安全又顺便修复"长数字串(如端口号 14012)内嵌 401 误命中"的潜在问题。
|
||||
// 白名单码(首位 4/5,故只查 4xx/5xx 区段,减少无效匹配):
|
||||
/// 从字符串(非 anyhow)提取诊断信息,用于 mpsc channel 中来自专用线程的错误字符串。
|
||||
fn extract_error_diag_from_str(raw: &str) -> (String, String) {
|
||||
// 1) 抠 HTTP 状态码
|
||||
const HTTP_CODES: &[&str] = &[
|
||||
"400", "401", "403", "404", "408", "409", "413",
|
||||
"422", "429", "500", "502", "503", "504",
|
||||
];
|
||||
// 把 raw 按 char 收集,索引即 char 下标(非字节),边界判断用 char 安全
|
||||
let chars: Vec<char> = raw.chars().collect();
|
||||
let n = chars.len();
|
||||
let mut i = 0;
|
||||
while i + 3 <= n {
|
||||
// 窗口必须是三个 ascii 数字
|
||||
if chars[i].is_ascii_digit() && chars[i + 1].is_ascii_digit() && chars[i + 2].is_ascii_digit() {
|
||||
let code: String = chars[i..i + 3].iter().collect();
|
||||
// 前后边界必须非数字(否则会从端口号 14012 里抠出 401)
|
||||
let prev_ok = i == 0 || !chars[i - 1].is_ascii_digit();
|
||||
let next_ok = i + 3 == n || !chars[i + 3].is_ascii_digit();
|
||||
if prev_ok && next_ok && HTTP_CODES.contains(&code.as_str()) {
|
||||
return (format!("HTTP {}", code), raw);
|
||||
return (format!("HTTP {}", code), raw.to_string());
|
||||
}
|
||||
i += 3; // 已确认是三连数字,跳过避免窗口重叠重复扫
|
||||
i += 3;
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
// 2) 传输层分类（reqwest Display 文本特征），不命名 reqwest 类型
|
||||
let lower = raw.to_lowercase();
|
||||
let class = if lower.contains("timeout") || lower.contains("超时") {
|
||||
"timeout"
|
||||
@@ -64,13 +55,20 @@ fn extract_error_diag(e: &anyhow::Error) -> (String, String) {
|
||||
} else {
|
||||
"unknown"
|
||||
};
|
||||
(class.to_string(), raw)
|
||||
(class.to_string(), raw.to_string())
|
||||
}
|
||||
|
||||
/// 从 anyhow::Error 提取诊断信息
|
||||
#[allow(dead_code)]
|
||||
fn extract_error_diag(e: &anyhow::Error) -> (String, String) {
|
||||
extract_error_diag_from_str(&e.to_string())
|
||||
}
|
||||
|
||||
/// AiError 诊断消息的上下文，区分「建连/首字节失败」与「流中途断」两类。
|
||||
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
|
||||
pub(crate) enum DiagKind {
|
||||
/// `provider.stream()` 直接返回 Err：连接/鉴权/HTTP non-2xx 等
|
||||
#[allow(dead_code)]
|
||||
Init,
|
||||
/// 流已建立，next() 返回 Err：SSE 传输断/解析错等
|
||||
MidStream,
|
||||
@@ -144,282 +142,335 @@ pub(crate) enum StreamResult {
|
||||
/// 前端 watchdog 据此 reset，区分「LLM 在跑」与「真断」（避免空气泡误报中断）
|
||||
/// - `stop_notify.notified()`：用户点停止即时打断，不再等 chunk 到或 120s idle timeout
|
||||
pub(crate) async fn stream_llm(
|
||||
provider: &dyn LlmProvider,
|
||||
provider: Arc<dyn LlmProvider>,
|
||||
request: CompletionRequest,
|
||||
app_handle: &AppHandle,
|
||||
stop_flag: &AtomicBool,
|
||||
notify: &tokio::sync::Notify,
|
||||
conv_id: &str,
|
||||
) -> StreamResult {
|
||||
/// 流式读取空闲超时：超过此时长无任何 chunk 即判定连接已断
|
||||
const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
/// 流式读取空闲超时:超过此时长无任何 chunk 即判定连接已断
|
||||
/// BUG-2026-07-17:原 120s 太长,后端挂死时用户等太久。改为 45s,正常流式每 chunk 间隔
|
||||
/// 远不到 45s(即使模型思考间隙),超时即判为连接异常走保文/报错路径
|
||||
const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
/// BUG-2026-07-08: 首 chunk 超时(等首字节)。
|
||||
/// 正常 provider 首 token <3s(即使 48 工具定义 + 长上下文),10s 足够宽容;
|
||||
/// 已有数据后中途静默仍用 STREAM_IDLE_TIMEOUT(120s,允许模型思考间隙)。
|
||||
const FIRST_CHUNK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// 心跳间隔：静默期向前端报「LLM 仍在跑」，reset watchdog
|
||||
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
match provider.stream(request).await {
|
||||
Ok(mut stream) => {
|
||||
let mut full_text = String::new();
|
||||
let mut tool_calls_acc: HashMap<u32, ToolCallDraft> = HashMap::new();
|
||||
let mut finished_received = false;
|
||||
let mut stopped = false;
|
||||
let mut final_usage: Option<df_ai::provider::TokenUsage> = None;
|
||||
// BUG-260617-12: DeepSeek thinking 模式推理内容累积（多轮需回传）
|
||||
let mut reasoning_content_acc: Option<String> = None;
|
||||
// ================================================================
|
||||
// BUG-2026-07-17 根治: 整个 LLM 流式请求运行在独立 OS 线程的
|
||||
// 专用 tokio runtime 上,通过 mpsc channel 将 chunk 流式送回主 runtime。
|
||||
// ================================================================
|
||||
//
|
||||
// 根因: Tauri 在 Windows 上使用单线程 tokio runtime。provider.stream()
|
||||
// 内部(eventsource-stream/reqwest TLS/网络建连)存在同步阻塞操作,
|
||||
// 阻塞唯一工作线程导致 tokio 计时器系统无法推进,timeout 不触发。
|
||||
//
|
||||
// 方案:
|
||||
// 1. 独立 OS 线程 + 多线程 tokio runtime 执行完整 LLM 请求
|
||||
// (provider.stream + stream.next 循环),隔离同步阻塞。
|
||||
// 2. 通过 mpsc channel 将 StreamChunk 逐块传回主 runtime。
|
||||
// 3. reqwest Response/Stream 的生命周期不跨 runtime,避免
|
||||
// "error decoding response body"(runtime 状态不兼容)。
|
||||
//
|
||||
// 资源: 专用线程运行至 LLM 流结束或 mpsc receiver 被 drop(超时/停止)。
|
||||
let (chunk_tx, mut chunk_rx) = tokio::sync::mpsc::channel::<
|
||||
std::result::Result<df_ai::provider::StreamChunk, String>
|
||||
>(64);
|
||||
let provider_for_thread = provider.clone();
|
||||
let conv_id_owned = conv_id.to_string();
|
||||
|
||||
// B-260615-15:heartbeat interval 提至 loop 外复用,避免每轮重建计时器
|
||||
// (每轮重建会丢已积累的节拍,且 interval 首次 tick 立即返回的特性会被误用)。
|
||||
// tokio interval 首 tick 立即返回——此处先丢弃首 tick,让心跳等满首个 30s 静默期才发
|
||||
// (心跳语义是"静默期仍在跑",循环入口立即报无意义且会与 stream.next() 抢分支错过首 chunk)。
|
||||
let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL);
|
||||
heartbeat.tick().await;
|
||||
|
||||
loop {
|
||||
// B-260615-04:stop 即时打断。stream.next() 阻塞等 chunk 时,
|
||||
// 用户点停止需等 chunk 到或 120s idle timeout 才轮到此处检查——
|
||||
// 合并到下方 select! 的 stop_notify 分支后此处为快路径(非阻塞首检)。
|
||||
// stop_flag 可能被 stopChat() 在 select! 阻塞期间置位,
|
||||
// select! 的 stop_notify 分支会唤醒；此处保留作冗余快检(非阻塞)。
|
||||
if stop_flag.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
stopped = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// 三分支 select!(B-260615-02 + B-260615-04 合并):
|
||||
// 1) stream.next():正常 chunk(idle timeout 120s 包裹,真断返回 InitFailed 交调用方处理)
|
||||
// 2) heartbeat.tick():静默期发 AiHeartbeat reset 前端 watchdog
|
||||
// 3) stop_notify.notified():用户停止即时打断
|
||||
//
|
||||
// 注意:stop_flag 是 AtomicBool 无 async 通知能力——
|
||||
// 此处用「timeout 包 stream.next() + 进入循环前/后查 stop_flag + 循环内 30s 心跳 tick」
|
||||
// 间接实现「≤30s 感知 stop」(每轮 select! 至多 120s,但心跳 tick 30s 一次会
|
||||
// 触发 select! 返回 → 循环回顶部 stop_flag 快检)。无 Notify 依赖,改动最小。
|
||||
tokio::select! {
|
||||
// 1) 正常 chunk(idle timeout 包裹)
|
||||
chunk_result = tokio::time::timeout(STREAM_IDLE_TIMEOUT, stream.next()) => {
|
||||
match chunk_result {
|
||||
Err(_elapsed) => {
|
||||
// UX-2025-04 / CR-30-2 / 决策 F-260616-07 a1:
|
||||
// idle timeout 属 MidStream 类失败——已有文本则保文(Partial),不重试。
|
||||
// 空文本无文可保,UX-260618-15: 不再 emit AiError(retryable=true 重试路径,
|
||||
// emit 权交 agentic 重试耗尽时统一发,避免 N 次重试 N+1 气泡)。
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: "流式响应超时（120 秒无数据，连接可能已断开）".to_string(),
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流中途 idle timeout,保文不重试(incomplete)",
|
||||
);
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
Ok(None) => break, // 流正常结束
|
||||
Ok(Some(chunk_result)) => match chunk_result {
|
||||
Ok(chunk) => {
|
||||
if !chunk.delta.is_empty() {
|
||||
full_text.push_str(&chunk.delta);
|
||||
// L3 emit 双写:提变量避免构造两次,publish 到事件总线(EVENT_BUS_ENABLED 门控在 publish 内)。
|
||||
let ev = AiChatEvent::AiTextDelta {
|
||||
delta: chunk.delta,
|
||||
conversation_id: Some(conv_id.to_string()),
|
||||
};
|
||||
let _ = app_handle.emit("ai-chat-event", ev.clone());
|
||||
let _ = app_handle.state::<crate::state::AppState>().ai_event_bus.publish_event(ev);
|
||||
}
|
||||
if let Some(tc_deltas) = &chunk.tool_calls {
|
||||
for tc_delta in tc_deltas {
|
||||
let draft = tool_calls_acc.entry(tc_delta.index).or_default();
|
||||
if let Some(id) = &tc_delta.id { draft.id = id.clone(); }
|
||||
if let Some(name) = &tc_delta.function_name { draft.name.push_str(name); }
|
||||
if let Some(args) = &tc_delta.function_arguments { draft.args.push_str(args); }
|
||||
}
|
||||
}
|
||||
if let Some(u) = &chunk.usage {
|
||||
final_usage = Some(u.clone());
|
||||
}
|
||||
// BUG-260617-12: DeepSeek thinking 模式推理内容累积
|
||||
if let Some(ref rc) = chunk.reasoning_content {
|
||||
reasoning_content_acc.get_or_insert_with(String::new).push_str(rc);
|
||||
}
|
||||
// provider 流式错误事件（Anthropic SSE `type=="error"` 等）：
|
||||
// UX-2025-04 / CR-30-2 / 决策 a1: MidStream 类失败——已有文本则保文(Partial),
|
||||
// 不重试。空文本无文可保,返回 InitFailed(保守 retryable=true,
|
||||
// err_msg 不可靠解析状态码,默认按可重试交调用方决定)。
|
||||
if let Some(err_msg) = &chunk.error {
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
error = %err_msg,
|
||||
text_len = full_text.len(),
|
||||
"[ai] provider 流式错误事件",
|
||||
);
|
||||
// UX-2025-04 / CR-30-2 / 决策 a1: MidStream 类失败——已有文本则保文(Partial),
|
||||
// 不重试。空文本无文可保,UX-260618-15: 不再 emit AiError(保守 retryable=true,
|
||||
// emit 权交 agentic 重试耗尽时统一发,避免 N 次重试 N+1 气泡)。
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: fmt_diag(
|
||||
provider.name(),
|
||||
DiagKind::MidStream,
|
||||
"stream-error",
|
||||
err_msg,
|
||||
),
|
||||
};
|
||||
}
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
if chunk.finished {
|
||||
finished_received = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// 诊断：流中途错误（多为 SSE 传输断），补 provider 标识 + HTTP 状态/分类 + 原始文本
|
||||
let (status_or_class, raw) = extract_error_diag(&e);
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
status = %status_or_class,
|
||||
conv_id = %conv_id,
|
||||
error = %raw,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流式接收中途错误",
|
||||
);
|
||||
// UX-2025-04 / CR-30-2 / 决策 F-260616-07 a1: MidStream chunk Err——
|
||||
// 已有文本则保文(Partial),不重试。空文本无文可保,返回
|
||||
// InitFailed{retryable=classify_status_or_class(status_or_class)}
|
||||
// (4xx Fatal 立即放弃,5xx/429/timeout/connect 可重试)。
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
// UX-260618-15: 不再 emit AiError。retryable=true 走重试路径,
|
||||
// retryable=false(4xx Fatal)也交 agentic Fatal 分支统一 emit
|
||||
// (InitFailed 携带 retryable + error,agentic 据此决定重试或 Fatal emit)。
|
||||
return StreamResult::InitFailed {
|
||||
retryable: classify_status_or_class(&status_or_class),
|
||||
error: fmt_diag(
|
||||
provider.name(),
|
||||
DiagKind::MidStream,
|
||||
&status_or_class,
|
||||
&raw,
|
||||
),
|
||||
};
|
||||
}
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2) 心跳:静默期 30s 发 AiHeartbeat,前端 watchdog reset(B-260615-02)
|
||||
_ = heartbeat.tick() => {
|
||||
// 心跳只在「仍在等下一 chunk」时有意义——若 stop_flag 已置,顶部快检会 break,无需发心跳
|
||||
// L3 emit 双写:提变量避免构造两次,publish 到事件总线(EVENT_BUS_ENABLED 门控在 publish 内)。
|
||||
let ev = AiChatEvent::AiHeartbeat {
|
||||
conversation_id: Some(conv_id.to_string()),
|
||||
std::thread::spawn(move || {
|
||||
let rt = match tokio::runtime::Runtime::new() {
|
||||
Ok(rt) => rt,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
conv_id = %conv_id_owned,
|
||||
"[ai] 创建专用 tokio runtime 失败: {}",
|
||||
e,
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
// BUG-2026-07-17 双重保险:在专用 runtime 内部也加 120s 超时。
|
||||
// 即使 provider.stream() 在专用线程上意外阻塞(理论不应发生,因专用 runtime
|
||||
// 多线程可推进独立 timer wheel),此 timeout 保证线程最终退出不永久挂起。
|
||||
rt.block_on(async move {
|
||||
use futures::StreamExt;
|
||||
let stream_result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(120),
|
||||
provider_for_thread.stream(request),
|
||||
).await;
|
||||
match stream_result {
|
||||
Ok(Ok(mut stream)) => {
|
||||
while let Some(chunk_result) = stream.next().await {
|
||||
let sent = match chunk_result {
|
||||
Ok(chunk) => chunk_tx.send(Ok(chunk)).await,
|
||||
Err(e) => chunk_tx.send(Err(e.to_string())).await,
|
||||
};
|
||||
let _ = app_handle.emit("ai-chat-event", ev.clone());
|
||||
let _ = app_handle.state::<crate::state::AppState>().ai_event_bus.publish_event(ev);
|
||||
}
|
||||
// 3) 即时停止唤醒(B-260615-14):用户点 stop → ai_chat_stop 置 stop_flag 后 notify_one()。
|
||||
// stream.next() 正阻塞等 chunk 时立即被唤醒,不再等 30s 心跳 tick 或 120s idle timeout。
|
||||
// 唤醒后判 stop_flag:真值仍由 AtomicBool 决定,Notify 仅承载「即时唤醒」职责
|
||||
// (防误唤醒继续跑——若非 stop 触发的 notify,flag 仍 false 则照常进下一轮循环)。
|
||||
_ = notify.notified() => {
|
||||
if stop_flag.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
stopped = true;
|
||||
if sent.is_err() {
|
||||
// receiver dropped (timeout/stop/error) — 停止发送
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 用户停止：已生成文本（可能残缺）交调用方入库展示。Complete(incomplete=false)——
|
||||
// 用户主动停止语义非异常中断,正常入库 + AiCompleted(决策 a1: 不重试路径)。
|
||||
if stopped {
|
||||
return StreamResult::Complete {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
|
||||
// 断连检测：流尽但从未收到 finished 信号 = 异常中断。
|
||||
// B-260615-05:空内容无 finished emit AiError + InitFailed{retryable=true}(无文可保,交重试)。
|
||||
// UX-2025-04:有 partial_text 则保文(Partial),不 emit AiError,不重试。
|
||||
if !finished_received {
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
// B-260615-05:空内容无 finished。UX-260618-15: 不再 emit AiError(retryable=true
|
||||
// 重试路径,emit 权交 agentic 重试耗尽时统一发)。
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: "流式响应意外中断（未收到完成信号，已丢弃残缺响应）".to_string(),
|
||||
};
|
||||
Ok(Err(e)) => {
|
||||
let _ = chunk_tx.send(Err(e.to_string())).await;
|
||||
}
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流尽未收到 finished 但有 partial_text,保文不重试(incomplete)",
|
||||
);
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
Err(_elapsed) => {
|
||||
tracing::error!(
|
||||
conv_id = %conv_id_owned,
|
||||
"[ai] 专用 runtime 内部超时(120s),provider.stream 未返回"
|
||||
);
|
||||
let _ = chunk_tx.send(Err("LLM 调用超时(120s)".to_string())).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 在主 runtime 上通过 mpsc 接收 chunk,应用 timeout/heartbeat/stop 逻辑
|
||||
let mut full_text = String::new();
|
||||
let mut tool_calls_acc: HashMap<u32, ToolCallDraft> = HashMap::new();
|
||||
let mut finished_received = false;
|
||||
let mut stopped = false;
|
||||
let mut final_usage: Option<df_ai::provider::TokenUsage> = None;
|
||||
// BUG-260617-12: DeepSeek thinking 模式推理内容累积（多轮需回传）
|
||||
let mut reasoning_content_acc: Option<String> = None;
|
||||
|
||||
// B-260615-15:heartbeat interval 提至 loop 外复用,避免每轮重建计时器
|
||||
// (每轮重建会丢已积累的节拍,且 interval 首次 tick 立即返回的特性会被误用)。
|
||||
// tokio interval 首 tick 立即返回——此处先丢弃首 tick,让心跳等满首个 30s 静默期才发
|
||||
// (心跳语义是"静默期仍在跑",循环入口立即报无意义且会与 rx.recv() 抢分支错过首 chunk)。
|
||||
let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL);
|
||||
heartbeat.tick().await; // 丢弃首 tick
|
||||
|
||||
// BUG-2026-07-14 根治: wall-clock deadline 替代 select! 内 timeout 包裹。
|
||||
// idle_deadline 是绝对时刻:"下一次 chunk 必须在此前到达,否则判定超时"。
|
||||
// - 首 chunk 阶段: idle_deadline = now + 10s (FIRST_CHUNK_TIMEOUT)
|
||||
// - 收到首 chunk 后: idle_deadline = now + 120s (STREAM_IDLE_TIMEOUT),每次收到 chunk 更新
|
||||
// 心跳打断 select! 后回到循环顶部,deadline 检查在 select! 之前(不受竞争影响)。
|
||||
let mut idle_deadline = tokio::time::Instant::now() + FIRST_CHUNK_TIMEOUT;
|
||||
let mut first_chunk_done = false;
|
||||
|
||||
loop {
|
||||
// B-260615-04:stop 即时打断。
|
||||
if stop_flag.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
stopped = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// BUG-2026-07-14 根治: wall-clock deadline 检查(不依赖 select! timeout 语义)。
|
||||
if tokio::time::Instant::now() >= idle_deadline {
|
||||
if !first_chunk_done {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: format!(
|
||||
"流式响应超时（{}秒未收到首字节，服务无响应或请求体过大）",
|
||||
FIRST_CHUNK_TIMEOUT.as_secs()
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
// 正常完成：流尽 + finished 信号到位,Complete(incomplete=false)
|
||||
StreamResult::Complete {
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流中途 idle timeout(deadline),保文不重试(incomplete)",
|
||||
);
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
|
||||
// 三分支 select!:
|
||||
// 1) chunk_rx.recv():来自专用线程的 LLM chunk (15s 保底 timeout,防 select! 死等)
|
||||
// 2) heartbeat.tick():静默期发 AiHeartbeat reset 前端 watchdog
|
||||
// 3) notify.notified():用户停止即时打断
|
||||
tokio::select! {
|
||||
chunk_result = tokio::time::timeout(
|
||||
Duration::from_secs(15),
|
||||
chunk_rx.recv()
|
||||
) => {
|
||||
match chunk_result {
|
||||
Err(_elapsed) => {
|
||||
// 15s 保底 timeout 触发。正常路径不会到这里:心跳 30s 会先触发 select! 返回。
|
||||
if !first_chunk_done {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: format!(
|
||||
"流式响应超时（{}秒未收到首字节，服务无响应或请求体过大）",
|
||||
FIRST_CHUNK_TIMEOUT.as_secs()
|
||||
),
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流中途 idle timeout,保文不重试(incomplete)",
|
||||
);
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
Ok(None) => break, // mpsc channel 关闭 = 专用线程 stream 结束
|
||||
Ok(Some(Ok(chunk))) => {
|
||||
// BUG-2026-07-14 根治: 收到 chunk 更新 idle_deadline。
|
||||
if !first_chunk_done {
|
||||
first_chunk_done = true;
|
||||
}
|
||||
idle_deadline = tokio::time::Instant::now() + STREAM_IDLE_TIMEOUT;
|
||||
if !chunk.delta.is_empty() {
|
||||
full_text.push_str(&chunk.delta);
|
||||
// L3 emit 双写
|
||||
let ev = AiChatEvent::AiTextDelta {
|
||||
delta: chunk.delta,
|
||||
conversation_id: Some(conv_id.to_string()),
|
||||
};
|
||||
let _ = app_handle.emit("ai-chat-event", ev.clone());
|
||||
let _ = app_handle.state::<crate::state::AppState>().ai_event_bus.publish_event(ev);
|
||||
}
|
||||
if let Some(tc_deltas) = &chunk.tool_calls {
|
||||
for tc_delta in tc_deltas {
|
||||
let draft = tool_calls_acc.entry(tc_delta.index).or_default();
|
||||
if let Some(id) = &tc_delta.id { draft.id = id.clone(); }
|
||||
if let Some(name) = &tc_delta.function_name { draft.name.push_str(name); }
|
||||
if let Some(args) = &tc_delta.function_arguments { draft.args.push_str(args); }
|
||||
}
|
||||
}
|
||||
if let Some(u) = &chunk.usage {
|
||||
final_usage = Some(u.clone());
|
||||
}
|
||||
if let Some(ref rc) = chunk.reasoning_content {
|
||||
reasoning_content_acc.get_or_insert_with(String::new).push_str(rc);
|
||||
}
|
||||
// provider 流式错误事件
|
||||
if let Some(err_msg) = &chunk.error {
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
error = %err_msg,
|
||||
text_len = full_text.len(),
|
||||
"[ai] provider 流式错误事件",
|
||||
);
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: fmt_diag(
|
||||
provider.name(),
|
||||
DiagKind::MidStream,
|
||||
"stream-error",
|
||||
err_msg,
|
||||
),
|
||||
};
|
||||
}
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
if chunk.finished {
|
||||
finished_received = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(Some(Err(err_str))) => {
|
||||
// 专用线程上报的流错误(SSE 解析失败/HTTP 错误等)
|
||||
let (status_or_class, raw) = extract_error_diag_from_str(&err_str);
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
status = %status_or_class,
|
||||
conv_id = %conv_id,
|
||||
error = %raw,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流式接收中途错误(from 专用线程)",
|
||||
);
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: classify_status_or_class(&status_or_class),
|
||||
error: fmt_diag(
|
||||
provider.name(),
|
||||
DiagKind::MidStream,
|
||||
&status_or_class,
|
||||
&raw,
|
||||
),
|
||||
};
|
||||
}
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2) 心跳:静默期 30s 发 AiHeartbeat,前端 watchdog reset(B-260615-02)
|
||||
_ = heartbeat.tick() => {
|
||||
let ev = AiChatEvent::AiHeartbeat {
|
||||
conversation_id: Some(conv_id.to_string()),
|
||||
};
|
||||
let _ = app_handle.emit("ai-chat-event", ev.clone());
|
||||
let _ = app_handle.state::<crate::state::AppState>().ai_event_bus.publish_event(ev);
|
||||
}
|
||||
// 3) 即时停止唤醒(B-260615-14)
|
||||
_ = notify.notified() => {
|
||||
if stop_flag.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
stopped = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// 诊断：连接/鉴权/HTTP 错误，补 provider 标识 + HTTP 状态码/分类 + 原始文本，
|
||||
// 便于区分 401(key)/404(url)/429(限流)/timeout/连接失败(provider_type 或 base_url 不对)。
|
||||
let (status_or_class, raw) = extract_error_diag(&e);
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
status = %status_or_class,
|
||||
conv_id = %conv_id,
|
||||
error = %raw,
|
||||
"[ai] LLM 流式调用失败",
|
||||
);
|
||||
// UX-260618-15: 不再 emit AiError(retryable 走重试路径,Fatal 立即放弃,均交 agentic 统一 emit
|
||||
// 最终错误气泡,避免重试过程 N 次失败 push N+1 气泡)。诊断文本经 fmt_diag 拼好携带出,
|
||||
// agentic 重试耗尽/Fatal 时 emit AiError 用此文本,诊断信息不丢失。
|
||||
// CR-30-1: Init 失败分类——retryable 据 status_or_class 镜像 retry::is_status_retryable:
|
||||
// 5xx/429/timeout/connect=true 可重试;4xx(非429)/鉴权/参数错=false Fatal 立即放弃。
|
||||
StreamResult::InitFailed {
|
||||
retryable: classify_status_or_class(&status_or_class),
|
||||
error: fmt_diag(
|
||||
provider.name(),
|
||||
DiagKind::Init,
|
||||
&status_or_class,
|
||||
&raw,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// 用户停止
|
||||
if stopped {
|
||||
return StreamResult::Complete {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
|
||||
// 断连检测
|
||||
if !finished_received {
|
||||
if full_text.is_empty() && tool_calls_acc.is_empty() {
|
||||
return StreamResult::InitFailed {
|
||||
retryable: true,
|
||||
error: "流式响应意外中断（未收到完成信号，已丢弃残缺响应）".to_string(),
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
provider = %provider.name(),
|
||||
conv_id = %conv_id,
|
||||
text_len = full_text.len(),
|
||||
"[ai] 流尽未收到 finished 但有 partial_text,保文不重试(incomplete)",
|
||||
);
|
||||
return StreamResult::Partial {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
};
|
||||
}
|
||||
|
||||
// 正常完成
|
||||
StreamResult::Complete {
|
||||
text: full_text,
|
||||
tool_calls: tool_calls_acc,
|
||||
usage: final_usage.unwrap_or_default(),
|
||||
reasoning_content: reasoning_content_acc,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,12 +504,24 @@ fn classify_status_or_class(status_or_class: &str) -> bool {
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
// 无三位数字码:按文本分类(timeout/connect 可重试,unknown/其他保守不重试)
|
||||
// 无三位数字码:按文本分类(timeout/connect/transport/decode 可重试,其余 unknown 保守不重试)
|
||||
// BUG-260617-01:Anthropic 中文参数错误(如 `[1214][messages 参数非法]`)无 HTTP 数字前缀,
|
||||
// 走此文本分支,原 || true 致 retryable=true 空耗 4 次重试浪费 token。
|
||||
// 删 || true 对齐 retry::is_status_retryable 未知码不可重试语义;timeout/connect 瞬态仍可重试。
|
||||
//
|
||||
// BUG-2026-07-17 根治:SSE 解码/传输错误(error decoding response body / Transport error)
|
||||
// 是传输层错误,不是 Fatal。原实现未识别这些关键词,返 unknown→Fatal,致连续重试 4 次均
|
||||
// 立即放弃。现补全传输层关键词,让这类错误走可重试路径。
|
||||
let lower = status_or_class.to_lowercase();
|
||||
lower.contains("timeout") || lower.contains("connect") // unknown 保守不重试(BUG-260617-01)
|
||||
lower.contains("timeout")
|
||||
|| lower.contains("connect")
|
||||
|| lower.contains("transport")
|
||||
|| lower.contains("decoding")
|
||||
|| lower.contains("decode")
|
||||
|| lower.contains("body")
|
||||
|| lower.contains("eof")
|
||||
|| lower.contains("reset")
|
||||
|| lower.contains("broken pipe")
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
|
||||
Reference in New Issue
Block a user