修复: 连接池收敛与传输错误分类重试
- 连接池空闲十五秒,陈旧连接复用致重置根治 - 流式发送错误改谓词分类,openai 路径不再裸奔 - decode 类失败纳入重试,正文中断可自愈 - 状态分类合并响应体,激活休眠词表分支 - 手动压缩失败降级兜底,不再弹红卡
This commit is contained in:
@@ -600,7 +600,9 @@ impl LlmProvider for AnthropicCompatProvider {
|
||||
source = ?std::error::Error::source(&e),
|
||||
"Anthropic 同步 send 失败"
|
||||
);
|
||||
if is_reqwest_error_retryable(&e) {
|
||||
// is_decode 补可重试:body 中途断连在 reqwest 归 decode 类,complete 重发
|
||||
// 幂等安全(受 3 次/30s 预算约束,真正格式错重试同败,预算内止损)。
|
||||
if is_reqwest_error_retryable(&e) || e.is_decode() {
|
||||
return AttemptOutcome::Retryable(format!("请求失败(可重试): {}", e));
|
||||
}
|
||||
return AttemptOutcome::Fatal(format!(
|
||||
@@ -626,7 +628,13 @@ impl LlmProvider for AnthropicCompatProvider {
|
||||
}
|
||||
let resp: AnthropicResponse = match resp.json().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => return AttemptOutcome::Fatal(format!("响应解析失败: {}", e)),
|
||||
// serde 结构错属 Fatal(重试同败);body 读取中途断连同归 is_decode,可重试。
|
||||
Err(e) => {
|
||||
if e.is_decode() {
|
||||
return AttemptOutcome::Retryable(format!("响应解码失败(可重试): {}", e));
|
||||
}
|
||||
return AttemptOutcome::Fatal(format!("响应解析失败: {}", e));
|
||||
}
|
||||
};
|
||||
// content 块中拼接 text,收集 tool_use
|
||||
let mut text = String::new();
|
||||
@@ -714,17 +722,24 @@ impl LlmProvider for AnthropicCompatProvider {
|
||||
let resp = match tokio::time::timeout(Duration::from_secs(60), send_future).await {
|
||||
Ok(Ok(r)) => r,
|
||||
Ok(Err(e)) => {
|
||||
// 谓词结构化分类(对齐 openai 路径,共用 crate::send_error_class)。原 bail 串
|
||||
// [timeout={} connect={}=...] 的字面词让一切错误(含 builder 错)侥幸可重试。
|
||||
// 类词改由谓词决定置串首;谓词转储缩写为 t/c/b/r 防误触分类词表,source 链
|
||||
// 保留真实传输词(reset/broken pipe 等)作串兜底。
|
||||
let class = crate::send_error_class(crate::SendErrPreds::of(&e));
|
||||
tracing::error!(
|
||||
is_timeout = e.is_timeout(),
|
||||
is_connect = e.is_connect(),
|
||||
is_body = e.is_body(),
|
||||
is_request = e.is_request(),
|
||||
class = class,
|
||||
url = %self.messages_url(),
|
||||
source = ?std::error::Error::source(&e),
|
||||
"Anthropic 流式 send 失败"
|
||||
);
|
||||
anyhow::bail!(
|
||||
"send 失败: {} [timeout={} connect={} body={} request={} src={:?}]",
|
||||
"send 失败({}): {} [t={} c={} b={} r={} src={:?}]",
|
||||
class,
|
||||
e,
|
||||
e.is_timeout(),
|
||||
e.is_connect(),
|
||||
|
||||
@@ -64,6 +64,11 @@ pub use df_ai_core;
|
||||
pub(crate) fn build_provider_client() -> Client {
|
||||
Client::builder()
|
||||
.connect_timeout(std::time::Duration::from_secs(30))
|
||||
// 连接池收敛:idle 连接 15s 弃用 + 每主机最多存 2 条。reqwest 默认 idle 90s 且
|
||||
// 无上限,服务端/前置 keepalive 更短时复用陈旧连接首包即被 RST(ConnectionReset
|
||||
// 10054),呈现「隔几十秒一发就失败」的间歇形态;15s 保守低于常见网关 60s keepalive。
|
||||
.pool_idle_timeout(std::time::Duration::from_secs(15))
|
||||
.pool_max_idle_per_host(2)
|
||||
.build()
|
||||
.unwrap_or_else(|e| {
|
||||
tracing::warn!("reqwest builder 失败,回退默认 client: {}", e);
|
||||
@@ -71,6 +76,58 @@ pub(crate) fn build_provider_client() -> Client {
|
||||
})
|
||||
}
|
||||
|
||||
/// 流式 send 阶段 reqwest 错误的谓词快照。
|
||||
///
|
||||
/// reqwest::Error 无公开构造器,抽谓词结构体过纯函数,单测才能锁定分类
|
||||
/// (send 阶段 ConnectionReset → 可重试类词)。
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct SendErrPreds {
|
||||
pub is_timeout: bool,
|
||||
pub is_connect: bool,
|
||||
pub is_decode: bool,
|
||||
pub is_body: bool,
|
||||
pub is_request: bool,
|
||||
pub is_builder: bool,
|
||||
}
|
||||
|
||||
impl SendErrPreds {
|
||||
pub(crate) fn of(e: &reqwest::Error) -> Self {
|
||||
Self {
|
||||
is_timeout: e.is_timeout(),
|
||||
is_connect: e.is_connect(),
|
||||
is_decode: e.is_decode(),
|
||||
is_body: e.is_body(),
|
||||
is_request: e.is_request(),
|
||||
is_builder: e.is_builder(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 流式 send 阶段错误分类,返回类词(置于 bail 串首位供下游按串分类)。
|
||||
///
|
||||
/// reqwest 0.12 的 Display 只给 "error sending request for url",不含 source 关键词,
|
||||
/// 纯 Display 串分类必得 unknown → Fatal 零重试。此处用谓词确定性给类词:
|
||||
/// timeout/connect/decode/body 各归本类;is_request(非 builder)覆盖陈旧池化连接
|
||||
/// 首包 RST(ConnectionReset 10054)等传输层发送错,归 connect(与 retry 的
|
||||
/// is_reqwest_error_retryable 对 is_request 可重试口径一致);builder 构造错归
|
||||
/// request(不可重试)。类词须落在 stream_recv classify_status_or_class 传输词表内,
|
||||
/// 改任一侧需同步。
|
||||
pub(crate) fn send_error_class(p: SendErrPreds) -> &'static str {
|
||||
if p.is_timeout {
|
||||
"timeout"
|
||||
} else if p.is_connect {
|
||||
"connect"
|
||||
} else if p.is_decode {
|
||||
"decode"
|
||||
} else if p.is_body {
|
||||
"body"
|
||||
} else if p.is_request && !p.is_builder {
|
||||
"connect"
|
||||
} else {
|
||||
"request"
|
||||
}
|
||||
}
|
||||
|
||||
/// 按 provider_type 构建 LLM Provider 实例(统一选择逻辑,消除调用方重复 match)
|
||||
///
|
||||
/// `anthropic` 协议走 AnthropicCompatProvider(GLM 订阅端点 / Claude 官方),
|
||||
|
||||
@@ -502,7 +502,9 @@ impl LlmProvider for OpenAICompatProvider {
|
||||
let resp = match resp {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
if is_reqwest_error_retryable(&e) {
|
||||
// is_decode 补可重试:body 中途断连在 reqwest 归 decode 类,complete 重发
|
||||
// 幂等安全(受 3 次/30s 预算约束,真正格式错重试同败,预算内止损)。
|
||||
if is_reqwest_error_retryable(&e) || e.is_decode() {
|
||||
return AttemptOutcome::Retryable(format!("请求失败(可重试): {}", e));
|
||||
}
|
||||
return AttemptOutcome::Fatal(format!("请求失败(不可重试): {}", e));
|
||||
@@ -520,10 +522,16 @@ impl LlmProvider for OpenAICompatProvider {
|
||||
error!(%status, %body, "LLM API 调用失败(不可重试)");
|
||||
return AttemptOutcome::Fatal(msg);
|
||||
}
|
||||
// body 解析: 解析错属 Fatal(响应已成功送达,重试也会因同样格式失败)
|
||||
// body 解析: serde 结构错属 Fatal(响应已成功送达,重试同样失败);
|
||||
// body 读取中途断连在 reqwest 同归 is_decode,重发幂等安全 → 可重试。
|
||||
let body: OpenAiResponse = match resp.json().await {
|
||||
Ok(b) => b,
|
||||
Err(e) => return AttemptOutcome::Fatal(format!("响应解析失败: {}", e)),
|
||||
Err(e) => {
|
||||
if e.is_decode() {
|
||||
return AttemptOutcome::Retryable(format!("响应解码失败(可重试): {}", e));
|
||||
}
|
||||
return AttemptOutcome::Fatal(format!("响应解析失败: {}", e));
|
||||
}
|
||||
};
|
||||
let choice = match body.choices.into_iter().next() {
|
||||
Some(c) => c,
|
||||
@@ -570,8 +578,23 @@ impl LlmProvider for OpenAICompatProvider {
|
||||
let resp = match tokio::time::timeout(Duration::from_secs(60), send_future).await {
|
||||
Ok(Ok(r)) => r,
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(error = %e, is_timeout = e.is_timeout(), "OpenAI 流式 send 失败");
|
||||
return Err(e.into());
|
||||
// 谓词结构化分类:Display 串不含 source 关键词("error sending request for url"),
|
||||
// 纯串分类必 unknown → Fatal 零重试(陈旧连接首包 RST 就此裸奔)。类词置串首,
|
||||
// source 链兜底真实传输词(reset/broken pipe 等),下游按类词走可重试。
|
||||
let class = crate::send_error_class(crate::SendErrPreds::of(&e));
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
class = class,
|
||||
url = %self.chat_url(),
|
||||
source = ?std::error::Error::source(&e),
|
||||
"OpenAI 流式 send 失败"
|
||||
);
|
||||
anyhow::bail!(
|
||||
"流式 send 失败({}): {} [src={:?}]",
|
||||
class,
|
||||
e,
|
||||
std::error::Error::source(&e)
|
||||
);
|
||||
}
|
||||
Err(_elapsed) => {
|
||||
tracing::error!(url = %self.chat_url(), "OpenAI 流式 send 超时(60s 未返回响应头)");
|
||||
@@ -1279,4 +1302,61 @@ mod tests {
|
||||
let tc4 = c4.tool_calls.as_ref().expect("应有 tool_calls").first().unwrap();
|
||||
assert_eq!(tc4.id.as_deref(), Some("call_xyz"), "非空 id 原样透传");
|
||||
}
|
||||
|
||||
// ---------- 流式 send 错误分类(谓词纯函数,reqwest::Error 无公开构造器以快照模拟) ----------
|
||||
|
||||
/// send 阶段陈旧连接首包 RST(reqwest 形态 is_request=true + 源 ConnectionReset)
|
||||
/// → connect 类词 → 下游可重试。根治 Display 串无 source 关键词被分 unknown 零重试。
|
||||
#[test]
|
||||
fn send_error_connection_reset_is_retryable_class() {
|
||||
let class = crate::send_error_class(crate::SendErrPreds {
|
||||
is_request: true,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(class, "connect");
|
||||
}
|
||||
|
||||
/// timeout/connect/decode/body/request 形态各归本类,且类词均须落在下游传输可重试
|
||||
/// 词表内(镜像 src-tauri stream_recv classify_status_or_class 词表,改任一侧需同步)。
|
||||
#[test]
|
||||
fn send_error_transport_classes_are_retryable_words() {
|
||||
const RETRYABLE_WORDS: &[&str] = &[
|
||||
"timeout",
|
||||
"connect",
|
||||
"transport",
|
||||
"decoding",
|
||||
"decode",
|
||||
"body",
|
||||
"eof",
|
||||
"reset",
|
||||
"broken pipe",
|
||||
];
|
||||
let cases = [
|
||||
crate::SendErrPreds { is_timeout: true, ..Default::default() },
|
||||
crate::SendErrPreds { is_connect: true, ..Default::default() },
|
||||
crate::SendErrPreds { is_decode: true, ..Default::default() },
|
||||
crate::SendErrPreds { is_body: true, ..Default::default() },
|
||||
crate::SendErrPreds { is_request: true, ..Default::default() },
|
||||
];
|
||||
for p in cases {
|
||||
let class = crate::send_error_class(p);
|
||||
assert!(
|
||||
RETRYABLE_WORDS.contains(&class),
|
||||
"类词 {} 应在传输可重试词表内",
|
||||
class
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// builder 构造错(请求本体问题,重试同败)→ request 类词,不在词表 → 下游 Fatal,
|
||||
/// 与 retry::is_reqwest_error_retryable 的 is_builder 排除口径一致。
|
||||
#[test]
|
||||
fn send_error_builder_is_fatal_class() {
|
||||
let class = crate::send_error_class(crate::SendErrPreds {
|
||||
is_request: true,
|
||||
is_builder: true,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(class, "request");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user