新增: 弱网韧性: send 网络感知重试与基准面探针
- connect 类失败重试: 10 分钟窗口+基准面感知(IP 直连与 DNS 双测判本机断网), 断网进长等待自适应探测(5s 快探到 30s 封顶省电), 恢复即重试; 单点故障指数短退避 1s→8s - 两协议(anthropic/openai)同构, 隧道级分钟级弱网不再直接吐中断
This commit is contained in:
@@ -743,47 +743,94 @@ impl LlmProvider for AnthropicCompatProvider {
|
||||
// 流式长生成任务会被误砍。改用 tokio::time::timeout
|
||||
// 包裹 send().await,只管建连+首响应头,不管后续 body 读取(后续由 stream_llm idle timeout 兜底)。
|
||||
// 60s 选型:正常 send(建连+收 200 headers)<5s,60s 足够宽容。
|
||||
let send_future = self
|
||||
.auth_headers(self.client.post(self.messages_url()))
|
||||
.json(&body)
|
||||
.version(reqwest::Version::HTTP_11)
|
||||
.send();
|
||||
let resp = match tokio::time::timeout(Duration::from_secs(60), send_future).await {
|
||||
Ok(Ok(r)) => r,
|
||||
Ok(Err(e)) => {
|
||||
// 谓词结构化分类(对齐 openai 路径,共用 crate::send_error_class)。原 bail 串
|
||||
// [timeout={} connect={}=...] 的字面词让一切错误(含 builder 错)侥幸可重试。
|
||||
// 类词改由谓词决定置串首;谓词转储缩写为 t/c/b/r 防误触分类词表,source 链
|
||||
// 保留真实传输词(reset/broken pipe 等)作串兜底。
|
||||
let class = crate::send_error_class(crate::SendErrPreds::of(&e));
|
||||
tracing::error!(
|
||||
is_timeout = e.is_timeout(),
|
||||
is_connect = e.is_connect(),
|
||||
is_body = e.is_body(),
|
||||
is_request = e.is_request(),
|
||||
class = class,
|
||||
url = %self.messages_url(),
|
||||
source = ?std::error::Error::source(&e),
|
||||
"Anthropic 流式 send 失败"
|
||||
);
|
||||
anyhow::bail!(
|
||||
"send 失败({}): {} [t={} c={} b={} r={} src={:?}]",
|
||||
class,
|
||||
e,
|
||||
e.is_timeout(),
|
||||
e.is_connect(),
|
||||
e.is_body(),
|
||||
e.is_request(),
|
||||
std::error::Error::source(&e)
|
||||
);
|
||||
}
|
||||
Err(_elapsed) => {
|
||||
// send 阶段超时(60s 未返回 HTTP 响应头):GLM 端点可能不可达或极慢
|
||||
tracing::error!(
|
||||
url = %self.messages_url(),
|
||||
"Anthropic 流式 send 超时(60s 未返回响应头)"
|
||||
);
|
||||
anyhow::bail!("流式请求超时(60秒未收到 HTTP 响应，可能服务不可达或被防火墙拦截)");
|
||||
// send 阶段弱网退避: connect 类失败(隧道断网/基站切换/DNS 抖动)请求未发出, 重试零副作用。
|
||||
// 网络感知分速(net_probe 基准面): 本机断网进长等待按自适应间隔探测恢复(隧道级分钟级波动),
|
||||
// 单点故障指数短退避 1s→8s; 10 分钟窗口耗尽才放弃。timeout/decode/HTTP 状态错不重试。
|
||||
// 与 openai_compat 同构。
|
||||
let resp = {
|
||||
let start = std::time::Instant::now();
|
||||
let mut offline_logged = false;
|
||||
let mut offline_polls = 0usize;
|
||||
let mut attempt = 0usize;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let send_future = self
|
||||
.auth_headers(self.client.post(self.messages_url()))
|
||||
.json(&body)
|
||||
.version(reqwest::Version::HTTP_11)
|
||||
.send();
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(60), send_future).await;
|
||||
match outcome {
|
||||
Ok(Ok(r)) => break r,
|
||||
Ok(Err(e)) => {
|
||||
// 谓词结构化分类(对齐 openai 路径,共用 crate::send_error_class)。原 bail 串
|
||||
// [timeout={} connect={}=...] 的字面词让一切错误(含 builder 错)侥幸可重试。
|
||||
// 类词改由谓词决定置串首;谓词转储缩写为 t/c/b/r 防误触分类词表,source 链
|
||||
// 保留真实传输词(reset/broken pipe 等)作串兜底。
|
||||
let preds = crate::SendErrPreds::of(&e);
|
||||
let retryable = preds.is_connect || (preds.is_request && !preds.is_builder);
|
||||
if retryable && start.elapsed() < crate::net_probe::SEND_RETRY_WINDOW {
|
||||
// 本机断网: 等基准面恢复(自适应间隔), 不消耗退避档位
|
||||
if !crate::net_probe::baseline_alive().await {
|
||||
offline_polls += 1;
|
||||
let wait = crate::net_probe::offline_probe_wait(offline_polls);
|
||||
if !offline_logged {
|
||||
warn!(
|
||||
url = %self.messages_url(),
|
||||
window_secs = crate::net_probe::SEND_RETRY_WINDOW.as_secs(),
|
||||
"本机网络不可达(疑似隧道/电梯弱网), 按自适应间隔探测恢复, 恢复后立即重试"
|
||||
);
|
||||
offline_logged = true;
|
||||
}
|
||||
tokio::time::sleep(wait).await;
|
||||
continue;
|
||||
}
|
||||
offline_polls = 0;
|
||||
offline_logged = false;
|
||||
let delay = Duration::from_millis(
|
||||
1000u64.saturating_mul(1 << (attempt - 1).min(3)).min(8000),
|
||||
);
|
||||
warn!(
|
||||
url = %self.messages_url(),
|
||||
attempt,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
error = %e,
|
||||
"流式 send 连接失败, 退避重试"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
continue;
|
||||
}
|
||||
let class = crate::send_error_class(preds);
|
||||
tracing::error!(
|
||||
is_timeout = e.is_timeout(),
|
||||
is_connect = e.is_connect(),
|
||||
is_body = e.is_body(),
|
||||
is_request = e.is_request(),
|
||||
class = class,
|
||||
url = %self.messages_url(),
|
||||
source = ?std::error::Error::source(&e),
|
||||
"Anthropic 流式 send 失败"
|
||||
);
|
||||
anyhow::bail!(
|
||||
"send 失败({}): {} [t={} c={} b={} r={} src={:?}]",
|
||||
class,
|
||||
e,
|
||||
e.is_timeout(),
|
||||
e.is_connect(),
|
||||
e.is_body(),
|
||||
e.is_request(),
|
||||
std::error::Error::source(&e)
|
||||
);
|
||||
}
|
||||
Err(_elapsed) => {
|
||||
// send 阶段超时(60s 未返回 HTTP 响应头):GLM 端点可能不可达或极慢
|
||||
tracing::error!(
|
||||
url = %self.messages_url(),
|
||||
"Anthropic 流式 send 超时(60s 未返回响应头)"
|
||||
);
|
||||
anyhow::bail!("流式请求超时(60秒未收到 HTTP 响应，可能服务不可达或被防火墙拦截)");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user