新增: 弱网韧性: send 网络感知重试与基准面探针

- connect 类失败重试: 10 分钟窗口+基准面感知(IP 直连与 DNS 双测判本机断网), 断网进长等待自适应探测(5s 快探到 30s 封顶省电), 恢复即重试; 单点故障指数短退避 1s→8s - 两协议(anthropic/openai)同构, 隧道级分钟级弱网不再直接吐中断
2026-10-07 18:33:41 +08:00
parent 76ace953be
commit c2382325cb
4 changed files with 230 additions and 72 deletions
@@ -593,37 +593,83 @@ impl LlmProvider for OpenAICompatProvider {

        // send 阶段需 timeout 防 hang(同 Anthropic 路径)。
        // 不能用 reqwest .timeout()(会砍流式 body),改用 tokio::time::timeout 包裹 send。
-        let send_future = self
-            .client
-            .post(self.chat_url())
-            .header("Authorization", format!("Bearer {}", self.api_key))
-            .header("Content-Type", "application/json")
-            .json(&openai_req)
-            .send();
-        let resp = match tokio::time::timeout(Duration::from_secs(60), send_future).await {
-            Ok(Ok(r)) => r,
-            Ok(Err(e)) => {
-                // 谓词结构化分类:Display 串不含 source 关键词("error sending request for url"),
-                // 纯串分类必 unknown → Fatal 零重试(陈旧连接首包 RST 就此裸奔)。类词置串首,
-                // source 链兜底真实传输词(reset/broken pipe 等),下游按类词走可重试。
-                let class = crate::send_error_class(crate::SendErrPreds::of(&e));
-                tracing::error!(
-                    error = %e,
-                    class = class,
-                    url = %self.chat_url(),
-                    source = ?std::error::Error::source(&e),
-                    "OpenAI 流式 send 失败"
-                );
-                anyhow::bail!(
-                    "流式 send 失败({}): {} [src={:?}]",
-                    class,
-                    e,
-                    std::error::Error::source(&e)
-                );
-            }
-            Err(_elapsed) => {
-                tracing::error!(url = %self.chat_url(), "OpenAI 流式 send 超时(60s 未返回响应头)");
-                anyhow::bail!("流式请求超时(60秒未收到 HTTP 响应，可能服务不可达或被防火墙拦截)");
+        // send 阶段弱网退避: connect 类失败重试零副作用; 网络感知分速(net_probe 基准面):
+        // 本机断网长等待按自适应间隔探测恢复, 单点故障指数短退避 1s→8s, 10 分钟窗口耗尽才放弃。
+        // 与 anthropic_compat 同构。
+        let resp = {
+            let start = std::time::Instant::now();
+            let mut offline_logged = false;
+            let mut offline_polls = 0usize;
+            let mut attempt = 0usize;
+            loop {
+                attempt += 1;
+                let send_future = self
+                    .client
+                    .post(self.chat_url())
+                    .header("Authorization", format!("Bearer {}", self.api_key))
+                    .header("Content-Type", "application/json")
+                    .json(&openai_req)
+                    .send();
+                let outcome = tokio::time::timeout(Duration::from_secs(60), send_future).await;
+                match outcome {
+                    Ok(Ok(r)) => break r,
+                    Ok(Err(e)) => {
+                        // 谓词结构化分类:Display 串不含 source 关键词("error sending request for url"),
+                        // 纯串分类必 unknown → Fatal 零重试(陈旧连接首包 RST 就此裸奔)。类词置串首,
+                        // source 链兜底真实传输词(reset/broken pipe 等),下游按类词走可重试。
+                        let preds = crate::SendErrPreds::of(&e);
+                        let retryable = preds.is_connect || (preds.is_request && !preds.is_builder);
+                        if retryable && start.elapsed() < crate::net_probe::SEND_RETRY_WINDOW {
+                            // 本机断网: 等基准面恢复, 自适应间隔探测(首段 5s 快探, 确认脱网后 30s 封顶省电)
+                            if !crate::net_probe::baseline_alive().await {
+                                offline_polls += 1;
+                                let wait = crate::net_probe::offline_probe_wait(offline_polls);
+                                if !offline_logged {
+                                    warn!(
+                                        url = %self.chat_url(),
+                                        window_secs = crate::net_probe::SEND_RETRY_WINDOW.as_secs(),
+                                        "本机网络不可达(疑似隧道/电梯弱网), 按自适应间隔探测恢复, 恢复后立即重试"
+                                    );
+                                    offline_logged = true;
+                                }
+                                tokio::time::sleep(wait).await;
+                                continue;
+                            }
+                            offline_polls = 0;
+                            offline_logged = false;
+                            let delay = Duration::from_millis(
+                                1000u64.saturating_mul(1 << (attempt - 1).min(3)).min(8000),
+                            );
+                            warn!(
+                                url = %self.chat_url(),
+                                attempt,
+                                delay_ms = delay.as_millis() as u64,
+                                error = %e,
+                                "流式 send 连接失败, 退避重试"
+                            );
+                            tokio::time::sleep(delay).await;
+                            continue;
+                        }
+                        let class = crate::send_error_class(preds);
+                        tracing::error!(
+                            error = %e,
+                            class = class,
+                            url = %self.chat_url(),
+                            source = ?std::error::Error::source(&e),
+                            "OpenAI 流式 send 失败"
+                        );
+                        anyhow::bail!(
+                            "流式 send 失败({}): {} [src={:?}]",
+                            class,
+                            e,
+                            std::error::Error::source(&e)
+                        );
+                    }
+                    Err(_elapsed) => {
+                        tracing::error!(url = %self.chat_url(), "OpenAI 流式 send 超时(60s 未返回响应头)");
+                        anyhow::bail!("流式请求超时(60秒未收到 HTTP 响应，可能服务不可达或被防火墙拦截)");
+                    }
+                }
            }
        };