新增: 工具失败连败升级与熔断停靠
- 同工具连败三级提示:再败软提示,三败强制换路并引导检索经验 - 连拦五次强制安全停靠,不给无视熔断的余地 - 连击按会话计数防参数抖动绕过,成功即清零
This commit is contained in:
@@ -39,7 +39,7 @@ use df_storage::models::AiProviderRecord;
|
||||
// ProviderAcquire 已随 stream_round 迁至 stream_relay.rs。
|
||||
use crate::state::{AppState, LlmConcurrency};
|
||||
|
||||
use super::audit::{process_tool_calls, ToolExecPolicy};
|
||||
use super::audit::{process_tool_calls, take_tool_fail_park, ToolExecPolicy};
|
||||
use super::event_bus::emit_ai_chat_event;
|
||||
// compress_via_llm 已随压缩逻辑迁至 context_lifecycle.rs(maybe_auto_compress 内调用)。
|
||||
use super::conversation::{save_conversation, TokenAccumulator};
|
||||
@@ -2417,6 +2417,24 @@ async fn process_round_tools(
|
||||
return ToolsOutcome::Exit;
|
||||
}
|
||||
|
||||
// B5d 工具连败熔断终局:同会话失败记忆拦截连击达阈且期间无成功 → 强制停靠。
|
||||
// 拦截计数在 audit 层(execute_with_heartbeat)累积,停靠在 loop 层执行:guard 复位
|
||||
// 与 emit_task_parked 正典序列只有本层触达,audit 层只判定不执行。停靠即终局,
|
||||
// 不再给模型无视 blocked 提示继续重试的机会(实证:拦截后模型仍原样重试 10+ 次)。
|
||||
if take_tool_fail_park(conv_id) {
|
||||
tracing::warn!(
|
||||
conv_id = %conv_id,
|
||||
"[ai] 工具连续失败熔断:失败记忆拦截连击达阈,任务强制停靠"
|
||||
);
|
||||
guard.reset().await;
|
||||
crate::commands::ai::emit_task_parked(
|
||||
app_handle, conv_id,
|
||||
"工具连续失败熔断(同一调用被失败记忆反复拦截仍重试),任务安全停靠;已完成进度已保存,调整思路后重发消息可继续",
|
||||
"circuit_breaker",
|
||||
);
|
||||
return ToolsOutcome::Exit;
|
||||
}
|
||||
|
||||
// AC-2 ①:单轮工具调用数超限 → 注入软提示(机制化,非静默,弱模型能感知)。
|
||||
// 治单轮并行调用爆炸:LLM 看不到自己发了多少调用,必须在消息流里显式警告。
|
||||
if is_tool_call_over_limit(round_tool_call_count) {
|
||||
|
||||
@@ -228,6 +228,76 @@ fn sub_agent_tool_clamped(allowed: Option<&[String]>, tool_name: &str) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
// B5d 工具连败升级(快速放弃+换路引导)。三级全部复用既有失败记忆(tool_failure_memory)
|
||||
// 的连败计数,不另造计数轨道:首败=裸错误原样;第 2 次失败=错误包附软提示(勿原样重试);
|
||||
// 第 3 次起=入口拦截包升级为强制换路通牒(换工具/查知识库/向用户说明)。拦截在本会话
|
||||
// 连续达 PARK_AFTER_BLOCKS 且期间无任何工具成功 → take_tool_fail_park 请求 loop 层停靠
|
||||
// (2026-10-12 实证:拦截后模型仍原样重试 10+ 次,软提示拦不住,须有终局)。
|
||||
|
||||
/// 第 2 次失败附带的软提示:提醒而非禁止,给模型自行调参换路的机会。
|
||||
const HINT_AVOID_RETRY: &str = "已连续失败 2 次,勿盲目重试同命令;考虑调整参数或换路径";
|
||||
|
||||
/// 第 3 次起拦截包附带的强制换路提示(终局通牒,含知识库检索引导)。
|
||||
const HINT_MUST_SWITCH: &str = "连续失败≥3次,禁止重试同命令,必须:①换工具或换路径 ②调 knowledge_search 查既有经验 ③仍无法绕过则向用户说明并停手";
|
||||
|
||||
/// 同会话失败记忆拦截连续达此值(且期间无任何工具成功)→ 请求 loop 层强制停靠。
|
||||
const PARK_AFTER_BLOCKS: u32 = 5;
|
||||
|
||||
/// 同会话失败记忆拦截连击计数:键=conv_id,值=自该会话最近一次成功执行以来的拦截次数。
|
||||
/// 与失败记忆分开记:拦截不触发工具执行,失败记忆的连败数停在阈值不再增长,连击数
|
||||
/// 只能另记;按会话而非按调用键记,参数抖动绕开调用键时连击依旧成立。
|
||||
static BLOCK_STREAKS: std::sync::OnceLock<std::sync::Mutex<HashMap<String, u32>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
fn bump_block_streak(conv_id: &str) -> u32 {
|
||||
let streaks = BLOCK_STREAKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()));
|
||||
let mut m = streaks.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let n = m.entry(conv_id.to_string()).or_insert(0);
|
||||
*n += 1;
|
||||
*n
|
||||
}
|
||||
|
||||
/// 任一工具成功即打断连击(会话有产出,不再是纯挣扎状态)。
|
||||
fn clear_block_streak(conv_id: &str) {
|
||||
if let Some(m) = BLOCK_STREAKS.get() {
|
||||
m.lock().unwrap_or_else(|p| p.into_inner()).remove(conv_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// loop 层消费停靠请求:连击达 PARK_AFTER_BLOCKS 且未被成功清零 → 取走标志返 true。
|
||||
pub(crate) fn take_tool_fail_park(conv_id: &str) -> bool {
|
||||
let Some(m) = BLOCK_STREAKS.get() else { return false };
|
||||
let mut map = m.lock().unwrap_or_else(|p| p.into_inner());
|
||||
match map.get(conv_id) {
|
||||
Some(&n) if n >= PARK_AFTER_BLOCKS => {
|
||||
map.remove(conv_id);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 第 2 次失败的错误包附软提示;首败原样透传。
|
||||
fn with_streak_hint(err: anyhow::Error, streak_hit: bool) -> anyhow::Error {
|
||||
if streak_hit {
|
||||
anyhow::anyhow!("{err}\n{HINT_AVOID_RETRY}")
|
||||
} else {
|
||||
err
|
||||
}
|
||||
}
|
||||
|
||||
/// 拦截包升级:hint 字段换为强制换路通牒,status/tool/consecutive_failures/last_error
|
||||
/// 保留不丢。提示文本跨次恒定(不带递增计数):L1 断路器按错误文本哈希归一 key,
|
||||
/// 变化文本会打散其连败累计;连击数走 tracing 日志与停靠判定,不进模型可见包。
|
||||
fn upgrade_blocked_hint(blocked: &str) -> String {
|
||||
let mut v: serde_json::Value = serde_json::from_str(blocked)
|
||||
.unwrap_or_else(|_| serde_json::json!({ "status": "blocked_by_failure_memory" }));
|
||||
if let Some(obj) = v.as_object_mut() {
|
||||
obj.insert("hint".into(), serde_json::json!(HINT_MUST_SWITCH));
|
||||
}
|
||||
v.to_string()
|
||||
}
|
||||
|
||||
/// 处理流式接收的工具调用:Low 风险并行执行(join_all),Med/High 进审批门控
|
||||
/// 返回待审批的工具数量(0 = 全部自动执行完成)
|
||||
///
|
||||
@@ -261,17 +331,20 @@ pub(crate) async fn execute_with_heartbeat(
|
||||
use tokio::time::Duration;
|
||||
|
||||
// ── B5d 工具失败记忆:执行前拦截「同 (conv,tool,args) 连败 ≥2 次」的重复调用 ──
|
||||
// 治「404 连环十几次无退避换路」:拦截时回填结构化换路提示(改参数/换工具/报告放弃),
|
||||
// 治「404 连环十几次无退避换路」:拦截时回填结构化换路提示(强制换路通牒,升级版),
|
||||
// 引导 LLM 换路而非原样重试。成功即清零(record 侧),防误杀。
|
||||
// 拦截直接返 Err(错误 tool_result),不执行工具——process_tool_calls 侧按既有错误路径处理。
|
||||
// 拦截同时累加会话连击,达 PARK_AFTER_BLOCKS 由 loop 层(process_round_tools)停靠收场。
|
||||
if let Some(blocked) = super::tool_failure_memory::precheck(name, &args) {
|
||||
let blocks = bump_block_streak(conv_id);
|
||||
tracing::warn!(
|
||||
conv_id = %conv_id,
|
||||
tool = %name,
|
||||
tool_call_id = %tool_call_id,
|
||||
"[failure-memory] 同调用连败达标,执行前拦截"
|
||||
blocks,
|
||||
"[failure-memory] 同调用连败达标,执行前拦截(本会话累计第 {blocks} 次)"
|
||||
);
|
||||
return Err(anyhow::anyhow!(blocked));
|
||||
return Err(anyhow::anyhow!(upgrade_blocked_hint(&blocked)));
|
||||
}
|
||||
// record 侧需 args 计算 memory_key,但下方 tools.execute(name, args) 会 move args。
|
||||
// 此处留一份 clone 供执行后 record 使用(仅工具执行路径,非热循环,开销可接受)。
|
||||
@@ -516,13 +589,22 @@ pub(crate) async fn execute_with_heartbeat(
|
||||
exec_result
|
||||
};
|
||||
|
||||
// ── B5d 工具失败记忆:执行后回写(成功清零 / 失败递增并记最近错误) ──
|
||||
// 与入口 precheck 配对:成功即删记忆(知识保鲜,防误杀);失败累加计数,
|
||||
// ── B5d 工具失败记忆:执行后回写 + 连败升级提示 ──
|
||||
// 与入口 precheck 配对:成功即删记忆(知识保鲜,防误杀)并打断拦截连击;失败累加计数,
|
||||
// 达阈值后下次 precheck 拦截。error 摘要取 Err 文案(超时/提前退出/工具自身错误同源)。
|
||||
match &exec_result {
|
||||
Ok(_) => super::tool_failure_memory::record(name, &args_for_memory, true, None),
|
||||
Err(e) => super::tool_failure_memory::record(name, &args_for_memory, false, Some(e.to_string())),
|
||||
// 失败侧回探 precheck:命中即本次为第 2 次失败,错误包附软提示(第 3 次起由入口拦截包接管)。
|
||||
let exec_result = match exec_result {
|
||||
Ok(v) => {
|
||||
super::tool_failure_memory::record(name, &args_for_memory, true, None);
|
||||
clear_block_streak(conv_id);
|
||||
Ok(v)
|
||||
}
|
||||
Err(e) => {
|
||||
super::tool_failure_memory::record(name, &args_for_memory, false, Some(e.to_string()));
|
||||
let streak_hit = super::tool_failure_memory::precheck(name, &args_for_memory).is_some();
|
||||
Err(with_streak_hint(e, streak_hit))
|
||||
}
|
||||
};
|
||||
|
||||
exec_result
|
||||
}
|
||||
@@ -1726,4 +1808,78 @@ mod tests {
|
||||
let (reason_0, _) = approval_timeout_park_parts(0);
|
||||
assert!(reason_0.contains("0"), "任意分钟数可格式化(0=禁用超时不会到点,防御)");
|
||||
}
|
||||
|
||||
/// B5d 连败升级三级:首败裸错误 / 第 2 次失败附软提示 / 第 3 次起拦截包升级强制换路
|
||||
/// 通牒(含 knowledge_search 引导);成功一次即清零,再败回到首败无提示。
|
||||
/// 走真失败记忆(进程级静态),键取本测试专属路径防并行互踩。
|
||||
#[test]
|
||||
fn failure_streak_hint_escalates_three_levels() {
|
||||
use crate::commands::ai::tool_failure_memory::{precheck, record};
|
||||
let args = serde_json::json!({ "path": "/tmp/b5d-escalation-check.txt" });
|
||||
|
||||
// 首败:未达拦截阈值,无提示
|
||||
record("read_file", &args, false, Some("not found".into()));
|
||||
assert!(!precheck("read_file", &args).is_some(), "首败不拦截");
|
||||
|
||||
// 第 2 次失败:记录后回探预检命中,错误包附软提示
|
||||
record("read_file", &args, false, Some("not found".into()));
|
||||
let streak_hit = precheck("read_file", &args).is_some();
|
||||
assert!(streak_hit, "两连败后回探应命中");
|
||||
let hinted = with_streak_hint(anyhow::anyhow!("工具执行失败"), streak_hit).to_string();
|
||||
assert!(hinted.contains(HINT_AVOID_RETRY), "第 2 次失败附软提示: {hinted}");
|
||||
assert!(hinted.contains("工具执行失败"), "原始错误保留: {hinted}");
|
||||
|
||||
// 第 3 次起:拦截包升级为强制换路通牒,状态字段保留,文本跨次恒定(L1 断路器
|
||||
// 按错误文本哈希归 key,变化文本会打散其累计)
|
||||
let blocked = precheck("read_file", &args).unwrap();
|
||||
let upgraded = upgrade_blocked_hint(&blocked);
|
||||
let upgraded_again = upgrade_blocked_hint(&blocked);
|
||||
assert!(upgraded.contains("blocked_by_failure_memory"), "状态字段保留: {upgraded}");
|
||||
assert!(upgraded.contains(HINT_MUST_SWITCH), "拦截包附强制换路通牒: {upgraded}");
|
||||
assert!(upgraded.contains("knowledge_search"), "通牒含知识库检索引导: {upgraded}");
|
||||
assert_eq!(upgraded, upgraded_again, "升级文本跨次恒定");
|
||||
|
||||
// 成功一次即清零:再败回到首败,无提示
|
||||
record("read_file", &args, true, None);
|
||||
record("read_file", &args, false, Some("not found".into()));
|
||||
assert!(!precheck("read_file", &args).is_some(), "成功清零后首败无提示");
|
||||
assert!(
|
||||
!with_streak_hint(anyhow::anyhow!("x"), precheck("read_file", &args).is_some())
|
||||
.to_string()
|
||||
.contains(HINT_AVOID_RETRY),
|
||||
"首败错误包不附提示"
|
||||
);
|
||||
}
|
||||
|
||||
/// B5d 熔断终局:同会话失败记忆拦截连击达 PARK_AFTER_BLOCKS 触发停靠,取走即消费
|
||||
/// (不重复触发);未达阈不触发。
|
||||
#[test]
|
||||
fn blocked_streak_parks_after_five() {
|
||||
let conv = "conv-b5d-park-check";
|
||||
clear_block_streak(conv);
|
||||
for _ in 0..(PARK_AFTER_BLOCKS - 1) {
|
||||
bump_block_streak(conv);
|
||||
}
|
||||
assert!(!take_tool_fail_park(conv), "未达阈不停靠");
|
||||
bump_block_streak(conv);
|
||||
assert!(take_tool_fail_park(conv), "拦截连击达 5 触发停靠");
|
||||
assert!(!take_tool_fail_park(conv), "停靠取走即消费,不重复触发");
|
||||
}
|
||||
|
||||
/// B5d 成功打断连击:达阈前任一工具成功即清零,连击从头再计(会话有产出不停靠)。
|
||||
#[test]
|
||||
fn tool_success_clears_block_streak() {
|
||||
let conv = "conv-b5d-success-clear";
|
||||
clear_block_streak(conv);
|
||||
for _ in 0..(PARK_AFTER_BLOCKS - 1) {
|
||||
bump_block_streak(conv);
|
||||
}
|
||||
clear_block_streak(conv);
|
||||
assert!(!take_tool_fail_park(conv), "成功打断连击,不停靠");
|
||||
for _ in 0..PARK_AFTER_BLOCKS {
|
||||
bump_block_streak(conv);
|
||||
}
|
||||
assert!(take_tool_fail_park(conv), "清零后重新连击达阈仍停靠");
|
||||
clear_block_streak(conv);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user