优化: run_command 实时流式 + 审批浮窗修复 + 文档探索闭环(fetch_url+obscura引导+model_fetch兼容+prompt策略+厂商预设)
This commit is contained in:
@@ -100,6 +100,7 @@ async fn execute_with_heartbeat(
|
||||
args: serde_json::Value,
|
||||
app: &AppHandle,
|
||||
conv_id: &str,
|
||||
tool_call_id: &str,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::time::Duration;
|
||||
@@ -155,7 +156,20 @@ async fn execute_with_heartbeat(
|
||||
} else {
|
||||
60
|
||||
};
|
||||
match tokio::time::timeout(Duration::from_secs(outer_secs), tools.execute(name, args)).await {
|
||||
// run_command 实时流式:经 task-local sink 注入(AppHandle + tool_call_id + conv_id),
|
||||
// handler 内 execute_streaming 每行 emit AiCommandOutput(治执行中黑盒)。
|
||||
// 非 run_command 不注入 sink(handler 内 emit_output 静默 noop),零行为变更。
|
||||
let sink = if name == "run_command" {
|
||||
Some(crate::commands::ai::command_stream::CommandSink::new(
|
||||
app.clone(),
|
||||
tool_call_id.to_string(),
|
||||
Some(conv_id.to_string()),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let exec_fut = crate::commands::ai::command_stream::scope(sink, tools.execute(name, args));
|
||||
match tokio::time::timeout(Duration::from_secs(outer_secs), exec_fut).await {
|
||||
Ok(result) => result,
|
||||
Err(_elapsed) => {
|
||||
tracing::error!(
|
||||
@@ -460,7 +474,7 @@ pub(crate) async fn process_tool_calls(
|
||||
let app_clone = app_handle.clone();
|
||||
let conv_clone = conv_id.to_string();
|
||||
async move {
|
||||
let exec_result = execute_with_heartbeat(&tools, &draft.name, args, &app_clone, &conv_clone).await;
|
||||
let exec_result = execute_with_heartbeat(&tools, &draft.name, args, &app_clone, &conv_clone, &draft.id).await;
|
||||
match exec_result {
|
||||
Ok(val) => {
|
||||
let content = val.to_string();
|
||||
@@ -573,7 +587,7 @@ pub(crate) async fn process_tool_calls(
|
||||
let app_clone = app_handle.clone();
|
||||
let conv_clone = conv_id.to_string();
|
||||
async move {
|
||||
let result = execute_with_heartbeat(&tools, &draft.name, args, &app_clone, &conv_clone).await;
|
||||
let result = execute_with_heartbeat(&tools, &draft.name, args, &app_clone, &conv_clone, &draft.id).await;
|
||||
match result {
|
||||
Ok(val) => {
|
||||
// L3 emit 双写：Low 风险工具执行成功 emit Completed 双路发布。
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
//! run_command 实时流式输出 — task-local sink 机制。
|
||||
//!
|
||||
//! 治「run_command 执行中黑盒」:execute() 等 exit 才返回整块 stdout/stderr,
|
||||
//! 长命令(cargo/npm 构建)期间前端只看 Started→Completed,中间进度不可见。
|
||||
//!
|
||||
//! 架构约束:工具 handler 注册为 `Box<dyn Fn(Value) -> Future>`(ai_tools.rs:48),
|
||||
//! 签名只收 args 不收 AppHandle/tool_call_id,无法直接 emit 事件。改 schema 把 id
|
||||
//! 塞进 args 会泄漏给 LLM,不可取。
|
||||
//!
|
||||
//! 本模块用 [`tokio::task_local!`] 解耦:调用方(execute_with_heartbeat / ai_approve)
|
||||
//! 持有 AppHandle + tool_call_id + conv_id,调 `tools.execute` 前用 [`scope`] 把一个
|
||||
//! [`CommandSink`] 注入当前 task 上下文;run_command handler 在 tools/file.rs 内
|
||||
//! 读 task-local(同一 task,因 tools.execute 不 spawn 直接 await handler),
|
||||
//! 命中则改走 shell `execute_streaming`,每行回调 [`emit_output`] → AiCommandOutput。
|
||||
//!
|
||||
//! 未注入 sink(非 run_command / 调用方未配 scope)时 [`emit_output`] 静默 noop,
|
||||
//! 兜底不报错不阻断。
|
||||
|
||||
use tauri::{AppHandle, Emitter, Manager};
|
||||
|
||||
use super::AiChatEvent;
|
||||
use df_execute::shell::StreamKind;
|
||||
|
||||
/// 一次 run_command 调用的输出下沉目标(emit 事件所需上下文)。
|
||||
#[derive(Clone)]
|
||||
pub struct CommandSink {
|
||||
app: AppHandle,
|
||||
tool_call_id: String,
|
||||
conversation_id: Option<String>,
|
||||
}
|
||||
|
||||
impl CommandSink {
|
||||
pub fn new(app: AppHandle, tool_call_id: String, conversation_id: Option<String>) -> Self {
|
||||
Self { app, tool_call_id, conversation_id }
|
||||
}
|
||||
|
||||
/// emit 一行 stdout/stderr(AiCommandOutput,双写 app.emit + ai_event_bus)。
|
||||
/// emit 失败静默吞(前端未 listen / 总线无订阅不阻断命令执行)。
|
||||
fn emit(&self, kind: StreamKind, line: &str) {
|
||||
let ev = AiChatEvent::AiCommandOutput {
|
||||
id: self.tool_call_id.clone(),
|
||||
stream: kind.as_str().to_string(),
|
||||
line: line.to_string(),
|
||||
conversation_id: self.conversation_id.clone(),
|
||||
};
|
||||
let _ = self.app.emit("ai-chat-event", ev.clone());
|
||||
let _ = self.app.state::<crate::state::AppState>().ai_event_bus.publish_event(ev);
|
||||
}
|
||||
}
|
||||
|
||||
// task-local 槽:tools.execute 调用期间(同 task)handler 可读。
|
||||
// 嵌套 Option:外层是 task-local 是否 set,内层是是否注入 sink(None = 已 scope 但无 sink)。
|
||||
tokio::task_local! {
|
||||
static SINK: Option<CommandSink>;
|
||||
}
|
||||
|
||||
/// 在 sink 作用域内执行 future。future 完成后自动清理(无残留)。
|
||||
///
|
||||
/// 调用方:execute_with_heartbeat / ai_approve 在调 `tools.execute("run_command", args)`
|
||||
/// 前包一层 `command_stream::scope(Some(sink), async { tools.execute(...).await })`。
|
||||
pub async fn scope<F, R>(sink: Option<CommandSink>, fut: F) -> R
|
||||
where
|
||||
F: std::future::Future<Output = R>,
|
||||
{
|
||||
SINK.scope(sink, fut).await
|
||||
}
|
||||
|
||||
/// run_command handler 读取:回调每行输出 → emit AiCommandOutput。
|
||||
///
|
||||
/// task-local 未 set(非经 scope 调用,如直接单测调 handler)或内层 None → 静默 noop。
|
||||
/// 返回值忽略(emit 失败不阻断命令)。
|
||||
pub fn emit_output(kind: StreamKind, line: &str) {
|
||||
// LocalKey::with 在 task-local 未 scope 时返 AccessError,静默吞(noop 兜底)。
|
||||
let _ = SINK.with(|maybe_sink: &Option<CommandSink>| {
|
||||
if let Some(sink) = maybe_sink {
|
||||
sink.emit(kind, line);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -653,9 +653,17 @@ pub async fn ai_approve(
|
||||
// 防 list_directory 大目录/run_command 慢命令卡死 → 到不了 audit_finalize/emit/try_continue
|
||||
// → IPC 永挂 → per_conv.generating 永真 → 前端 130s 看门狗静默吞消息(F-260620 同型故障,
|
||||
// 之前只给 ai_authorize_dir 加超时,ai_approve 漏)。
|
||||
// run_command 实时流式:经 task-local sink 注入(AppHandle + tool_call_id + conv_id),
|
||||
// handler 内 execute_streaming 每行 emit AiCommandOutput(治执行中黑盒)。非 run_command 不注入。
|
||||
let sink = if approval.tool_name == "run_command" {
|
||||
Some(super::super::command_stream::CommandSink::new(
|
||||
app.clone(), id.clone(), conv_id.clone(),
|
||||
))
|
||||
} else { None };
|
||||
let exec_fut = super::super::command_stream::scope(sink, state.ai_tools.execute(&approval.tool_name, args.clone()));
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(60),
|
||||
state.ai_tools.execute(&approval.tool_name, args.clone()),
|
||||
exec_fut,
|
||||
).await {
|
||||
Ok(r) => r,
|
||||
// 任务2: ai_approve 路径同步 run_command 失败提示(超时为最常见失败场景)。
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
//! fetch_url AI 工具 — URL → markdown 文档嗅探(GET HTML → markdown,去噪音 + 截断)
|
||||
//!
|
||||
//! 设计目标:让 LLM 高效理解网页文档(API 文档/博客/技术资料),替代 `http_request` 拿原始 HTML。
|
||||
//! 原始 HTML 充满 nav/footer/script/style/广告等噪声,直接灌进 prompt 既爆 token 又稀释信号;
|
||||
//! 本工具 GET HTML 后用 htmd(turndown.js 移植)转 markdown,剥离非内容节点 + 按 char 截断,
|
||||
//! 让 LLM 拿到的是干净文本主体。
|
||||
//!
|
||||
//! ## 与 http_request 的分工
|
||||
//!
|
||||
//! - `http_request`:结构化 API 调用(GET/POST/鉴权/JSON),拿原始响应 body。写方法有副作用=High。
|
||||
//! - `fetch_url`:**只读 GET** 网页文档嗅探,输出已清洗的 markdown。Low risk(只读,无副作用)。
|
||||
//!
|
||||
//! ## 安全(SSRF 防护 — 与 http_request 共享同一套)
|
||||
//!
|
||||
//! 复用 `http.rs` 的 `validate_url` / `resolve_and_check_host` / `build_client` / `execute_with_redirects`:
|
||||
//! ① 协议白名单(仅 http/https)② 私网 IP 黑名单(RFC1918 + 链路本地 + 环回 + 元数据)③ DNS resolve 后
|
||||
//! 校验 IP(防 rebinding)④ 重定向 ≤3 跳每跳重校验。fetch_url 只发 GET,无 body/写副作用。
|
||||
//!
|
||||
//! ## 去噪音策略
|
||||
//!
|
||||
//! htmd 本身会把 script/style/noscript 等非可见节点忽略(转空),但转换后可能残留少量空白行、
|
||||
//! 重复 nav 链接文本。本工具追加一道轻量清理:折叠 ≥3 连续空行为 2 行、trim 行尾空白。
|
||||
//! 不做激进的 DOM 删节点(易误伤正文,且 htmd 已处理主要噪声)——稳健优先。
|
||||
//!
|
||||
//! ## 截断
|
||||
//!
|
||||
//! 按 char 截断(非字节,避免切多字节 UTF-8 中间):默认 max_length=8000 chars,clamp [500, 50000]。
|
||||
//! 截断尾部加 `... [markdown 已截断]` 标记,LLM 可知内容不完整。
|
||||
//!
|
||||
//! ## render 模式(obscura 引导,JS 渲染 SPA/反爬文档)
|
||||
//!
|
||||
//! `render=false`(默认)走 reqwest + htmd 静态管线,所有用户可用。
|
||||
//! `render=true` 时检测本地 `obscura`(用户自带 Rust 无头浏览器,h4ckf0r0day/obscura,自带 V8):
|
||||
//! ① 已装 → spawn `obscura fetch --dump markdown <URL>` → 拿 stdout → 截断 → 返回(同静态格式)。
|
||||
//! ② 未装 / 启动失败 / 超时 → 回退静态管线 + `hint` 字段提示装 obscura 可增强 JS 渲染。
|
||||
//! DevFlow 不打包 obscura(按需引导),保证 render=true 在任何环境都不致工具失败。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// 默认 markdown 截断长度(chars)。LLM 单次 fetch 应足以覆盖一篇中等文档的主体。
|
||||
const DEFAULT_MAX_LENGTH: usize = 8000;
|
||||
/// markdown 截断下限(chars)。防 LLM 传极小值拿到无意义片段。
|
||||
const MIN_MAX_LENGTH: usize = 500;
|
||||
/// markdown 截断上限(chars)。防 LLM 传极大值仍撑爆 context(50K chars ≈ 12-15K tokens)。
|
||||
const MAX_MAX_LENGTH: usize = 50_000;
|
||||
/// 默认请求超时(秒)。与 http_request 一致。
|
||||
const DEFAULT_TIMEOUT_SECS: u64 = 30;
|
||||
/// 超时硬上限(秒)。与 http_request 一致。
|
||||
const MAX_TIMEOUT_SECS: u64 = 60;
|
||||
/// 响应 body 字节上限。HTML 转 markdown 前先挡超大原始响应,防 OOM(50KB markdown 足够,
|
||||
/// 但 HTML 可能远大于其 markdown,故放宽到 2MB 原始 HTML 上限)。
|
||||
const MAX_HTML_BYTES: usize = 2 * 1024 * 1024;
|
||||
/// obscura 子进程超时(秒)。obscura 默认 page timeout=30s,外加 V8 启动/JIT 余量给到 45s。
|
||||
/// 超时即杀子进程并回退静态管线(render=true 兜底,不让 SPA 拖垮工具)。
|
||||
const OBSCURA_TIMEOUT_SECS: u64 = 45;
|
||||
/// obscura 未安装时的引导提示。回退静态管线后通过返回 JSON 的 `hint` 字段透传给 LLM/用户。
|
||||
const OBSCURA_HINT: &str = "obscura 未安装,已用静态模式。装 obscura(Rust 无头浏览器,自带 V8)可增强 JS 渲染文档(SPA/反爬):见 github.com/h4ckf0r0day/obscura";
|
||||
|
||||
/// fetch_url 工具 handler 入口(供 tools/fetch_url.rs register 调用)。
|
||||
///
|
||||
/// 参数:
|
||||
/// - url: 必填,http/https
|
||||
/// - max_length: 可选,markdown 截断长度 chars(默认 8000,clamp [500, 50000])
|
||||
/// - render: 可选,bool,默认 false。true 时用 obscura(JS 渲染 SPA/反爬),未装/失败回退静态。
|
||||
///
|
||||
/// 返回 {url, title, markdown, length, truncated, render_mode, ?hint}
|
||||
pub(crate) async fn execute_fetch_url(args: Value) -> anyhow::Result<Value> {
|
||||
// ── 参数解析 ──
|
||||
let url_raw = args.get("url").and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("缺少 url 参数"))?
|
||||
.trim()
|
||||
.to_string();
|
||||
if url_raw.is_empty() {
|
||||
anyhow::bail!("url 不能为空");
|
||||
}
|
||||
let max_length = (args.get("max_length").and_then(|v| v.as_u64())
|
||||
.unwrap_or(DEFAULT_MAX_LENGTH as u64) as usize)
|
||||
.clamp(MIN_MAX_LENGTH, MAX_MAX_LENGTH);
|
||||
let timeout_secs = args.get("timeout_secs").and_then(|v| v.as_u64())
|
||||
.unwrap_or(DEFAULT_TIMEOUT_SECS)
|
||||
.min(MAX_TIMEOUT_SECS)
|
||||
.max(1);
|
||||
let render = args.get("render").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
|
||||
// SSRF 校验对所有模式都必做。校验放入口而非 fetch_static 内,确保 render 分支与静态分支
|
||||
// 都先过 SSRF(原 fetch_static 内的校验保留作第二道,双重校验对安全是好事,无害)。
|
||||
let (_, host, port) = crate::commands::ai::http::validate_url(&url_raw)?;
|
||||
crate::commands::ai::http::resolve_and_check_host(&host, port).await?;
|
||||
|
||||
// ── render 分支 ──
|
||||
if render {
|
||||
match fetch_with_obscura(&url_raw, max_length).await {
|
||||
Ok(value) => {
|
||||
if value.get("render_mode").and_then(|v| v.as_str()) == Some("static_fallback") {
|
||||
tracing::info!("obscura 回退静态模式:hint={:?}",
|
||||
value.get("hint").and_then(|v| v.as_str()).unwrap_or(""));
|
||||
}
|
||||
return Ok(value);
|
||||
}
|
||||
Err(e) => {
|
||||
// 不该发生(fetch_with_obscura 内部已把所有错误转成回退+hint),但兜底再保险一道。
|
||||
tracing::warn!("obscura 分支异常({}),回退静态", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── 静态管线(reqwest + htmd,默认/回退共用) ──
|
||||
let mut result = fetch_static(&url_raw, max_length, timeout_secs, &args).await?;
|
||||
if render {
|
||||
// render=true 但走到了静态管线(未装/失败/异常)→ 标记并塞 hint。
|
||||
result["render_mode"] = json!("static_fallback");
|
||||
result["hint"] = json!(OBSCURA_HINT);
|
||||
} else {
|
||||
result["render_mode"] = json!("static");
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// 静态管线:reqwest GET + htmd 转 markdown + 去噪音 + 截断。
|
||||
///
|
||||
/// 从原 execute_fetch_url 抽出,供默认/回退共用。SSRF 校验在 execute_fetch_url 入口已做,
|
||||
/// 这里保留第二道校验(execute_fetch_url 改动前即有,移除会破坏单元测试触发的早期拒绝路径)。
|
||||
async fn fetch_static(
|
||||
url_raw: &str,
|
||||
max_length: usize,
|
||||
timeout_secs: u64,
|
||||
args: &Value,
|
||||
) -> anyhow::Result<Value> {
|
||||
// headers: 容许 LLM 传自定义头(如 User-Agent,部分站点拒默认 UA),复用 http_request 同款解析。
|
||||
let headers: HashMap<String, String> = match args.get("headers") {
|
||||
Some(Value::Object(m)) => m.iter()
|
||||
.filter_map(|(k, v)| {
|
||||
let s = match v {
|
||||
Value::String(s) => s.clone(),
|
||||
other => other.to_string(),
|
||||
};
|
||||
Some((k.clone(), s))
|
||||
})
|
||||
.collect(),
|
||||
_ => {
|
||||
// 默认带一个浏览器 UA:不少站点(如 MDN/GitHub)对非浏览器 UA 返回简化/拒绝页面,
|
||||
// 给一个主流 UA 拿到完整渲染 HTML,提升 markdown 质量。LLM 显式传 headers 则不覆盖。
|
||||
let mut h = HashMap::new();
|
||||
h.insert(
|
||||
"User-Agent".to_string(),
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36".to_string(),
|
||||
);
|
||||
h
|
||||
}
|
||||
};
|
||||
let body: Option<String> = None; // fetch_url 只 GET,无 body
|
||||
|
||||
// ── SSRF 校验(第二道,见函数注释) ──
|
||||
let (scheme, host, port) = crate::commands::ai::http::validate_url(url_raw)?;
|
||||
crate::commands::ai::http::resolve_and_check_host(&host, port).await?;
|
||||
|
||||
// ── 构建限制 client + 执行(GET,手动重定向循环复用 http.rs) ──
|
||||
let client = crate::commands::ai::http::build_client(Duration::from_secs(timeout_secs))?;
|
||||
let started = Instant::now();
|
||||
let resp = crate::commands::ai::http::execute_with_redirects(
|
||||
&client,
|
||||
reqwest::Method::GET,
|
||||
url_raw.to_string(),
|
||||
&headers,
|
||||
&body,
|
||||
crate::commands::ai::http::MAX_REDIRECTS,
|
||||
)
|
||||
.await?;
|
||||
let elapsed_ms = started.elapsed().as_millis() as u64;
|
||||
|
||||
let final_url = resp.url().to_string();
|
||||
let status = resp.status().as_u16();
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"fetch 失败:HTTP {} {}({})",
|
||||
status,
|
||||
resp.status().canonical_reason().unwrap_or(""),
|
||||
final_url
|
||||
);
|
||||
}
|
||||
|
||||
// ── 读取 body + 大小挡板 ──
|
||||
let bytes = resp.bytes().await
|
||||
.map_err(|e| anyhow::anyhow!("读取响应 body 失败: {}", e))?;
|
||||
let total_bytes = bytes.len();
|
||||
if total_bytes > MAX_HTML_BYTES {
|
||||
anyhow::bail!(
|
||||
"响应过大:{} 字节超过 {} 上限,fetch_url 不适合超大原始 HTML(改用 http_request 分段)",
|
||||
total_bytes, MAX_HTML_BYTES
|
||||
);
|
||||
}
|
||||
// 含 \0 → 二进制(非 HTML),直接拒(PDF/图片等走专用工具,fetch_url 只处理文本网页)
|
||||
if bytes.contains(&0u8) {
|
||||
anyhow::bail!("响应为二进制(非 HTML 文本),fetch_url 仅处理网页文档");
|
||||
}
|
||||
let html = String::from_utf8_lossy(&bytes).into_owned();
|
||||
|
||||
// ── 提取 title(<title>...</title>,正则避免引 HTML 解析重依赖) ──
|
||||
let title = extract_title(&html);
|
||||
|
||||
// ── HTML → markdown(htmd:turndown.js 移植,自动忽略 script/style) ──
|
||||
// htmd::convert 返回 Result(解析可能失败,如畸形 HTML)。失败时不阻塞 —— 降级用原文 lossy 文本,
|
||||
// LLM 仍可读到内容(质量差于 markdown 但不致工具整体失败)。
|
||||
let mut markdown = match htmd::convert(&html) {
|
||||
Ok(md) => md,
|
||||
Err(e) => {
|
||||
tracing::warn!("htmd 转 markdown 失败({}),降级用原始 HTML 文本", e);
|
||||
html.clone()
|
||||
}
|
||||
};
|
||||
|
||||
// ── 去噪音:折叠 ≥3 连续空行 → 2 行、trim 行尾空白 ──
|
||||
markdown = cleanup_markdown(&markdown);
|
||||
|
||||
// ── 截断(按 char 边界) ──
|
||||
let (markdown, truncated) = truncate_chars(&markdown, max_length);
|
||||
|
||||
Ok(json!({
|
||||
"url": final_url,
|
||||
"scheme": scheme,
|
||||
"status": status,
|
||||
"title": title,
|
||||
"markdown": markdown,
|
||||
"length": markdown.chars().count(),
|
||||
"html_bytes": total_bytes,
|
||||
"truncated": truncated,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 用 obscura(JS 渲染)抓取 URL 并转 markdown。
|
||||
///
|
||||
/// 流程:检测 obscura 在 PATH/已知路径 → spawn `obscura fetch --dump markdown <URL>`(带超时)
|
||||
/// → 取 stdout → 截断 → 返回与静态管线同构的 JSON(render_mode=obscura)。
|
||||
///
|
||||
/// 任何环节失败(未装 / spawn 失败 / 非零退出 / 超时)都不向上抛,而是回退静态管线:
|
||||
/// 内部调 fetch_static 拿结果,塞 hint 字段提示装 obscura。这样 render=true 在任何环境
|
||||
/// 都不会因 obscura 缺失/故障而让工具整体失败(引导式安装的核心保证)。
|
||||
async fn fetch_with_obscura(url_raw: &str, max_length: usize) -> anyhow::Result<Value> {
|
||||
let started = Instant::now();
|
||||
|
||||
// ── 检测 obscura ──
|
||||
let obscura_path = match find_obscura() {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
// 未装:回退静态 + hint。
|
||||
let mut fallback = fetch_static(url_raw, max_length, DEFAULT_TIMEOUT_SECS, &Value::Null).await?;
|
||||
fallback["render_mode"] = json!("static_fallback");
|
||||
fallback["hint"] = json!(OBSCURA_HINT);
|
||||
fallback["elapsed_ms"] = json!(started.elapsed().as_millis() as u64);
|
||||
return Ok(fallback);
|
||||
}
|
||||
};
|
||||
|
||||
// ── spawn obscura(带进程级超时) ──
|
||||
// obscura fetch --dump markdown <URL>:obscura 内部管 page load timeout(默认 30s),
|
||||
// 我们外加 tokio timeout 兜底 V8 启动/JIT 卡死等 obscura 自身超时管不到的场景。
|
||||
let mut cmd = tokio::process::Command::new(&obscura_path);
|
||||
cmd.args(["fetch", "--dump", "markdown", url_raw])
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped());
|
||||
|
||||
let output = match tokio::time::timeout(
|
||||
Duration::from_secs(OBSCURA_TIMEOUT_SECS),
|
||||
cmd.output(),
|
||||
).await {
|
||||
Ok(Ok(o)) => o,
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!("obscura spawn 失败({}),回退静态", e);
|
||||
let mut fallback = fetch_static(url_raw, max_length, DEFAULT_TIMEOUT_SECS, &Value::Null).await?;
|
||||
fallback["render_mode"] = json!("static_fallback");
|
||||
fallback["hint"] = json!(format!("obscura 启动失败({}),已用静态模式。{}", e, OBSCURA_HINT));
|
||||
fallback["elapsed_ms"] = json!(started.elapsed().as_millis() as u64);
|
||||
return Ok(fallback);
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!("obscura 超时({}s),回退静态", OBSCURA_TIMEOUT_SECS);
|
||||
let mut fallback = fetch_static(url_raw, max_length, DEFAULT_TIMEOUT_SECS, &Value::Null).await?;
|
||||
fallback["render_mode"] = json!("static_fallback");
|
||||
fallback["hint"] = json!(format!("obscura 渲染超时({}s),已用静态模式。{}", OBSCURA_TIMEOUT_SECS, OBSCURA_HINT));
|
||||
fallback["elapsed_ms"] = json!(started.elapsed().as_millis() as u64);
|
||||
return Ok(fallback);
|
||||
}
|
||||
};
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
tracing::warn!("obscura 非零退出({}),stderr: {}", output.status, stderr.trim());
|
||||
let mut fallback = fetch_static(url_raw, max_length, DEFAULT_TIMEOUT_SECS, &Value::Null).await?;
|
||||
fallback["render_mode"] = json!("static_fallback");
|
||||
fallback["hint"] = json!(format!("obscura 渲染失败(exit {}),已用静态模式。{}", output.status, OBSCURA_HINT));
|
||||
fallback["elapsed_ms"] = json!(started.elapsed().as_millis() as u64);
|
||||
return Ok(fallback);
|
||||
}
|
||||
|
||||
// ── 取 stdout(已是 markdown) ──
|
||||
let mut markdown = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
// obscura 偶发在 markdown 前后带空白/日志行,trim 首尾;内部仍走 cleanup 折叠空行。
|
||||
markdown = cleanup_markdown(markdown.trim());
|
||||
let html_bytes_est = markdown.len(); // obscura 不给原始 HTML 字节数,用 markdown 长度近似(仅信息字段)
|
||||
|
||||
// 截断(按 char 边界,与静态一致)
|
||||
let (markdown, truncated) = truncate_chars(&markdown, max_length);
|
||||
let elapsed_ms = started.elapsed().as_millis() as u64;
|
||||
|
||||
// title:obscura --dump markdown 不单独给 title,从 markdown 首个 H1/首行启发式取;取不到则 null。
|
||||
let title = extract_title_from_markdown(&markdown);
|
||||
// scheme:obscura 路径下重新 validate_url 拿 scheme(已在入口校验过,这里仅取字段值,失败兜底 https)。
|
||||
let scheme = crate::commands::ai::http::validate_url(url_raw)
|
||||
.map(|(s, _, _)| s)
|
||||
.unwrap_or_else(|_| "https".to_string());
|
||||
|
||||
Ok(json!({
|
||||
"url": url_raw,
|
||||
"scheme": scheme,
|
||||
"status": 200u16, // obscura 成功路径不暴露 HTTP status(已渲染),用 200 占位
|
||||
"title": title,
|
||||
"markdown": markdown,
|
||||
"length": markdown.chars().count(),
|
||||
"html_bytes": html_bytes_est,
|
||||
"truncated": truncated,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"render_mode": "obscura",
|
||||
}))
|
||||
}
|
||||
|
||||
/// 检测 obscura 是否已安装且可执行。优先级:PATH > 已知 npm-global 路径。
|
||||
///
|
||||
/// 用 `obscura --version` 探活(比 `which` 跨平台:Windows 无 which,且 --version 能确认
|
||||
/// 二进制可跑而非仅存在)。同步探活带 5s 超时防卡(对齐 module.rs::run_command 的 thread+channel 风格)。
|
||||
fn find_obscura() -> Option<String> {
|
||||
let candidates: &[&str] = &[
|
||||
"obscura", // PATH 优先(用户 npm i -g 后即在 PATH)
|
||||
"/d/NodeJS/npm-global/obscura", // 已知安装路径(用户环境,POSIX 形式 git-bash 友好)
|
||||
"/d/NodeJS/npm-global/obscura.exe",
|
||||
"D:\\NodeJS\\npm-global\\obscura.exe", // Windows 原生路径(tauri 在 Windows 跑)
|
||||
];
|
||||
|
||||
for cand in candidates {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let cand_owned = cand.to_string();
|
||||
std::thread::spawn(move || {
|
||||
let out = std::process::Command::new(&cand_owned)
|
||||
.arg("--version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.output();
|
||||
let _ = tx.send(out);
|
||||
});
|
||||
match rx.recv_timeout(Duration::from_secs(5)) {
|
||||
Ok(Ok(o)) if o.status.success() => return Some(cand.to_string()),
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// 从 markdown 启发式提取 title:首个 `# 一级标题` 或首个非空文本行(≤120 chars)。
|
||||
/// obscura dump markdown 无独立 title 字段,此为最佳近似(静态管线有 <title>,此函数仅 render 分支用)。
|
||||
fn extract_title_from_markdown(md: &str) -> Option<String> {
|
||||
for line in md.lines() {
|
||||
let t = line.trim();
|
||||
if t.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// 首个 H1 标题
|
||||
if let Some(rest) = t.strip_prefix("# ") {
|
||||
let title = rest.trim();
|
||||
if !title.is_empty() {
|
||||
return Some(title.to_string());
|
||||
}
|
||||
}
|
||||
// 否则取首个非空、非表格/代码块的行(去掉 markdown 强调前缀)
|
||||
if !t.starts_with('|') && !t.starts_with("```") {
|
||||
let cleaned: String = t.trim_start_matches(|c: char| c == '*' || c == '-').trim().to_string();
|
||||
if !cleaned.is_empty() && cleaned.chars().count() <= 120 {
|
||||
return Some(cleaned);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// 提取 HTML <title> 内容。case-insensitive,取首个,trim 空白。
|
||||
/// 用简单字符串扫描而非 regex/HTML 解析:title 标签结构固定且少,正则/解析引依赖不值。
|
||||
fn extract_title(html: &str) -> Option<String> {
|
||||
let lower = html.to_lowercase();
|
||||
let open = lower.find("<title")?;
|
||||
// 跳过 `<title ...>` 到 '>'(可能含属性,但 <title> 几乎不带属性)
|
||||
let after_open_tag = &html[open..];
|
||||
let gt = after_open_tag.find('>')?;
|
||||
let content_start = open + gt + 1;
|
||||
let rest = &html[content_start..];
|
||||
// 找 </title>(case-insensitive)
|
||||
let rest_lower = &lower[content_start..];
|
||||
let close = rest_lower.find("</title>")?;
|
||||
let title = &rest[..close];
|
||||
let trimmed = title.trim();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
// 去换行/多空格压缩,防 title 跨行带大量空白
|
||||
let squashed: String = trimmed.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
Some(squashed)
|
||||
}
|
||||
}
|
||||
|
||||
/// 折叠 ≥3 连续空行 → 恰好 2 行,trim 每行尾空白。htmd 输出常有连续空行(nav 块转空)。
|
||||
fn cleanup_markdown(md: &str) -> String {
|
||||
let mut out = String::with_capacity(md.len());
|
||||
let mut blank_run = 0usize;
|
||||
for line in md.split_inclusive('\n') {
|
||||
// split_inclusive 保留 '\n';末行可能无 '\n'
|
||||
let content = line.trim_end_matches('\n');
|
||||
let is_blank = content.trim().is_empty();
|
||||
if is_blank {
|
||||
blank_run += 1;
|
||||
if blank_run <= 2 {
|
||||
out.push('\n');
|
||||
}
|
||||
// blank_run > 2:丢弃(折叠)
|
||||
} else {
|
||||
blank_run = 0;
|
||||
// trim 行尾空白,保留行首缩进(markdown 列表/代码块缩进有意义)
|
||||
out.push_str(content.trim_end());
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// 按 char 边界截断 markdown。返回 (截断后 String, 是否截断)。
|
||||
/// 不切多字节 UTF-8 中间(char_indices 保证落在 char 边界)。
|
||||
fn truncate_chars(md: &str, max: usize) -> (String, bool) {
|
||||
let char_count = md.chars().count();
|
||||
if char_count <= max {
|
||||
return (md.to_string(), false);
|
||||
}
|
||||
// 找到第 max 个 char 的字节位置
|
||||
let byte_cutoff = md.char_indices()
|
||||
.nth(max)
|
||||
.map(|(idx, _)| idx)
|
||||
.unwrap_or(md.len());
|
||||
let head = &md[..byte_cutoff];
|
||||
let total = char_count;
|
||||
(
|
||||
format!(
|
||||
"{}\n\n... [markdown 已截断,原文 {} chars,仅保留前 {} chars]",
|
||||
head.trim_end(),
|
||||
total,
|
||||
max
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 单元测试
|
||||
//
|
||||
// 覆盖纯函数(extract_title / cleanup_markdown / truncate_chars),确定性零网络。
|
||||
// handler 集成层走真实网络 #[ignore](对齐 http.rs 测试策略)。
|
||||
// ============================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── extract_title ──
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_basic() {
|
||||
let html = "<html><head><title>Hello World</title></head><body>x</body></html>";
|
||||
assert_eq!(extract_title(html).as_deref(), Some("Hello World"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_case_insensitive() {
|
||||
let html = "<TITLE>Case Test</TITLE>";
|
||||
assert_eq!(extract_title(html).as_deref(), Some("Case Test"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_multiline_squashed() {
|
||||
let html = "<title>\n Multi\n Line Title \n</title>";
|
||||
assert_eq!(extract_title(html).as_deref(), Some("Multi Line Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_missing_returns_none() {
|
||||
let html = "<html><body>no title here</body></html>";
|
||||
assert_eq!(extract_title(html), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_empty_returns_none() {
|
||||
let html = "<title> </title>";
|
||||
assert_eq!(extract_title(html), None);
|
||||
}
|
||||
|
||||
// ── cleanup_markdown ──
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_collapses_many_blank_lines() {
|
||||
let md = "para1\n\n\n\n\npara2";
|
||||
let out = cleanup_markdown(md);
|
||||
// 4 连续空行折叠为 2
|
||||
assert_eq!(out, "para1\n\n\npara2\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_trims_trailing_whitespace() {
|
||||
let md = "line with trailing spaces \nnext";
|
||||
let out = cleanup_markdown(md);
|
||||
assert_eq!(out, "line with trailing spaces\nnext\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_preserves_leading_indent() {
|
||||
// markdown 代码块/列表缩进有意义,不能 trim 行首
|
||||
let md = " code line\n- list item";
|
||||
let out = cleanup_markdown(md);
|
||||
assert_eq!(out, " code line\n- list item\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_keeps_two_blank_lines() {
|
||||
let md = "a\n\n\nb";
|
||||
let out = cleanup_markdown(md);
|
||||
assert_eq!(out, "a\n\n\nb\n");
|
||||
}
|
||||
|
||||
// ── extract_title_from_markdown(仅 render=obscura 分支用) ──
|
||||
|
||||
#[test]
|
||||
fn test_md_title_from_h1() {
|
||||
let md = "# Rust 异步编程指南\n\n正文内容";
|
||||
assert_eq!(extract_title_from_markdown(md).as_deref(), Some("Rust 异步编程指南"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_md_title_fallback_first_line() {
|
||||
// 无 H1,取首个非空行(去掉强调前缀)
|
||||
let md = "**Welcome** to my site\n\n更多内容";
|
||||
assert_eq!(extract_title_from_markdown(md).as_deref(), Some("Welcome** to my site"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_md_title_skips_blank_and_code() {
|
||||
let md = "\n\n```\ncode block\n```\n# Real Title\nbody";
|
||||
assert_eq!(extract_title_from_markdown(md).as_deref(), Some("Real Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_md_title_empty_returns_none() {
|
||||
assert_eq!(extract_title_from_markdown(""), None);
|
||||
assert_eq!(extract_title_from_markdown(" \n\n "), None);
|
||||
}
|
||||
|
||||
// ── truncate_chars ──
|
||||
|
||||
#[test]
|
||||
fn test_truncate_under_limit_unchanged() {
|
||||
let md = "short content";
|
||||
let (out, trunc) = truncate_chars(md, 1000);
|
||||
assert!(!trunc);
|
||||
assert_eq!(out, "short content");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_over_limit_marks_truncated() {
|
||||
let md = "abcdefghij".repeat(1000); // 10000 chars
|
||||
let (out, trunc) = truncate_chars(&md, 100);
|
||||
assert!(trunc);
|
||||
assert!(out.contains("[markdown 已截断"));
|
||||
assert!(out.contains("原文 10000 chars"));
|
||||
assert!(out.contains("仅保留前 100 chars"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_respects_char_boundary_unicode() {
|
||||
// 中文每字 3 字节,截 5 char 应得 15 字节(不在中间切)
|
||||
let md = "你好世界测试内容"; // 8 chars,每字 3 字节
|
||||
let (out, trunc) = truncate_chars(md, 5);
|
||||
assert!(trunc);
|
||||
// 截断点应在 "你好世界测" 后(5 chars = 15 bytes),无半个字
|
||||
assert!(out.starts_with("你好世界测\n\n"));
|
||||
// 不应出现 U+FFFD(乱码替换符)
|
||||
assert!(!out.contains('\u{FFFD}'));
|
||||
}
|
||||
|
||||
// ── handler 参数边界 ──
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_missing_url_errors() {
|
||||
let args = json!({ "max_length": 1000 });
|
||||
let err = execute_fetch_url(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("缺少 url"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_empty_url_errors() {
|
||||
let args = json!({ "url": " " });
|
||||
let err = execute_fetch_url(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("url 不能为空"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_rejects_localhost() {
|
||||
let args = json!({ "url": "http://localhost:8080/" });
|
||||
let err = execute_fetch_url(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("localhost") || format!("{}", err).contains("SSRF"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_rejects_private_ip() {
|
||||
let args = json!({ "url": "http://169.254.169.254/latest/" });
|
||||
let err = execute_fetch_url(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("私网") || format!("{}", err).contains("SSRF"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_rejects_non_http_scheme() {
|
||||
let args = json!({ "url": "file:///etc/passwd" });
|
||||
let err = execute_fetch_url(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("协议"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_max_length_clamped_low() {
|
||||
// max_length=1 应 clamp 到 500,不发请求只验证参数解析不 panic(用 localhost 触发 SSRF 早期拒绝)
|
||||
let args = json!({ "url": "http://localhost/", "max_length": 1 });
|
||||
let _ = execute_fetch_url(args).await; // 不 panic 即通过
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_max_length_clamped_high() {
|
||||
let args = json!({ "url": "http://localhost/", "max_length": 999999 });
|
||||
let _ = execute_fetch_url(args).await; // 不 panic 即通过
|
||||
}
|
||||
|
||||
// ── 真实网络集成(#[ignore]:CI 无网时跳过) ──
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "需真实网络(example.com),CI 无网时跳过:cargo test -- --ignored"]
|
||||
async fn integration_fetch_example_com_markdown() {
|
||||
let args = json!({ "url": "https://example.com/" });
|
||||
let result = execute_fetch_url(args).await.expect("fetch example.com 应成功");
|
||||
assert_eq!(result["status"].as_u64().unwrap(), 200);
|
||||
assert_eq!(result["title"].as_str().unwrap_or(""), "Example Domain");
|
||||
let md = result["markdown"].as_str().unwrap_or("");
|
||||
assert!(md.contains("Example Domain"));
|
||||
assert!(!md.contains("<html")); // 应是 markdown 非 HTML
|
||||
assert_eq!(result["truncated"], false);
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,7 @@ const DEFAULT_TIMEOUT_SECS: u64 = 30;
|
||||
/// 超时硬上限(秒)。防 LLM 传超大值冻结会话(对齐 run_command MAX 思路,封顶更紧)。
|
||||
const MAX_TIMEOUT_SECS: u64 = 60;
|
||||
/// 重定向最大跳转数。防开放重定向被利用做 SSRF 中转(302 链绕过到内网)。
|
||||
const MAX_REDIRECTS: usize = 3;
|
||||
pub(crate) const MAX_REDIRECTS: usize = 3;
|
||||
/// body/headers 各自大小上限(防 LLM 传超大请求体)。
|
||||
const MAX_REQUEST_BODY_BYTES: usize = 1 * 1024 * 1024; // 1MB
|
||||
|
||||
@@ -85,7 +85,7 @@ pub(crate) fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
/// (词法层只挡字面量 IP/localhost,DNS rebinding 靠 resolve 后校验补防)。
|
||||
///
|
||||
/// 安全顺序:① scheme 白名单 ② host 非空 ③ 字面量 IP 私网拦截 ④ localhost/.*local 等域名拦截。
|
||||
fn validate_url(raw: &str) -> anyhow::Result<(String, String, u16)> {
|
||||
pub(crate) fn validate_url(raw: &str) -> anyhow::Result<(String, String, u16)> {
|
||||
let parsed = reqwest::Url::parse(raw)
|
||||
.map_err(|e| anyhow::anyhow!("URL 解析失败: {} ({})", raw, e))?;
|
||||
let scheme = parsed.scheme().to_lowercase();
|
||||
@@ -123,7 +123,7 @@ fn validate_url(raw: &str) -> anyhow::Result<(String, String, u16)> {
|
||||
///
|
||||
/// 注:用 blocking resolve(tokio::net 已在 runtime,但单次 resolve 短且同步 DNS 系统调用,
|
||||
/// spawn_blocking 会增加调度开销)。直接用 tokio::net::lookup_host 异步解析。
|
||||
async fn resolve_and_check_host(host: &str, port: u16) -> anyhow::Result<Vec<std::net::SocketAddr>> {
|
||||
pub(crate) async fn resolve_and_check_host(host: &str, port: u16) -> anyhow::Result<Vec<std::net::SocketAddr>> {
|
||||
// lookup_host 需 host:port 形式;host 可能含 IPv6 字面量,SocketAddr::to_string 会自动加 []
|
||||
let target = format!("{}:{}", host, port);
|
||||
let addrs: Vec<std::net::SocketAddr> = tokio::net::lookup_host(target)
|
||||
@@ -149,7 +149,7 @@ async fn resolve_and_check_host(host: &str, port: u16) -> anyhow::Result<Vec<std
|
||||
///
|
||||
/// reqwest 默认 follow ≤10 重定向且不暴露每跳 URL 校验钩子,故 Policy::none 关闭自动跟随,
|
||||
/// 在 execute_with_redirects 循环里逐跳校验 Location。
|
||||
fn build_client(timeout: Duration) -> anyhow::Result<reqwest::Client> {
|
||||
pub(crate) fn build_client(timeout: Duration) -> anyhow::Result<reqwest::Client> {
|
||||
reqwest::Client::builder()
|
||||
.timeout(timeout)
|
||||
.connect_timeout(Duration::from_secs(15))
|
||||
@@ -162,7 +162,7 @@ fn build_client(timeout: Duration) -> anyhow::Result<reqwest::Client> {
|
||||
///
|
||||
/// 返回最终响应(reqwest::Response)。重定向链每跳重新 validate_url + resolve_and_check_host,
|
||||
/// 防 302 Location: http://127.0.0.1/ 绕过初始 URL 校验。
|
||||
async fn execute_with_redirects(
|
||||
pub(crate) async fn execute_with_redirects(
|
||||
client: &reqwest::Client,
|
||||
method: reqwest::Method,
|
||||
initial_url: String,
|
||||
|
||||
@@ -30,11 +30,13 @@ pub mod agentic;
|
||||
pub mod augmentation;
|
||||
pub mod audit;
|
||||
pub mod code_intel;
|
||||
pub mod command_stream;
|
||||
pub mod commands;
|
||||
pub mod compress;
|
||||
pub mod conversation;
|
||||
pub mod event_bus;
|
||||
pub mod http;
|
||||
pub mod fetch_url;
|
||||
pub mod knowledge_inject;
|
||||
pub mod prompt;
|
||||
pub mod provider_pool;
|
||||
@@ -119,6 +121,19 @@ pub enum AiChatEvent {
|
||||
AiToolCallStarted { id: String, name: String, args: serde_json::Value, conversation_id: Option<String> },
|
||||
/// 工具调用完成
|
||||
AiToolCallCompleted { id: String, result: serde_json::Value, conversation_id: Option<String> },
|
||||
/// run_command 实时流式输出(治执行中黑盒)。
|
||||
///
|
||||
/// execute_with_heartbeat / ai_approve 调用 run_command 前,经 task-local sink 注入
|
||||
/// (AppHandle + tool_call_id + conv_id)。run_command handler 读 task-local,改走 shell
|
||||
/// `execute_streaming`,每读一行 stdout/stderr 即 emit 本事件(双写 app.emit + ai_event_bus)。
|
||||
/// 前端 listen 此事件可在工具卡 Started→Completed 间实时展示进度(编译/构建输出)。
|
||||
///
|
||||
/// 字段:
|
||||
/// - `id`:tool_call_id(对齐 Started/Completed,前端按 id 路由到对应工具卡)
|
||||
/// - `stream`:"stdout" | "stderr"(来源流,前端可差异化着色)
|
||||
/// - `line`:单行内容(已去换行,行级粒度 emit)
|
||||
/// - `conversation_id`:多对话路由
|
||||
AiCommandOutput { id: String, stream: String, line: String, conversation_id: Option<String> },
|
||||
/// 会话级信任自动放行（AE-2025-04 Session Trust）：
|
||||
/// 同会话已批准过同类操作（同工具+同目录），下次命中自动放行，前端显示轻量 toast。
|
||||
/// 与 AiToolCallStarted/Completed 正交——这三个事件描述一次信任放行调用生命周期：
|
||||
|
||||
@@ -116,6 +116,40 @@ fn system_prompt_parts(lang: &str) -> (&'static str, &'static str, &'static str)
|
||||
}
|
||||
}
|
||||
|
||||
/// 文档探索策略段(中/英):引导 AI 收到 URL 时走「嗅探→识别→提取→验证→应用」,
|
||||
/// 而非盲目对 URL 发 http_request 瞎试。
|
||||
///
|
||||
/// 机制治症状:实测 AI 收到产品/文档 URL 时,会直接 http_request 打原始 URL(HTML 噪声爆 token、
|
||||
/// 也调不出真正 API 端点),应先用 fetch_url 转成可读 markdown 理解文档,提取 base_url/鉴权/
|
||||
/// 端点路径后,再用 http_request 打正确端点验证。
|
||||
///
|
||||
/// - fetch_url = 读文档(URL→markdown,去 HTML 噪声);http_request = 调 API(POST/鉴权/原始响应)
|
||||
/// - 不瞎试 URL:先读文档提取端点,再验证;杜绝未读文档即对原始 URL 盲发请求
|
||||
///
|
||||
/// 仅段加在 prefix 末尾(聚焦段后),不破坏现有结构。
|
||||
fn doc_exploration_strategy_section(lang: &str) -> &'static str {
|
||||
match lang {
|
||||
"en" => "\n\
|
||||
## Documentation Exploration Strategy\n\
|
||||
When the user gives you a URL (product page / API docs / SDK guide), follow this flow instead of blindly firing http_request at the raw URL:\n\
|
||||
1. **Sniff**: call `fetch_url` to turn the URL into clean markdown (strips HTML noise) and read what the page actually says.\n\
|
||||
2. **Identify intent**: is it an API doc (endpoints/auth), a product capability page, a config spec, or a code sample?\n\
|
||||
3. **Extract keys**: pull out base_url, provider_type, auth scheme, model list, endpoint paths — write them down explicitly before calling anything.\n\
|
||||
4. **Verify**: only now use `http_request` against the correct endpoint you extracted (e.g. `GET {base_url}/v1/models` with the auth header) to confirm it works.\n\
|
||||
5. **Apply**: configure the Provider / write code / transcribe into a file via `write_file`.\n\
|
||||
Rule: `fetch_url` is for *reading* a URL (docs/web/API description, markdown output); `http_request` is for *calling* an API (POST/auth/raw response). Never guess an endpoint — read the doc first, extract, then verify.\n",
|
||||
_ => "\n\
|
||||
## 文档探索策略\n\
|
||||
收到用户给的 URL(产品页/API 文档/SDK 说明)时,按以下流程走,不要盲目对原始 URL 发 http_request 瞎试:\n\
|
||||
1. **嗅探**:先调 `fetch_url` 把 URL 转成干净的 markdown(剥离 HTML 噪声),读明白页面到底写了什么。\n\
|
||||
2. **识别意图**:是 API 文档(端点/鉴权)、产品能力页、配置规格,还是代码示例?\n\
|
||||
3. **提取关键**:把 base_url、provider_type、鉴权方式、模型清单、端点路径显式列出来,再决定调什么。\n\
|
||||
4. **验证**:此时才用 `http_request` 打你提取出的正确端点(如带鉴权头 `GET {base_url}/v1/models`)确认可用。\n\
|
||||
5. **应用**:配置 Provider / 编码 / 转写到文件(write_file)。\n\
|
||||
铁律:`fetch_url` 用于**读** URL(文档/网页/API 说明,输出 markdown);`http_request` 用于**调** API(POST/鉴权/原始响应)。严禁猜端点——先读文档提取,再验证。\n",
|
||||
}
|
||||
}
|
||||
|
||||
/// 构建系统提示词（环境信息 + 固定前缀 + 当前项目/任务**全局清单**）
|
||||
///
|
||||
/// 本函数注入"全貌"清单:最近 20 项目 + 20 任务的 name/status/description(无 path),
|
||||
@@ -155,6 +189,8 @@ pub(crate) async fn build_system_prompt_with_excluded(
|
||||
let (prefix, proj_label, task_label) = system_prompt_parts(lang);
|
||||
let mut prompt = env_profile_line();
|
||||
prompt.push_str(prefix);
|
||||
// 文档探索策略段(URL→fetch_url 嗅探→识别→提取→验证→应用,治盲目 http_request 瞎试)
|
||||
prompt.push_str(doc_exploration_strategy_section(lang));
|
||||
|
||||
// 附加当前数据上下文
|
||||
if let Ok(projects) = state.projects.list_active().await {
|
||||
@@ -343,6 +379,28 @@ mod tests {
|
||||
assert!(prefix.contains("聚焦准则"));
|
||||
}
|
||||
|
||||
// 文档探索策略段(中/英)存在且含关键引导
|
||||
#[test]
|
||||
fn doc_exploration_strategy_section_present_zh_en() {
|
||||
let zh = doc_exploration_strategy_section("zh-CN");
|
||||
assert!(zh.contains("## 文档探索策略"));
|
||||
assert!(zh.contains("fetch_url"));
|
||||
assert!(zh.contains("http_request"));
|
||||
assert!(zh.contains("嗅探"));
|
||||
assert!(zh.contains("识别意图"));
|
||||
assert!(zh.contains("提取关键"));
|
||||
assert!(zh.contains("验证"));
|
||||
assert!(zh.contains("应用"));
|
||||
|
||||
let en = doc_exploration_strategy_section("en");
|
||||
assert!(en.contains("## Documentation Exploration Strategy"));
|
||||
assert!(en.contains("Sniff"));
|
||||
assert!(en.contains("Identify intent"));
|
||||
assert!(en.contains("Extract keys"));
|
||||
assert!(en.contains("Verify"));
|
||||
assert!(en.contains("Apply"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_prompt_unaffected_by_focus_addition() {
|
||||
// 压缩 prompt 是独立函数,聚焦段改动不应波及
|
||||
|
||||
@@ -457,6 +457,7 @@ pub fn build_ai_tool_registry(
|
||||
register_data_tools(&mut registry, db);
|
||||
register_file_tools(&mut registry, allowed_dirs, data_dir);
|
||||
register_http_tools(&mut registry);
|
||||
register_fetch_url_tool(&mut registry);
|
||||
registry
|
||||
}
|
||||
|
||||
@@ -476,6 +477,19 @@ fn register_http_tools(registry: &mut AiToolRegistry) {
|
||||
super::tools::http::register(registry);
|
||||
}
|
||||
|
||||
/// fetch_url AI 工具注册(1 个:URL → markdown 文档嗅探)— 只读 GET 网页文档。
|
||||
/// 不持 db,纯 reqwest GET + htmd HTML→markdown 转换。SSRF 防护复用 commands/ai/http.rs
|
||||
/// (validate_url / resolve_and_check_host / execute_with_redirects,经 pub(crate) 暴露)。
|
||||
///
|
||||
/// 风险:Low(只读 GET,无副作用)。与 http_request 分工:fetch_url 输出已清洗 markdown 供 LLM 理解文档,
|
||||
/// http_request 输出原始响应 body 供 API 调用(POST/鉴权)。
|
||||
///
|
||||
/// tool_registry 拆分:声明式注册(tools/fetch_url.rs),handler 在 commands/ai/fetch_url.rs。
|
||||
/// 基线测试 test_build_ai_tool_registry_baseline_tool_count 守护总量 + 工具名集合稳定。
|
||||
fn register_fetch_url_tool(registry: &mut AiToolRegistry) {
|
||||
super::tools::fetch_url::register(registry);
|
||||
}
|
||||
|
||||
/// 数据层 AI 工具注册(25 个持 db 的 CRUD/状态机/工作流/知识图谱工具)——从 build_ai_tool_registry 抽出。
|
||||
///
|
||||
/// 工具闭包捕获 `db: &Arc<Database>` Arc 重建 Repo(列表/创建/更新/删除/状态推进/工作流/任务关联)。
|
||||
@@ -1117,7 +1131,7 @@ mod tests {
|
||||
// 任一层漏移 register 调用,此测试立即红。工具名集合也断言,防 rename 致 LLM tool 突变。
|
||||
// ============================================================
|
||||
|
||||
/// build_ai_tool_registry 应注册恰好 41 个工具(27 data + 13 file + 1 http),且工具名集合稳定。
|
||||
/// build_ai_tool_registry 应注册恰好 51 个工具(36 data + 13 file + 1 http + 1 fetch_url),且工具名集合稳定。
|
||||
///
|
||||
/// 用 in-memory SQLite(Database::open_in_memory 自跑迁移),构造零外部依赖的 db,
|
||||
// 不实际执行任何 handler——仅断言注册阶段的定义完整性,故无需真实数据。
|
||||
@@ -1151,10 +1165,13 @@ mod tests {
|
||||
// update_idea/delete_idea(2026-08-01): data 层 34→36(补 AI 改/删灵感专用工具,
|
||||
// idea_repo.update_field / soft_delete 早已具备,工具层此前缺失致 AI 误用 update_task)。
|
||||
// 50 = 36 data + 13 file + 1 http。
|
||||
// fetch_url(2026-08-01): http 层 1→2(新增 URL→markdown 文档嗅探,只读 GET,
|
||||
// htmd HTML→markdown + 去噪音 + 截断,SSRF 复用 http.rs)。
|
||||
// 51 = 36 data + 13 file + 1 http + 1 fetch_url。
|
||||
assert_eq!(
|
||||
registry.len(),
|
||||
50,
|
||||
"工具总数应为 50(36 data + 13 file + 1 http),实际 {}", registry.len()
|
||||
51,
|
||||
"工具总数应为 51(36 data + 13 file + 1 http + 1 fetch_url),实际 {}", registry.len()
|
||||
);
|
||||
|
||||
// 工具名集合基线:防 rename / 漏注册 / 误删除。
|
||||
@@ -1192,6 +1209,8 @@ mod tests {
|
||||
"grep", "detect_environment",
|
||||
// ── http 层 (1) ──
|
||||
"http_request",
|
||||
// ── fetch_url 层 (1) ──(URL → markdown 文档嗅探,只读 GET,与 http_request 分工)
|
||||
"fetch_url",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
//! fetch_url AI 工具声明式注册(URL → markdown 文档嗅探,只读 GET)。
|
||||
//!
|
||||
//! 工具职责:GET 网页 URL → HTML 转 markdown(htmd)→ 去噪音 + 截断 → 返回 {url,title,markdown,...}。
|
||||
//! 让 LLM 高效理解网页文档,替代 http_request 拿原始 HTML(噪声大、爆 token)。
|
||||
//!
|
||||
//! 风险:Low(只读 GET,无副作用)。SSRF 防护与 http_request 共享同一套(validate_url +
|
||||
//! resolve_and_check_host + 重定向每跳校验,见 commands/ai/http.rs)。
|
||||
//!
|
||||
//! handler body 在 commands/ai/fetch_url.rs::execute_fetch_url,声明式注册收敛样板。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use df_ai::ai_tools::{AiToolRegistry, RiskLevel};
|
||||
use df_ai::declare_tool;
|
||||
|
||||
/// 注册 fetch_url 工具到 `$registry`(无 db 捕获,纯网络 GET + htmd 转换)。
|
||||
///
|
||||
/// - name/desc/schema 面向 LLM 的工具说明
|
||||
/// - risk=Low(只读 GET)
|
||||
/// - handler 转调 fetch_url.rs::execute_fetch_url(SSRF 防护 + htmd + 截断全在那)
|
||||
pub fn register(registry: &mut AiToolRegistry) {
|
||||
// 无捕获:占位 Arc<()>(handler 不持 db,仅转调 fetch_url.rs)。
|
||||
let dummy: Arc<()> = Arc::new(());
|
||||
// schema:url(必填)+ max_length(可选,默认 8000)+ render(可选,默认 false)。
|
||||
// headers 用手工 serde_json::Map 表达(additionalProperties 需对象 schema,object_schema 不支持)。
|
||||
let schema = {
|
||||
let mut props = serde_json::Map::new();
|
||||
props.insert("url".into(), serde_json::json!({ "type": "string", "description": "要获取的网页 URL,仅 http/https(拒私网/localhost,SSRF 防护)" }));
|
||||
props.insert("max_length".into(), serde_json::json!({ "type": "integer", "description": "返回 markdown 的截断长度(chars,默认 8000,clamp [500,50000])", "minimum": 500, "maximum": 50000, "default": 8000 }));
|
||||
props.insert("render".into(), serde_json::json!({ "type": "boolean", "description": "是否用 obscura(JS 渲染)抓取,默认 false(静态 reqwest+htmd)。render=true 适合 SPA/JS 动态渲染/反爬文档:检测本地 obscura(Rust 无头浏览器,自带 V8),已装则 spawn 渲染,未装/失败/超时自动回退静态并附 hint。需装 obscura:github.com/h4ckf0r0day/obscura", "default": false }));
|
||||
props.insert("headers".into(), serde_json::json!({ "type": "object", "description": "可选请求头 map<string,string>(如自定义 User-Agent)。默认带主流浏览器 UA 以拿完整渲染 HTML(仅静态模式生效,render=true 时由 obscura 自管 UA)", "additionalProperties": { "type": "string" } }));
|
||||
props.insert("timeout_secs".into(), serde_json::json!({ "type": "integer", "description": "超时秒数(默认 30,上限 60,仅静态模式生效;render=true 用固定 45s obscura 超时)", "minimum": 1, "maximum": 60 }));
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": props,
|
||||
"required": ["url"],
|
||||
})
|
||||
};
|
||||
declare_tool!(
|
||||
registry,
|
||||
dummy: Arc<()>,
|
||||
"fetch_url",
|
||||
"获取网页 URL 内容并转为 markdown(HTML 清洗 + 截断),用于高效理解网页文档(API 文档/博客/技术资料)。只读 GET,自动剥离 script/style/nav 等噪声,提取 title,按 max_length 截断。返回 {url, title, markdown, length, truncated, render_mode, ?hint}。安全:仅 http/https,拒绝私网/保留 IP(SSRF 防护含 DNS resolve 后校验),重定向≤3 跳。render=true 用 obscura 渲染 JS(SPA/反爬,需装 obscura,未装自动回退静态)。如需 POST/鉴权/原始响应用 http_request",
|
||||
RiskLevel::Low,
|
||||
schema: schema,
|
||||
args => {
|
||||
// 转调 fetch_url.rs handler(SSRF 防护 + htmd 转换 + 去噪音 + 截断全在那)
|
||||
crate::commands::ai::fetch_url::execute_fetch_url(args).await
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -45,9 +45,10 @@ use crate::commands::ai::tool_registry::{
|
||||
FILE_LOCKS, FileGrepHit,
|
||||
};
|
||||
|
||||
// run_command 依赖 df_execute::shell::{execute, ShellRequest} + std HashMap
|
||||
// new_id: delete_file 软删除备份命名(原 register_file_tools 闭包引用,经 df_types::types::new_id)
|
||||
use df_execute::shell::{execute, ShellRequest};
|
||||
// run_command 依赖 df_execute::shell::{execute_streaming, ShellRequest, StreamKind} + std HashMap
|
||||
// execute_streaming:run_command 专用流式(spawn 逐行读 stdout/stderr,回调 emit AiCommandOutput 治执行黑盒)。
|
||||
// new_id: delete_file 软删除备份命名(原 register_file_tools 闭包引用,经 df_types::types::new_id)。
|
||||
use df_execute::shell::{execute_streaming, ShellRequest, StreamKind};
|
||||
use df_types::types::new_id;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -1032,7 +1033,12 @@ pub fn register(
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let result = execute(request).await.map_err(|e| {
|
||||
let result = execute_streaming(request, |kind: StreamKind, line: &str| {
|
||||
// 实时流式 emit:每读一行 stdout/stderr 即 emit AiCommandOutput。
|
||||
// task-local 未注入(command_stream::scope 未调用)→ emit_output 静默 noop,
|
||||
// 不阻断命令(等价原 execute 一次性返回,无副作用)。
|
||||
crate::commands::ai::command_stream::emit_output(kind, line);
|
||||
}).await.map_err(|e| {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("命令执行超时") {
|
||||
anyhow::anyhow!(
|
||||
|
||||
@@ -14,6 +14,7 @@ pub mod task;
|
||||
pub mod task_graph;
|
||||
pub mod git;
|
||||
pub mod http;
|
||||
pub mod fetch_url;
|
||||
pub mod workflow;
|
||||
pub mod idea;
|
||||
pub mod trash;
|
||||
|
||||
Reference in New Issue
Block a user