新增: AI 工具(download_file + http output_file + grep 窗口 + fetch_search 搜索 + get_app_config + generate_image)
download_file 跨平台 URL 到文件流式下载;http_request output_file 落盘 + 截断可配; grep context_chars 大单行窗口;fetch_search DuckDuckGo 免 key 搜索;get_app_config 只读返当前配置(治 AI 查配置绕 PowerShell);generate_image SenseNova 图像生成。
This commit is contained in:
@@ -0,0 +1,781 @@
|
||||
//! fetch_search AI 工具 — 搜索引擎查询(DuckDuckGo HTML 免 key,GET → 解析结果列表)
|
||||
//!
|
||||
//! 设计目标:让 LLM 拿到「搜索」能力。用户给"调研 X"时,AI 需先搜索找文档入口,
|
||||
//! 再 fetch_url 抓。当前无搜索工具,LLM 只能瞎猜 URL。本工具 GET DuckDuckGo HTML 端点
|
||||
//! (免 API key),解析结果列表(标题+URL+摘要),让 AI 据此挑入口用 fetch_url 深读。
|
||||
//!
|
||||
//! ## 与 fetch_url / http_request 的分工
|
||||
//!
|
||||
//! - `http_request`:结构化 API 调用(POST/鉴权/JSON),拿原始响应 body。
|
||||
//! - `fetch_url`:**已知 URL** → markdown 文档嗅探,LLM 拿到正文。
|
||||
//! - `fetch_search`:**未知 URL** → 搜索引擎查询,LLM 拿到候选 URL 列表 + 摘要,
|
||||
//! 再据结果挑入口用 fetch_url 深读。补「搜索」缺口。
|
||||
//!
|
||||
//! ## HTML 解析方式:手写字符串扫描(非 scraper/html5ever)
|
||||
//!
|
||||
//! DDG HTML 端点结构稳定且单一来源,引入 scraper(html5ever + selectors + cssparser,
|
||||
//! 重依赖 ~5MB 编译产物)不值。对齐 fetch_url::extract_title 的简单字符串扫描风格:
|
||||
//! 在 HTML 中找 `class="result__a"` 锚点 → 取 href + 文本;找 `result__snippet` → 取摘要。
|
||||
//! 解析纯函数化(extract_results),零网络依赖,单测用 fixture HTML 字符串验证。
|
||||
//!
|
||||
//! ## DDG 重定向 URL 解包
|
||||
//!
|
||||
//! DDG 把结果 URL 包成 `https://duckduckgo.com/l/?uddg=<percent-encoded real url>&rut=...`
|
||||
//! 重定向跳板(统计/反爬)。本工具解析 `uddg=` 查询参数,percent-decode 还原真实 URL,
|
||||
//! LLM 拿到的就是可直接 fetch_url 的目标地址。解析失败(无 uddg 参数 / 非 DDG 跳板 URL)
|
||||
//! 保留原 href 兜底(可能是 //duckduckgo.com/... 内部页,LLM 至少能看到)。
|
||||
//!
|
||||
//! ## 安全(SSRF 防护 — 与 http_request / fetch_url 共享同一套)
|
||||
//!
|
||||
//! 复用 `http.rs` 的 `validate_url` / `resolve_and_check_host` / `build_client` /
|
||||
//! `execute_with_redirects`:① 协议白名单(仅 http/https)② 私网 IP 黑名单 ③ DNS resolve
|
||||
//! 后校验 IP(防 rebinding)④ 重定向 ≤3 跳每跳重校验。fetch_search 只发 GET 到固定 DDG 端点。
|
||||
//!
|
||||
//! ## 反爬评估
|
||||
//!
|
||||
//! DDG HTML 端点(html.duckduckgo.com/html/)对带浏览器 UA 的请求通常友好(无 key/无 token)。
|
||||
//! 本工具默认带主流浏览器 UA(与 fetch_url 一致)。若反爬升级(403/空结果),不硬上 ——
|
||||
//! handler 把 HTTP 状态/空结果如实返回,LLM 据此判断(不改用代理/绕过,避免对抗反爬)。
|
||||
//! 间歇限流:单次工具调用是单 GET,无高频请求,不内置 rate limit(并发由 LLM loop 控制,
|
||||
//! 搜索是低频操作,不至触发频控)。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// DuckDuckGo HTML 搜索端点(免 API key,GET 返回结果列表 HTML)。
|
||||
const DDG_HTML_ENDPOINT: &str = "https://html.duckduckgo.com/html/";
|
||||
|
||||
/// 默认返回结果数。LLM 单次搜索 5 条足以挑入口(太多撑爆 context)。
|
||||
const DEFAULT_MAX_RESULTS: usize = 5;
|
||||
/// 返回结果数下限。防 LLM 传极小值拿到无意义单条。
|
||||
const MIN_MAX_RESULTS: usize = 1;
|
||||
/// 返回结果数上限。防 LLM 传极大值仍撑爆 context(10 条覆盖多数场景)。
|
||||
const MAX_MAX_RESULTS: usize = 10;
|
||||
|
||||
/// 默认请求超时(秒)。与 http_request / fetch_url 一致。
|
||||
const DEFAULT_TIMEOUT_SECS: u64 = 30;
|
||||
/// 超时硬上限(秒)。与 http_request / fetch_url 一致。
|
||||
const MAX_TIMEOUT_SECS: u64 = 60;
|
||||
|
||||
/// 响应 body 字节上限。DDG HTML 结果页通常 <500KB,2MB 足够且防 OOM(对齐 fetch_url)。
|
||||
const MAX_HTML_BYTES: usize = 2 * 1024 * 1024;
|
||||
|
||||
/// fetch_search 工具 handler 入口(供 tools/fetch_search.rs register 调用)。
|
||||
///
|
||||
/// 参数:
|
||||
/// - query: 必填,搜索关键词
|
||||
/// - max_results: 可选,返回结果数(默认 5,clamp [1, 10])
|
||||
///
|
||||
/// 返回 {query, results:[{title, url, snippet}], count}
|
||||
pub(crate) async fn execute_fetch_search(args: Value) -> anyhow::Result<Value> {
|
||||
// ── 参数解析 ──
|
||||
let query = args.get("query").and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("缺少 query 参数"))?
|
||||
.trim()
|
||||
.to_string();
|
||||
if query.is_empty() {
|
||||
anyhow::bail!("query 不能为空");
|
||||
}
|
||||
let max_results = (args.get("max_results").and_then(|v| v.as_u64())
|
||||
.unwrap_or(DEFAULT_MAX_RESULTS as u64) as usize)
|
||||
.clamp(MIN_MAX_RESULTS, MAX_MAX_RESULTS);
|
||||
let timeout_secs = args.get("timeout_secs").and_then(|v| v.as_u64())
|
||||
.unwrap_or(DEFAULT_TIMEOUT_SECS)
|
||||
.min(MAX_TIMEOUT_SECS)
|
||||
.max(1);
|
||||
|
||||
// ── 构造请求 URL:percent-encode query 拼 DDG 端点 ?q=<encoded> ──
|
||||
// DDG HTML 端点接受 GET ?q=<url-encoded query>。encode 严格 RFC 3986(query 段)。
|
||||
let encoded_query = percent_encode_query(&query);
|
||||
let search_url = format!("{}?q={}", DDG_HTML_ENDPOINT, encoded_query);
|
||||
|
||||
// ── SSRF 校验(对固定 DDG 端点也走标准防护:协议/host/DNS resolve IP) ──
|
||||
let (_scheme, host, port) = crate::commands::ai::http::validate_url(&search_url)?;
|
||||
crate::commands::ai::http::resolve_and_check_host(&host, port).await?;
|
||||
|
||||
// ── 构建限制 client + 执行 GET(复用 http.rs 重定向循环,带浏览器 UA) ──
|
||||
// DDG HTML 端点对非浏览器 UA 可能返回简化页/拒,带主流 UA 拿完整结果列表(对齐 fetch_url)。
|
||||
let mut headers: HashMap<String, String> = HashMap::new();
|
||||
headers.insert(
|
||||
"User-Agent".to_string(),
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36".to_string(),
|
||||
);
|
||||
// Accept-Language:让 DDG 返回匹配用户语言偏好的结果(中英文兼顾)。
|
||||
headers.insert(
|
||||
"Accept-Language".to_string(),
|
||||
"en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7".to_string(),
|
||||
);
|
||||
|
||||
let client = crate::commands::ai::http::build_client(Duration::from_secs(timeout_secs))?;
|
||||
let started = Instant::now();
|
||||
let resp = crate::commands::ai::http::execute_with_redirects(
|
||||
&client,
|
||||
reqwest::Method::GET,
|
||||
search_url.clone(),
|
||||
&headers,
|
||||
&None,
|
||||
crate::commands::ai::http::MAX_REDIRECTS,
|
||||
)
|
||||
.await?;
|
||||
let elapsed_ms = started.elapsed().as_millis() as u64;
|
||||
|
||||
let final_url = resp.url().to_string();
|
||||
let status = resp.status().as_u16();
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"搜索失败:HTTP {} {}({})。DDG 可能限流/反爬,稍后重试或换关键词",
|
||||
status,
|
||||
resp.status().canonical_reason().unwrap_or(""),
|
||||
final_url
|
||||
);
|
||||
}
|
||||
|
||||
// ── 读取 body + 大小挡板 ──
|
||||
let bytes = resp.bytes().await
|
||||
.map_err(|e| anyhow::anyhow!("读取响应 body 失败: {}", e))?;
|
||||
let total_bytes = bytes.len();
|
||||
if total_bytes > MAX_HTML_BYTES {
|
||||
anyhow::bail!(
|
||||
"响应过大:{} 字节超过 {} 上限,fetch_search 拿到异常大响应",
|
||||
total_bytes, MAX_HTML_BYTES
|
||||
);
|
||||
}
|
||||
let html = String::from_utf8_lossy(&bytes).into_owned();
|
||||
|
||||
// ── 解析结果列表(纯函数,单测用 fixture HTML 验证) ──
|
||||
let mut results = extract_results(&html);
|
||||
// 截断到 max_results(解析可能拿到更多,DDG 单页通常 ~30 条)
|
||||
results.truncate(max_results);
|
||||
|
||||
let count = results.len();
|
||||
tracing::info!(
|
||||
"fetch_search: query={:?} 拿到 {} 条结果(HTML {} 字节,{}ms)",
|
||||
query, count, total_bytes, elapsed_ms
|
||||
);
|
||||
|
||||
Ok(json!({
|
||||
"query": query,
|
||||
"results": results,
|
||||
"count": count,
|
||||
"status": status,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 从 DDG HTML 结果页解析结果列表(标题+URL+摘要)。
|
||||
///
|
||||
/// DDG HTML 每条结果结构(简化):
|
||||
/// ```text
|
||||
/// <div class="result ...">
|
||||
/// <h2 class="result__title">
|
||||
/// <a rel="nofollow" class="result__a" href="https://duckduckgo.com/l/?uddg=<encoded>&rut=...">Title</a>
|
||||
/// </h2>
|
||||
/// <a class="result__snippet" href="...">Snippet text...</a>
|
||||
/// </div>
|
||||
/// ```
|
||||
///
|
||||
/// 解析策略:扫描所有 `class="result__a"` 锚点 → 取 href + 内联文本;同序找
|
||||
/// `class="result__snippet"` → 取摘要。两者各扫一遍再按出现顺序对齐(DDG 标题与摘要
|
||||
/// 在每条结果内同序出现,故按位置一一对应;若数量不等则按较少者对齐)。
|
||||
///
|
||||
/// URL 解包:DDG 把真实 URL 包在 `uddg=<percent-encoded>` 查询参数(跳板重定向),
|
||||
/// 此处还原(percent-decode)让 LLM 拿到可直 fetch_url 的目标 URL。
|
||||
fn extract_results(html: &str) -> Vec<Value> {
|
||||
let titles_with_url: Vec<(String, String)> = extract_result_anchors(html);
|
||||
let snippets: Vec<String> = extract_snippets(html);
|
||||
|
||||
let n = titles_with_url.len().max(snippets.len());
|
||||
let mut results: Vec<Value> = Vec::with_capacity(n);
|
||||
for i in 0..n {
|
||||
let title_url = titles_with_url.get(i);
|
||||
let snippet = snippets.get(i);
|
||||
match (title_url, snippet) {
|
||||
(Some((title, href)), snip) => {
|
||||
let url = unwrap_ddg_redirect(href);
|
||||
results.push(json!({
|
||||
"title": title,
|
||||
"url": url,
|
||||
"snippet": snip.cloned().unwrap_or_default(),
|
||||
}));
|
||||
}
|
||||
(None, Some(snip)) => {
|
||||
// 残留 snippet 无对应标题(罕见,DDG 结构异常时兜底):跳过,无标题无 URL 无意义。
|
||||
// 不构造半残条目,保持每条都有 title+url(让 LLM 能 fetch_url)。
|
||||
}
|
||||
(None, None) => break,
|
||||
}
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
/// 提取所有 `class="result__a"` 锚点的 (内联文本, href),保持文档顺序。
|
||||
///
|
||||
/// 用字符串扫描(对齐 fetch_url::extract_title 风格),不引 scraper 重依赖。
|
||||
/// class 属性顺序不固定(DDG 偶有 `<a rel="nofollow" class="result__a" ...>` 或
|
||||
/// `<a class="result__a" ...>`),扫 `class="result__a"` 子串命中锚点起点,再从该点向后
|
||||
/// 找最近的 `<a ...>` 起标签尾(`>`),取 href 值 + 标签内文本(到 `</a>`)。
|
||||
fn extract_result_anchors(html: &str) -> Vec<(String, String)> {
|
||||
// MARKER 用 `result__a`(裸 class 名),命中后检查后续字符做边界判定:
|
||||
// 必须后跟非标识符字符(" 或空格,即 class 值结束或多个 class 间的分隔),避免 `result__a`
|
||||
// 子串误命中 `result__article` / `result__a_extra` 等同类 class(DDG 有 result__* 系列)。
|
||||
const MARKER: &str = "result__a";
|
||||
let marker_end_byte = MARKER.len();
|
||||
let mut out: Vec<(String, String)> = Vec::new();
|
||||
let mut search_from = 0usize;
|
||||
while search_from < html.len() {
|
||||
// 找下一个 "result__a" 出现位置(锚点 class 标记)
|
||||
let Some(rel_class) = html[search_from..].find(MARKER) else { break };
|
||||
let class_pos = search_from + rel_class;
|
||||
let after_marker = class_pos + marker_end_byte;
|
||||
// 边界检查:MARKER 后的字符必须是非标识符字符(" 或空格 或 >,即 class 值结束),
|
||||
// 否则可能是 `result__article` 这类以 `result__a` 为前缀的 class,跳过。
|
||||
let next_char = html[after_marker..].chars().next();
|
||||
let is_boundary = match next_char {
|
||||
Some(c) => !(c.is_ascii_alphanumeric() || c == '_' || c == '-'),
|
||||
None => true, // 字符串尾(罕见,视为边界)
|
||||
};
|
||||
if !is_boundary {
|
||||
// 非边界(如 result__article):跳过本命中继续找
|
||||
search_from = after_marker;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 从 class 位置向前找最近的 `<a ` 起点(class 属性必在 <a ...> 标签内)
|
||||
// 倒扫到 class_pos 之前的最后一个 `<a `(`<a` 后接空白表示开标签)
|
||||
let head = &html[..class_pos];
|
||||
let Some(a_open_rel) = head.rfind("<a ") else {
|
||||
search_from = after_marker;
|
||||
continue;
|
||||
};
|
||||
// 从 a 开标签起点向后找标签尾 `>`
|
||||
let a_open_end_rel = html[a_open_rel..].find('>');
|
||||
let Some(a_open_end_offset) = a_open_end_rel else {
|
||||
search_from = after_marker;
|
||||
continue;
|
||||
};
|
||||
let a_open_end = a_open_rel + a_open_end_offset; // `>` 位置
|
||||
let a_open_tag = &html[a_open_rel..=a_open_end]; // 含 `<a ...>`
|
||||
|
||||
// 提取 href 值(href="..." 或 href='...')
|
||||
let href = extract_attr(a_open_tag, "href").unwrap_or_default();
|
||||
|
||||
// 取标签内文本(从 `>` 后到 `</a>`)
|
||||
let text_start = a_open_end + 1;
|
||||
let text_end = html[text_start..]
|
||||
.find("</a>")
|
||||
.map(|p| text_start + p)
|
||||
.unwrap_or(html.len());
|
||||
let raw_text = &html[text_start..text_end];
|
||||
let title = clean_html_text(raw_text);
|
||||
|
||||
if !title.is_empty() {
|
||||
out.push((title, href.to_string()));
|
||||
}
|
||||
|
||||
// 推进搜索位置到本锚点结束之后,避免重复命中同一锚点的 class
|
||||
search_from = text_end + "</a>".len();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// 提取所有 `class="result__snippet"` 元素的内联文本,保持文档顺序。
|
||||
///
|
||||
/// DDG snippet 用 `<a class="result__snippet" ...>text</a>`(同时也是链接)。
|
||||
/// 扫 `result__snippet` 子串 → 向前找 `<a ` → 向后找 `>` → 取到 `</a>` 间文本。
|
||||
/// 与 extract_result_anchors 同款逻辑,仅 marker 不同。
|
||||
fn extract_snippets(html: &str) -> Vec<String> {
|
||||
const MARKER: &str = "result__snippet";
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let mut search_from = 0usize;
|
||||
while search_from < html.len() {
|
||||
let Some(rel) = html[search_from..].find(MARKER) else { break };
|
||||
let class_pos = search_from + rel;
|
||||
|
||||
let head = &html[..class_pos];
|
||||
let Some(a_open_rel) = head.rfind("<a ") else {
|
||||
search_from = class_pos + MARKER.len();
|
||||
continue;
|
||||
};
|
||||
let a_open_end_offset = match html[a_open_rel..].find('>') {
|
||||
Some(o) => o,
|
||||
None => {
|
||||
search_from = class_pos + MARKER.len();
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let a_open_end = a_open_rel + a_open_end_offset;
|
||||
|
||||
let text_start = a_open_end + 1;
|
||||
let text_end = html[text_start..]
|
||||
.find("</a>")
|
||||
.map(|p| text_start + p)
|
||||
.unwrap_or(html.len());
|
||||
let raw_text = &html[text_start..text_end];
|
||||
let snippet = clean_html_text(raw_text);
|
||||
if !snippet.is_empty() {
|
||||
out.push(snippet);
|
||||
}
|
||||
search_from = text_end + "</a>".len();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// 从 `<a href="..." ...>` 等开标签中提取指定属性值。
|
||||
///
|
||||
/// 支持 `attr="..."` / `attr='...'`(单双引号),不支持无引号属性值(DDG href 总是带引号)。
|
||||
/// 值内的 HTML entity 不解码(简单场景 DDG href 不含 entity;若需可后续加 decode_html_entities)。
|
||||
///
|
||||
/// 已知局限:不做属性单词边界检查,故 `data-href="..."` 会被 `attr="href"` 匹配命中
|
||||
/// (因 "href=" 是 "data-href=" 的子串)。真实 DDG `<a>` 不同时带 data-href 与 href,
|
||||
/// 实务无影响;若未来需精确边界,改用前一个字符非字母数字的边界检查。
|
||||
fn extract_attr(open_tag: &str, attr: &str) -> Option<String> {
|
||||
let lower = open_tag.to_lowercase();
|
||||
let dq_pattern = format!("{}=\"", attr);
|
||||
let sq_pattern = format!("{}='", attr);
|
||||
// 优先双引号形式,其次单引号。matched_quote 记录命中模式对应的引号字符,
|
||||
// 避免 backtrack 推断(原 starts_with('"') 路径在 attr 出现在开标签首时 saturating_sub 边界不稳)。
|
||||
let (value_start_rel, quote_char): (usize, char) = lower
|
||||
.find(&dq_pattern)
|
||||
.map(|p| (p + dq_pattern.len(), '"'))
|
||||
.or_else(|| lower.find(&sq_pattern).map(|p| (p + sq_pattern.len(), '\'')))?;
|
||||
let value_end_rel = open_tag[value_start_rel..].find(quote_char)?;
|
||||
Some(open_tag[value_start_rel..value_start_rel + value_end_rel].to_string())
|
||||
}
|
||||
|
||||
/// 清理 HTML 内联文本:剥离嵌套标签(如 `<b>highlight</b>`)+ 折叠空白。
|
||||
///
|
||||
/// DDG 标题/snippet 内常有 `<b>` 高亮匹配关键词,需剥离得到纯文本。
|
||||
/// 不解码 entity(& 等),简单场景够用;后续如见 entity 可扩展。
|
||||
fn clean_html_text(raw: &str) -> String {
|
||||
// 剥离所有 `<...>` 子串(嵌套标签如 <b>word</b> → word)
|
||||
let mut stripped = String::with_capacity(raw.len());
|
||||
let mut in_tag = false;
|
||||
for c in raw.chars() {
|
||||
match c {
|
||||
'<' => in_tag = true,
|
||||
'>' => in_tag = false,
|
||||
_ if !in_tag => stripped.push(c),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
// 折叠连续空白(含 \n\t)为单空格,trim 首尾
|
||||
let cleaned: String = stripped.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
cleaned
|
||||
}
|
||||
|
||||
/// 解包 DDG 重定向跳板 URL 还原真实目标 URL。
|
||||
///
|
||||
/// DDG 把结果 URL 包成 `https://duckduckgo.com/l/?uddg=<percent-encoded real url>&rut=...`。
|
||||
/// 提取 `uddg` 查询参数值并 percent-decode → 真实 URL(可直接 fetch_url)。
|
||||
///
|
||||
/// 非 DDG 跳板 URL(无 `uddg=` 参数 / 解码失败)→ 原样返回 href 兜底。
|
||||
/// DDG 偶尔返回 `//duckduckgo.com/...` 协议相对内部页(无 uddg),保留原样让 LLM 至少看到。
|
||||
fn unwrap_ddg_redirect(href: &str) -> String {
|
||||
// 在 href 中找 `uddg=` 参数(可能在 ? 或 & 后)
|
||||
let Some(uddg_pos) = href.find("uddg=") else {
|
||||
return href.to_string();
|
||||
};
|
||||
let value_start = uddg_pos + "uddg=".len();
|
||||
let rest = &href[value_start..];
|
||||
// 取到下一个 `&` 或字符串尾
|
||||
let value_end = rest.find('&').unwrap_or(rest.len());
|
||||
let encoded = &rest[..value_end];
|
||||
// percent-decode(query 段,将 + 转空格 DDG 不用,uddg 值内真实 URL 的空格已 %20 编码)
|
||||
use percent_encoding::percent_decode_str;
|
||||
let decoded = percent_decode_str(encoded).decode_utf8_lossy().into_owned();
|
||||
if decoded.is_empty() {
|
||||
href.to_string()
|
||||
} else {
|
||||
decoded
|
||||
}
|
||||
}
|
||||
|
||||
/// percent-encode 搜索关键词(query 段 RFC 3986)。
|
||||
///
|
||||
/// 不引 urlencoding crate(http.rs 测试用同款手写),覆盖 query 段需转义字符:
|
||||
/// 空格 → %20、&=?# 等保留字符转义。字母/数字/常见安全符号(-._~)保留。
|
||||
fn percent_encode_query(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for &b in s.as_bytes() {
|
||||
// RFC 3986 unreserved + 几个 query-safe 字符保留
|
||||
match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9'
|
||||
| b'-' | b'.' | b'_' | b'~' => out.push(b as char),
|
||||
_ => out.push_str(&format!("%{:02X}", b)),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 单元测试
|
||||
//
|
||||
// 覆盖纯函数(extract_results / extract_result_anchors / extract_snippets /
|
||||
// extract_attr / clean_html_text / unwrap_ddg_redirect / percent_encode_query),
|
||||
// 确定性零网络。handler 集成层走真实网络 #[ignore](对齐 http.rs / fetch_url 测试策略)。
|
||||
//
|
||||
// fixture HTML:基于 DDG HTML 端点真实结构简化(保留 result__a / result__snippet /
|
||||
// uddg 重定向跳板等关键标记),让解析逻辑可零网络验证。
|
||||
// ============================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// 真实 DDG HTML 结构简化 fixture(含 2 条结果,标题/snippet/uddg 重定向齐全)。
|
||||
/// 用于 extract_results 端到端解析验证(零网络)。
|
||||
const FIXTURE_DDG_HTML: &str = r#"<html><body>
|
||||
<div class="results">
|
||||
<div class="result results_links results_links_deep web-result">
|
||||
<h2 class="result__title">
|
||||
<a rel="nofollow" class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2Fbook%2F&rut=abc">The Rust Programming Language <b>book</b></a>
|
||||
</h2>
|
||||
<a class="result__snippet" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2Fbook%2F">An introductory book about Rust covering syntax, ownership and <b>async</b>.</a>
|
||||
</div>
|
||||
<div class="result results_links results_links_deep web-result">
|
||||
<h2 class="result__title">
|
||||
<a rel="nofollow" class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Ftokio.rs%2Ftutorial%2F&rut=def">Tokio Tutorial - Asynchronous Rust</a>
|
||||
</h2>
|
||||
<a class="result__snippet" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Ftokio.rs%2Ftutorial%2F">Learn how to write async Rust applications with the Tokio runtime.</a>
|
||||
</div>
|
||||
</div>
|
||||
</body></html>"#;
|
||||
|
||||
// ── extract_results:端到端解析(标题 + uddg 解包 URL + 摘要) ──
|
||||
|
||||
#[test]
|
||||
fn test_extract_results_end_to_end_two_results() {
|
||||
let results = extract_results(FIXTURE_DDG_HTML);
|
||||
assert_eq!(results.len(), 2, "应解析 2 条结果");
|
||||
|
||||
// 第 1 条
|
||||
let r0 = &results[0];
|
||||
assert_eq!(r0["title"].as_str().unwrap(), "The Rust Programming Language book");
|
||||
// uddg 解包后应是真实 URL,非 DDG 跳板
|
||||
assert_eq!(r0["url"].as_str().unwrap(), "https://doc.rust-lang.org/book/");
|
||||
assert!(r0["snippet"].as_str().unwrap().contains("introductory"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_results_second_result() {
|
||||
let results = extract_results(FIXTURE_DDG_HTML);
|
||||
let r1 = &results[1];
|
||||
assert_eq!(r1["title"].as_str().unwrap(), "Tokio Tutorial - Asynchronous Rust");
|
||||
assert_eq!(r1["url"].as_str().unwrap(), "https://tokio.rs/tutorial/");
|
||||
assert!(r1["snippet"].as_str().unwrap().contains("Tokio runtime"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_results_strips_b_tags_in_title() {
|
||||
// DDG 标题内常有 <b> 高亮匹配关键词,应剥离为纯文本
|
||||
let html = r#"<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2F">Hello <b>World</b></a>"#;
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert_eq!(anchors.len(), 1);
|
||||
assert_eq!(anchors[0].0, "Hello World");
|
||||
}
|
||||
|
||||
// ── 截断 ──
|
||||
|
||||
#[test]
|
||||
fn test_execute_search_truncates_results() {
|
||||
// 不发网络:仅验证 extract_results + truncate 的截断逻辑(纯函数路径)
|
||||
// 构造 3 条 fixture,模拟 max_results=2 截断
|
||||
let html = r#"
|
||||
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fa.com%2F">A</a>
|
||||
<a class="result__snippet" href="x">snip A</a>
|
||||
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fb.com%2F">B</a>
|
||||
<a class="result__snippet" href="x">snip B</a>
|
||||
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fc.com%2F">C</a>
|
||||
<a class="result__snippet" href="x">snip C</a>
|
||||
"#;
|
||||
let mut results = extract_results(html);
|
||||
results.truncate(2);
|
||||
assert_eq!(results.len(), 2, "应截断到 2 条");
|
||||
assert_eq!(results[0]["title"].as_str().unwrap(), "A");
|
||||
assert_eq!(results[1]["title"].as_str().unwrap(), "B");
|
||||
}
|
||||
|
||||
// ── extract_attr:开标签属性提取 ──
|
||||
|
||||
#[test]
|
||||
fn test_extract_attr_double_quote() {
|
||||
let tag = r#"<a href="https://example.com/" class="x">"#;
|
||||
assert_eq!(extract_attr(tag, "href").as_deref(), Some("https://example.com/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_attr_single_quote() {
|
||||
let tag = r#"<a href='https://example.com/'>"#;
|
||||
assert_eq!(extract_attr(tag, "href").as_deref(), Some("https://example.com/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_attr_missing_returns_none() {
|
||||
let tag = r#"<a class="x">"#;
|
||||
assert_eq!(extract_attr(tag, "href"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_attr_known_limitation_data_href() {
|
||||
// 已知局限记录:extract_attr 不做单词边界检查,"data-href=" 含 "href=" 子串会被命中。
|
||||
// 真实 DDG `<a>` 不同时带 data-href 与 href,实务无影响。本测试锁定当前行为供回归守护:
|
||||
// 若未来加边界检查改为不命中,此测试会红,提醒同步更新文档注释。
|
||||
let tag = r#"<a data-href="https://hidden.com/">"#;
|
||||
assert_eq!(
|
||||
extract_attr(tag, "href").as_deref(),
|
||||
Some("https://hidden.com/"),
|
||||
"已知局限:data-href 含 href= 子串会被命中(无单词边界检查)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── clean_html_text:剥离嵌套标签 + 折叠空白 ──
|
||||
|
||||
#[test]
|
||||
fn test_clean_html_text_strips_tags() {
|
||||
assert_eq!(clean_html_text("Hello <b>World</b>!"), "Hello World!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_html_text_collapses_whitespace() {
|
||||
assert_eq!(clean_html_text(" multiple\n spaces\t here "), "multiple spaces here");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_html_text_empty_after_strip() {
|
||||
assert_eq!(clean_html_text("<b></b>"), "");
|
||||
assert_eq!(clean_html_text(" "), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_html_text_no_tags_passthrough() {
|
||||
assert_eq!(clean_html_text("plain text"), "plain text");
|
||||
}
|
||||
|
||||
// ── unwrap_ddg_redirect:uddg 参数解包 ──
|
||||
|
||||
#[test]
|
||||
fn test_unwrap_ddg_redirect_decodes_uddg_param() {
|
||||
let href = "https://duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2F&rut=abc";
|
||||
assert_eq!(
|
||||
unwrap_ddg_redirect(href),
|
||||
"https://doc.rust-lang.org/"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unwrap_ddg_redirect_no_uddg_returns_original() {
|
||||
// 非 DDG 跳板(无 uddg 参数)→ 原样返回
|
||||
let href = "https://example.com/direct";
|
||||
assert_eq!(unwrap_ddg_redirect(href), "https://example.com/direct");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unwrap_ddg_redirect_uddg_last_param_no_amp() {
|
||||
// uddg 是最后一个参数(无尾随 &)→ 取到字符串尾
|
||||
let href = "https://duckduckgo.com/l/?uddg=https%3A%2F%2Ftokio.rs";
|
||||
assert_eq!(unwrap_ddg_redirect(href), "https://tokio.rs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unwrap_ddg_redirect_chinese_url_decoded() {
|
||||
// 中文 URL 经 percent-encode,decode 后应还原
|
||||
let href = "https://duckduckgo.com/l/?uddg=https%3A%2F%2Fzh.wikipedia.org%2Fwiki%2FRust&x=1";
|
||||
assert_eq!(
|
||||
unwrap_ddg_redirect(href),
|
||||
"https://zh.wikipedia.org/wiki/Rust"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unwrap_ddg_redirect_empty_encoded_returns_original() {
|
||||
// uddg= 空 → 解码空 → 返回原 href 兜底
|
||||
let href = "https://duckduckgo.com/l/?uddg=&rut=x";
|
||||
assert_eq!(unwrap_ddg_redirect(href), href);
|
||||
}
|
||||
|
||||
// ── percent_encode_query:搜索关键词编码 ──
|
||||
|
||||
#[test]
|
||||
fn test_percent_encode_query_space() {
|
||||
assert_eq!(percent_encode_query("rust async"), "rust%20async");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_percent_encode_query_chinese() {
|
||||
// 中文每字节转 %XX
|
||||
let encoded = percent_encode_query("Rust 语言");
|
||||
// "Rust " 保留 + "语"(3 字节 UTF-8)+ "言"(3 字节)
|
||||
assert!(encoded.starts_with("Rust%20"));
|
||||
assert!(encoded.contains("%E8%AF%AD")); // "语" 的 UTF-8 首三字节之一
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_percent_encode_query_safe_chars_preserved() {
|
||||
// unreserved 字母数字 -._~ 保留
|
||||
assert_eq!(percent_encode_query("abc-123_test.~"), "abc-123_test.~");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_percent_encode_query_special_chars_escaped() {
|
||||
// 保留字符 & = ? # 转义(防注入 query 解析)
|
||||
let encoded = percent_encode_query("a&b=c?d");
|
||||
assert_eq!(encoded, "a%26b%3Dc%3Fd");
|
||||
}
|
||||
|
||||
// ── extract_result_anchors:多结果顺序 + 跳过空标题 ──
|
||||
|
||||
#[test]
|
||||
fn test_extract_result_anchors_preserves_order() {
|
||||
let html = r#"
|
||||
<a class="result__a" href="https://x.com/1">First</a>
|
||||
<a class="result__a" href="https://x.com/2">Second</a>
|
||||
<a class="result__a" href="https://x.com/3">Third</a>
|
||||
"#;
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert_eq!(anchors.len(), 3);
|
||||
assert_eq!(anchors[0].0, "First");
|
||||
assert_eq!(anchors[1].0, "Second");
|
||||
assert_eq!(anchors[2].0, "Third");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_result_anchors_skips_empty_title() {
|
||||
// 空标题(标签内仅空白)跳过,不构造半残条目
|
||||
let html = r#"
|
||||
<a class="result__a" href="https://x.com/1"> </a>
|
||||
<a class="result__a" href="https://x.com/2">Valid</a>
|
||||
"#;
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert_eq!(anchors.len(), 1, "空标题应跳过");
|
||||
assert_eq!(anchors[0].0, "Valid");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_result_anchors_handles_rel_nofollow_first() {
|
||||
// DDG 真实顺序:`<a rel="nofollow" class="result__a" ...>`(rel 在 class 前)
|
||||
let html = r#"<a rel="nofollow" class="result__a" href="https://x.com/">Title</a>"#;
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert_eq!(anchors.len(), 1);
|
||||
assert_eq!(anchors[0].1, "https://x.com/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_result_anchors_no_results_returns_empty() {
|
||||
// 无 result__a 标记 → 空结果(无 panic)
|
||||
let html = "<html><body>no results here</body></html>";
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert!(anchors.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_result_anchors_boundary_check_rejects_result_article() {
|
||||
// 边界检查:`result__article`(以 result__a 为前缀)不应被误命中为 result__a。
|
||||
// 防止 DDG 其他 result__* class(如 result__article)污染标题列表。
|
||||
let html = r#"
|
||||
<a class="result__article" href="https://wrong.com/">Should Not Match</a>
|
||||
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fright.com%2F">Right Title</a>
|
||||
"#;
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert_eq!(anchors.len(), 1, "result__article 不应被误命中");
|
||||
assert_eq!(anchors[0].0, "Right Title");
|
||||
assert_eq!(anchors[0].1, "https://duckduckgo.com/l/?uddg=https%3A%2F%2Fright.com%2F");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_result_anchors_handles_multi_class_value() {
|
||||
// class 列表含多个 class(`class="result__a result__url"`)仍应命中(result__a 后是空格)
|
||||
let html = r#"<a class="result__a extra" href="https://x.com/">Multi Class Title</a>"#;
|
||||
let anchors = extract_result_anchors(html);
|
||||
assert_eq!(anchors.len(), 1);
|
||||
assert_eq!(anchors[0].0, "Multi Class Title");
|
||||
}
|
||||
|
||||
// ── extract_snippets ──
|
||||
|
||||
#[test]
|
||||
fn test_extract_snippets_basic() {
|
||||
let html = r#"
|
||||
<a class="result__snippet" href="x">First snippet</a>
|
||||
<a class="result__snippet" href="y">Second snippet</a>
|
||||
"#;
|
||||
let snippets = extract_snippets(html);
|
||||
assert_eq!(snippets.len(), 2);
|
||||
assert_eq!(snippets[0], "First snippet");
|
||||
assert_eq!(snippets[1], "Second snippet");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_snippets_strips_b_tags() {
|
||||
let html = r#"<a class="result__snippet" href="x">Text with <b>highlight</b> word</a>"#;
|
||||
let snippets = extract_snippets(html);
|
||||
assert_eq!(snippets.len(), 1);
|
||||
assert_eq!(snippets[0], "Text with highlight word");
|
||||
}
|
||||
|
||||
// ── extract_results:无 snippet 时空字符串 ──
|
||||
|
||||
#[test]
|
||||
fn test_extract_results_snippet_missing_defaults_empty() {
|
||||
// 标题有但 snippet 缺(解析时 snippet.get(i) 返 None → 空字符串)
|
||||
let html = r#"
|
||||
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fx.com%2F">Title Only</a>
|
||||
"#;
|
||||
let results = extract_results(html);
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0]["title"].as_str().unwrap(), "Title Only");
|
||||
assert_eq!(results[0]["snippet"].as_str().unwrap(), "", "无 snippet 应默认空字符串");
|
||||
}
|
||||
|
||||
// ── handler 参数边界(不发网络,触发早期拒绝) ──
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_missing_query_errors() {
|
||||
let args = json!({ "max_results": 5 });
|
||||
let err = execute_fetch_search(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("缺少 query"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_empty_query_errors() {
|
||||
let args = json!({ "query": " " });
|
||||
let err = execute_fetch_search(args).await.unwrap_err();
|
||||
assert!(format!("{}", err).contains("query 不能为空"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_max_results_clamped_low() {
|
||||
// max_results=0 应 clamp 到 1(用空 query 触发早期拒绝,验证 clamp 不 panic)
|
||||
let args = json!({ "query": " ", "max_results": 0 });
|
||||
let _ = execute_fetch_search(args).await; // 不 panic 即通过
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_handler_max_results_clamped_high() {
|
||||
let args = json!({ "query": " ", "max_results": 999 });
|
||||
let _ = execute_fetch_search(args).await; // 不 panic 即通过
|
||||
}
|
||||
|
||||
// ── 真实网络集成(#[ignore]:CI 无网时跳过,本地手跑) ──
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "需真实网络(DDG),CI 无网时跳过:cargo test -- --ignored"]
|
||||
async fn integration_search_rust_async_returns_results() {
|
||||
let args = json!({ "query": "rust async programming", "max_results": 3 });
|
||||
let result = execute_fetch_search(args).await.expect("搜索应成功");
|
||||
let count = result["count"].as_u64().unwrap_or(0);
|
||||
assert!(count >= 1, "至少应返回 1 条结果,实际 {}", count);
|
||||
let results = result["results"].as_array().expect("results 应为数组");
|
||||
// 每条应有非空 title + url
|
||||
for r in results {
|
||||
assert!(!r["title"].as_str().unwrap_or("").is_empty(), "title 不应为空");
|
||||
let url = r["url"].as_str().unwrap_or("");
|
||||
assert!(!url.is_empty(), "url 不应为空");
|
||||
// uddg 应已解包,非 DDG 跳板 URL(除非 DDG 返回内部页)
|
||||
// 不强制断言非 duckduckgo.com(偶尔 DDG 返回内部结果),但应能 fetch_url
|
||||
}
|
||||
// query 应回显
|
||||
assert_eq!(result["query"].as_str().unwrap(), "rust async programming");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user