diff --git a/crates/df-project/src/scan.rs b/crates/df-project/src/scan.rs index 22d10b2..0a9417e 100644 --- a/crates/df-project/src/scan.rs +++ b/crates/df-project/src/scan.rs @@ -5,11 +5,19 @@ //! //! 供 commands 层薄封装暴露为 IPC 命令,新建/导入项目时自动填充 `ProjectRecord.stack`; //! 未来 df-ai/df-workflow 亦可复用以感知「项目是什么技术栈」。 +//! +//! README 噪音处理与图片收集已抽离至 `scan_helpers`(2026-06-19 自底向上拆分), +//! 此处通过 `pub use` 重导出 `ImageRef` 等保持外部路径 `df_project::scan::*` 不变。 use std::path::Path; use anyhow::{Context, Result}; +#[path = "scan_helpers.rs"] +mod scan_helpers; +pub use scan_helpers::ImageRef; +use scan_helpers::{collect_images, strip_readme_noise}; + /// 规范化路径用于比较:canonicalize 解析绝对规范路径(失败降级), /// 统一正斜杠 + 小写。防 `C:\a\b` vs `C:/a/b/` 绕过重复检查。 /// 注:仅用于比较,存库保留用户输入的原始可读路径。 @@ -324,16 +332,6 @@ pub fn extract_description(root: &Path) -> Option { /// extract_description 最大字符数 const EXTRACT_DESC_MAX: usize = 200; -/// 内容图引用(README 内的架构图/截图等,喂 vision 用)。 -/// 采样层只收集 alt+src,Phase 2 上线后由 commands 层读 base64 喂 vision。 -/// 当前 ChatMessage.content:String(F-260614-05 未做)走纯文本降级, -/// 此结构仅为采样层留接口,不读 base64。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ImageRef { - pub alt: String, - pub src: String, -} - /// 项目采样结果 — README(剥噪音后) + 目录树(2层) + 清单文件片段 + 内容图引用 #[derive(Debug, Clone)] pub struct ProjectSample { @@ -355,6 +353,19 @@ const SAMPLE_IGNORED_DIRS: &[&str] = &[ ".turbo", ".angular", ".gradle", "vendor", ]; +/// 读 README 原始内容(不做处理) +fn read_readme_raw(root: &Path) -> Option { + for name in &["README.md", "README.MD", "README", "README.zh.md", "README_zh.md", "README_EN.md", "readme.md"] { + let p = root.join(name); + if p.is_file() { + if let Ok(content) = std::fs::read_to_string(&p) { + return Some(content); + } + } + } + None +} + /// 采集项目采样(README + 目录树 + 清单 + 内容图),供 LLM 分析填基础信息 pub fn collect_sample(root: &Path) -> Result { if !root.is_dir() { @@ -381,238 +392,6 @@ pub fn collect_sample(root: &Path) -> Result { }) } -/// 读 README 原始内容(不做处理) -fn read_readme_raw(root: &Path) -> Option { - for name in &["README.md", "README.MD", "README", "README.zh.md", "README_zh.md", "README_EN.md", "readme.md"] { - let p = root.join(name); - if p.is_file() { - if let Ok(content) = std::fs::read_to_string(&p) { - return Some(content); - } - } - } - None -} - -/// 徽章图域名(纯徽章图,跳过不喂 LLM) -const BADGE_HOSTS: &[&str] = &[ - "img.shields.io", - "shields.io", - "badge.fury.io", - "badgen.net", - "badges.frapsoft.com", - "github.com/workflows", // GitHub Actions 工作流状态徽章 - "travis-ci.org", - "coveralls.io", - "codecov.io", - "app.codacy.com", - "david-dm.org", - "circleci.com", - "ci.appveyor.com", -]; - -/// 徽章关键词(图 alt/src 含这些视为纯徽章图) -const BADGE_KEYWORDS: &[&str] = &[ - "build", - "version", - "license", - "coverage", - "downloads", - "download", - "npm", - "pypi", - "crates.io", - "codeclimate", - "maintainability", - "stars", - "forks", - "issues", - "contributors", - "dependencies", - "devdependencies", - "circleci", - "travis", - "appveyor", - "coveralls", - "codecov", -]; - -/// 判定 markdown 图片 `![alt](src)` 是否为纯徽章(架构图/截图等保留) -fn is_badge_image(alt: &str, src: &str) -> bool { - // 域名命中(shields.io / badge.fury / GitHub Actions 等) - let src_lower = src.to_lowercase(); - if BADGE_HOSTS.iter().any(|h| src_lower.contains(h)) { - return true; - } - // 关键词命中(alt 或 src 含 build/version/license/coverage 等) - let hay = format!("{alt} {src}").to_lowercase(); - if BADGE_KEYWORDS.iter().any(|k| hay.contains(k)) { - return true; - } - false -} - -/// 从 README 收集内容图(跳徽章),保留 markdown 原样引用 -fn collect_images(content: &str) -> Vec { - let mut out = Vec::new(); - let mut seen = std::collections::HashSet::new(); - // 匹配 ![alt](src) —— 简单行级扫描,够用且无 regex 依赖 - for line in content.lines() { - let mut rest = line; - while let Some(start) = rest.find("![") { - let after_bracket = &rest[start + 2..]; - let Some(alt_end) = after_bracket.find("](") else { break }; - let alt = after_bracket[..alt_end].to_string(); - let after_paren = &after_bracket[alt_end + 2..]; - let Some(paren_end) = after_paren.find(')') else { break }; - let src = after_paren[..paren_end].to_string(); - rest = &after_paren[paren_end + 1..]; - if is_badge_image(&alt, &src) { - continue; - } - // 同 src 去重 - if seen.insert(src.clone()) { - out.push(ImageRef { alt, src }); - } - if out.len() >= 20 { - return out; - } - } - } - out -} - -/// 剥 README 噪音:frontmatter(YAML/TOML 块) / HTML 注释 / TOC(纯链接目录行) / -/// 纯徽章图片行 / 纯徽章 HTML 行。保留标题、正文、内容图 markdown 原样。 -fn strip_readme_noise(content: &str) -> String { - let mut out = Vec::new(); - let mut lines = content.lines().peekable(); - while let Some(line) = lines.next() { - let trimmed = line.trim(); - // ── frontmatter 块(YAML `---` / TOML `+++`)── - if trimmed == "---" || trimmed == "+++" { - // 开头处的 frontmatter:跳到下一个匹配分隔符 - let delim = trimmed; - let mut consumed_any = false; - for next in lines.by_ref() { - consumed_any = true; - if next.trim() == delim { - break; - } - } - let _ = consumed_any; - continue; - } - // ── HTML 注释块(,可能跨行)── - if trimmed.starts_with("") { - // 单行注释,整行跳过 - continue; - } - // 多行:吃到含 --> 的行 - for next in lines.by_ref() { - if next.contains("-->") { - break; - } - } - continue; - } - // ── 纯徽章行:整行只剩图片/HTML 徽章(可能多个 ![..](..) 连排)── - if is_pure_badge_line(trimmed) { - continue; - } - // ── TOC:纯目录行(整行只有 [..](#anchor) 锚点链接,或 markdown 列表项仅锚点)── - if is_toc_line(trimmed) { - continue; - } - out.push(line.to_string()); - } - out.join("\n") -} - -/// 判定整行是否为纯徽章行(仅含图片/HTML badge,无其它实质文本)。 -/// 如 `![build](https://img.shields.io/x) ![license](...) ` -fn is_pure_badge_line(line: &str) -> bool { - if line.is_empty() { - return false; - } - // 仅含图片语法/HTML 标签/空白/[alt] 片段,且至少有一个图片或 HTML img 标签 - let mut content_found = false; - let mut text_only: String = String::new(); - let mut rest = line; - loop { - // 找下一个 ![ 或 &suffix[2..suffix.find("](").unwrap_or(end)]; 闭合 - if let Some(end) = suffix.find('>') { - content_found = true; - end + 1 - } else { - return false; - } - }; - rest = &suffix[consumed..]; - } - // 剩余 text_only 必须为空或纯空白/标点 - let residual = text_only - .chars() - .filter(|c| !c.is_whitespace() && *c != '|' && *c != '-') - .collect::(); - content_found && residual.is_empty() -} - -/// 判定 TOC 行:markdown 列表项仅含锚点链接 `- [..](#..)` 或整行仅锚点链接 -fn is_toc_line(line: &str) -> bool { - if line.is_empty() { - return false; - } - let t = line.trim_start_matches(['-', '*', '+', ' ']); - // 形如 [text](#anchor) 或 [text](#anchor "title") - if !(t.starts_with('[') && t.contains("](")) { - return false; - } - // 取 ]( 后到行尾或空格,须以 # 开头 - let Some(close) = t.find("](") else { return false }; - let after = &t[close + 2..]; - let target = after.trim_end_matches(')').split_whitespace().next().unwrap_or(""); - target.starts_with('#') -} - /// 目录树(根 + 一层子目录),过滤噪音目录,控条目数 fn collect_tree(root: &Path) -> Vec { let mut lines = Vec::new(); @@ -786,18 +565,6 @@ mod tests { fs::remove_dir_all(&d).ok(); } - #[test] - fn image_collection_skips_badges() { - let content = "![build](https://img.shields.io/x)\n![arch](./a.png)\n![version](https://badge.fury.io/js/y)\n![screenshot](screens/b.png)\n"; - let imgs = collect_images(content); - // 只有 arch + screenshot,build/version 跳 - assert_eq!(imgs.len(), 2); - assert!(imgs.iter().any(|i| i.src == "./a.png")); - assert!(imgs.iter().any(|i| i.src == "screens/b.png")); - assert!(imgs.iter().all(|i| !i.src.contains("shields.io"))); - assert!(imgs.iter().all(|i| !i.src.contains("badge.fury"))); - } - #[test] fn detects_monorepo_pnpm() { let d = scratch("mono-pnpm"); @@ -971,45 +738,4 @@ mod tests { fn is_monorepo_non_dir_returns_false() { assert!(!is_monorepo(Path::new("definitely-not-exist-xyz-456"))); } - - // ── collect_images 徽章过滤逻辑补强 ── - #[test] - fn collect_images_filters_badge_by_keyword() { - // 非徽章域名,但 alt/src 含关键词(build/license/coverage)应过滤 - let content = "![build status](https://example.com/build.svg)\n![my license](https://example.com/lic.svg)\n![coverage](https://example.com/cov.svg)\n![architecture](./arch.png)\n"; - let imgs = collect_images(content); - assert!(imgs.iter().all(|i| !i.alt.contains("build")), "build 关键词徽章未过滤"); - assert!(imgs.iter().all(|i| !i.alt.contains("license")), "license 关键词徽章未过滤"); - assert!(imgs.iter().all(|i| !i.alt.contains("coverage")), "coverage 关键词徽章未过滤"); - assert_eq!(imgs.len(), 1, "应仅保留架构图: {imgs:?}"); - assert_eq!(imgs[0].src, "./arch.png"); - } - - #[test] - fn collect_images_dedups_same_src() { - // 同 src 不同 alt 去重,只留首个 - let content = "![first](./dup.png)\n![second](./dup.png)\n"; - let imgs = collect_images(content); - assert_eq!(imgs.len(), 1, "同 src 应去重: {imgs:?}"); - assert_eq!(imgs[0].alt, "first"); - } - - #[test] - fn collect_images_handles_multiple_in_one_line() { - // 同行多图 + 徽章混排:徽章过滤、内容图保留 - let content = "![build](https://img.shields.io/x) ![shot](./shot.png) ![alt2](./two.png)\n"; - let imgs = collect_images(content); - let srcs: Vec<_> = imgs.iter().map(|i| i.src.as_str()).collect(); - assert!(srcs.contains(&"./shot.png"), "同行内容图丢失: {srcs:?}"); - assert!(srcs.contains(&"./two.png"), "同行内容图丢失: {srcs:?}"); - assert!(!srcs.contains(&"https://img.shields.io/x"), "同行徽章未过滤: {srcs:?}"); - } - - #[test] - fn collect_images_empty_and_malformed() { - // 无图片 / 不闭合语法不 panic,返回空 - assert!(collect_images("plain text no images").is_empty()); - assert!(collect_images("![unclosed alt").is_empty()); - assert!(collect_images("![alt](no-close-paren").is_empty()); - } } diff --git a/crates/df-project/src/scan_helpers.rs b/crates/df-project/src/scan_helpers.rs new file mode 100644 index 0000000..d8e6814 --- /dev/null +++ b/crates/df-project/src/scan_helpers.rs @@ -0,0 +1,324 @@ +//! README 噪音处理与图片收集 — 纯函数,无 IO 依赖。 +//! +//! 从 `scan.rs` 抽离(2026-06-19 自底向上拆分):徽章判定、内容图收集、 +//! frontmatter/TOC/纯徽章行剥离等纯文本规则。无文件系统访问,便于单测与复用。 +//! +//! 外部路径不变:`scan` 模块通过 `pub use` 重导出 `ImageRef` 等。 + +/// 内容图引用(README 内的架构图/截图等,喂 vision 用)。 +/// 采样层只收集 alt+src,Phase 2 上线后由 commands 层读 base64 喂 vision。 +/// 当前 ChatMessage.content:String(F-260614-05 未做)走纯文本降级, +/// 此结构仅为采样层留接口,不读 base64。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageRef { + pub alt: String, + pub src: String, +} + +/// 徽章图域名(纯徽章图,跳过不喂 LLM) +const BADGE_HOSTS: &[&str] = &[ + "img.shields.io", + "shields.io", + "badge.fury.io", + "badgen.net", + "badges.frapsoft.com", + "github.com/workflows", // GitHub Actions 工作流状态徽章 + "travis-ci.org", + "coveralls.io", + "codecov.io", + "app.codacy.com", + "david-dm.org", + "circleci.com", + "ci.appveyor.com", +]; + +/// 徽章关键词(图 alt/src 含这些视为纯徽章图) +const BADGE_KEYWORDS: &[&str] = &[ + "build", + "version", + "license", + "coverage", + "downloads", + "download", + "npm", + "pypi", + "crates.io", + "codeclimate", + "maintainability", + "stars", + "forks", + "issues", + "contributors", + "dependencies", + "devdependencies", + "circleci", + "travis", + "appveyor", + "coveralls", + "codecov", +]; + +/// 判定 markdown 图片 `![alt](src)` 是否为纯徽章(架构图/截图等保留) +fn is_badge_image(alt: &str, src: &str) -> bool { + // 域名命中(shields.io / badge.fury / GitHub Actions 等) + let src_lower = src.to_lowercase(); + if BADGE_HOSTS.iter().any(|h| src_lower.contains(h)) { + return true; + } + // 关键词命中(alt 或 src 含 build/version/license/coverage 等) + let hay = format!("{alt} {src}").to_lowercase(); + if BADGE_KEYWORDS.iter().any(|k| hay.contains(k)) { + return true; + } + false +} + +/// 从 README 收集内容图(跳徽章),保留 markdown 原样引用 +pub(crate) fn collect_images(content: &str) -> Vec { + let mut out = Vec::new(); + let mut seen = std::collections::HashSet::new(); + // 匹配 ![alt](src) —— 简单行级扫描,够用且无 regex 依赖 + for line in content.lines() { + let mut rest = line; + while let Some(start) = rest.find("![") { + let after_bracket = &rest[start + 2..]; + let Some(alt_end) = after_bracket.find("](") else { break }; + let alt = after_bracket[..alt_end].to_string(); + let after_paren = &after_bracket[alt_end + 2..]; + let Some(paren_end) = after_paren.find(')') else { break }; + let src = after_paren[..paren_end].to_string(); + rest = &after_paren[paren_end + 1..]; + if is_badge_image(&alt, &src) { + continue; + } + // 同 src 去重 + if seen.insert(src.clone()) { + out.push(ImageRef { alt, src }); + } + if out.len() >= 20 { + return out; + } + } + } + out +} + +/// 剥 README 噪音:frontmatter(YAML/TOML 块) / HTML 注释 / TOC(纯链接目录行) / +/// 纯徽章图片行 / 纯徽章 HTML 行。保留标题、正文、内容图 markdown 原样。 +pub(crate) fn strip_readme_noise(content: &str) -> String { + let mut out = Vec::new(); + let mut lines = content.lines().peekable(); + while let Some(line) = lines.next() { + let trimmed = line.trim(); + // ── frontmatter 块(YAML `---` / TOML `+++`)── + if trimmed == "---" || trimmed == "+++" { + // 开头处的 frontmatter:跳到下一个匹配分隔符 + let delim = trimmed; + let mut consumed_any = false; + for next in lines.by_ref() { + consumed_any = true; + if next.trim() == delim { + break; + } + } + let _ = consumed_any; + continue; + } + // ── HTML 注释块(,可能跨行)── + if trimmed.starts_with("") { + // 单行注释,整行跳过 + continue; + } + // 多行:吃到含 --> 的行 + for next in lines.by_ref() { + if next.contains("-->") { + break; + } + } + continue; + } + // ── 纯徽章行:整行只剩图片/HTML 徽章(可能多个 ![..](..) 连排)── + if is_pure_badge_line(trimmed) { + continue; + } + // ── TOC:纯目录行(整行只有 [..](#anchor) 锚点链接,或 markdown 列表项仅锚点)── + if is_toc_line(trimmed) { + continue; + } + out.push(line.to_string()); + } + out.join("\n") +} + +/// 判定整行是否为纯徽章行(仅含图片/HTML badge,无其它实质文本)。 +/// 如 `![build](https://img.shields.io/x) ![license](...) ` +pub(crate) fn is_pure_badge_line(line: &str) -> bool { + if line.is_empty() { + return false; + } + // 仅含图片语法/HTML 标签/空白/[alt] 片段,且至少有一个图片或 HTML img 标签 + let mut content_found = false; + let mut text_only: String = String::new(); + let mut rest = line; + loop { + // 找下一个 ![ 或 &suffix[2..suffix.find("](").unwrap_or(end)]; 闭合 + if let Some(end) = suffix.find('>') { + content_found = true; + end + 1 + } else { + return false; + } + }; + rest = &suffix[consumed..]; + } + // 剩余 text_only 必须为空或纯空白/标点 + let residual = text_only + .chars() + .filter(|c| !c.is_whitespace() && *c != '|' && *c != '-') + .collect::(); + content_found && residual.is_empty() +} + +/// 判定 TOC 行:markdown 列表项仅含锚点链接 `- [..](#..)` 或整行仅锚点链接 +fn is_toc_line(line: &str) -> bool { + if line.is_empty() { + return false; + } + let t = line.trim_start_matches(['-', '*', '+', ' ']); + // 形如 [text](#anchor) 或 [text](#anchor "title") + if !(t.starts_with('[') && t.contains("](")) { + return false; + } + // 取 ]( 后到行尾或空格,须以 # 开头 + let Some(close) = t.find("](") else { return false }; + let after = &t[close + 2..]; + let target = after.trim_end_matches(')').split_whitespace().next().unwrap_or(""); + target.starts_with('#') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn image_collection_skips_badges() { + let content = "![build](https://img.shields.io/x)\n![arch](./a.png)\n![version](https://badge.fury.io/js/y)\n![screenshot](screens/b.png)\n"; + let imgs = collect_images(content); + // 只有 arch + screenshot,build/version 跳 + assert_eq!(imgs.len(), 2); + assert!(imgs.iter().any(|i| i.src == "./a.png")); + assert!(imgs.iter().any(|i| i.src == "screens/b.png")); + assert!(imgs.iter().all(|i| !i.src.contains("shields.io"))); + assert!(imgs.iter().all(|i| !i.src.contains("badge.fury"))); + } + + #[test] + fn collect_images_filters_badge_by_keyword() { + // 非徽章域名,但 alt/src 含关键词(build/license/coverage)应过滤 + let content = "![build status](https://example.com/build.svg)\n![my license](https://example.com/lic.svg)\n![coverage](https://example.com/cov.svg)\n![architecture](./arch.png)\n"; + let imgs = collect_images(content); + assert!(imgs.iter().all(|i| !i.alt.contains("build")), "build 关键词徽章未过滤"); + assert!(imgs.iter().all(|i| !i.alt.contains("license")), "license 关键词徽章未过滤"); + assert!(imgs.iter().all(|i| !i.alt.contains("coverage")), "coverage 关键词徽章未过滤"); + assert_eq!(imgs.len(), 1, "应仅保留架构图: {imgs:?}"); + assert_eq!(imgs[0].src, "./arch.png"); + } + + #[test] + fn collect_images_dedups_same_src() { + // 同 src 不同 alt 去重,只留首个 + let content = "![first](./dup.png)\n![second](./dup.png)\n"; + let imgs = collect_images(content); + assert_eq!(imgs.len(), 1, "同 src 应去重: {imgs:?}"); + assert_eq!(imgs[0].alt, "first"); + } + + #[test] + fn collect_images_handles_multiple_in_one_line() { + // 同行多图 + 徽章混排:徽章过滤、内容图保留 + let content = "![build](https://img.shields.io/x) ![shot](./shot.png) ![alt2](./two.png)\n"; + let imgs = collect_images(content); + let srcs: Vec<_> = imgs.iter().map(|i| i.src.as_str()).collect(); + assert!(srcs.contains(&"./shot.png"), "同行内容图丢失: {srcs:?}"); + assert!(srcs.contains(&"./two.png"), "同行内容图丢失: {srcs:?}"); + assert!(!srcs.contains(&"https://img.shields.io/x"), "同行徽章未过滤: {srcs:?}"); + } + + #[test] + fn collect_images_empty_and_malformed() { + // 无图片 / 不闭合语法不 panic,返回空 + assert!(collect_images("plain text no images").is_empty()); + assert!(collect_images("![unclosed alt").is_empty()); + assert!(collect_images("![alt](no-close-paren").is_empty()); + } + + #[test] + fn is_pure_badge_line_true_for_shields_row() { + assert!(is_pure_badge_line( + "![build](https://img.shields.io/badge/x-y-green) ![license](https://img.shields.io/badge/l-mit-blue)" + )); + } + + #[test] + fn is_pure_badge_line_false_when_text_present() { + // 行内含实质文本 → 非纯徽章行 + assert!(!is_pure_badge_line("![build](https://img.shields.io/x) and some real text")); + } + + #[test] + fn is_pure_badge_line_false_for_content_image_only() { + // 纯内容图行(非徽章)不剥,保留 → 返回 false + assert!(!is_pure_badge_line("![arch](./docs/arch.png)")); + } + + #[test] + fn strip_readme_noise_removes_frontmatter_htmlcomment_toc_badge() { + let readme = "---\ntitle: Foo\n---\n\n# Foo\n\n\n\n[Install](#install)\n\n- [Usage](#usage)\n\n![build](https://img.shields.io/badge/build-passing)\n\nThis is the real intro.\n"; + let cleaned = strip_readme_noise(readme); + assert!(cleaned.contains("# Foo"), "标题应保留: {cleaned}"); + assert!(cleaned.contains("real intro"), "正文应保留: {cleaned}"); + assert!(!cleaned.contains("hidden comment"), "HTML 注释未剥: {cleaned}"); + assert!(!cleaned.contains("shields.io"), "徽章未剥: {cleaned}"); + assert!(!cleaned.contains("[Install](#install)"), "TOC 锚点未剥: {cleaned}"); + assert!(!cleaned.contains("[Usage](#usage)"), "TOC 列表项未剥: {cleaned}"); + assert!(!cleaned.contains("title: Foo"), "frontmatter 未剥: {cleaned}"); + } +}