DevFlow/crates/df-project/src/scan_helpers.rs

//! README 噪音处理与图片收集 — 纯函数,无 IO 依赖。
//!
//! 从 `scan.rs` 抽离(2026-06-19 自底向上拆分):徽章判定、内容图收集、
//! frontmatter/TOC/纯徽章行剥离等纯文本规则。无文件系统访问,便于单测与复用。
//!
//! 外部路径不变:`scan` 模块通过 `pub use` 重导出 `ImageRef` 等。

/// 内容图引用(README 内的架构图/截图等,喂 vision 用)。
/// 采样层只收集 alt+src,Phase 2 上线后由 commands 层读 base64 喂 vision。
/// 当前 ChatMessage.content:String(多模态未做)走纯文本降级,
/// 此结构仅为采样层留接口,不读 base64。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ImageRef {
    pub alt: String,
    pub src: String,
}

/// 徽章图域名(纯徽章图,跳过不喂 LLM)
const BADGE_HOSTS: &[&str] = &[
    "img.shields.io",
    "shields.io",
    "badge.fury.io",
    "badgen.net",
    "badges.frapsoft.com",
    "github.com/workflows", // GitHub Actions 工作流状态徽章
    "travis-ci.org",
    "coveralls.io",
    "codecov.io",
    "app.codacy.com",
    "david-dm.org",
    "circleci.com",
    "ci.appveyor.com",
];

/// 徽章关键词(图 alt/src 含这些视为纯徽章图)
const BADGE_KEYWORDS: &[&str] = &[
    "build",
    "version",
    "license",
    "coverage",
    "downloads",
    "download",
    "npm",
    "pypi",
    "crates.io",
    "codeclimate",
    "maintainability",
    "stars",
    "forks",
    "issues",
    "contributors",
    "dependencies",
    "devdependencies",
    "circleci",
    "travis",
    "appveyor",
    "coveralls",
    "codecov",
];

/// 判定 markdown 图片 `![alt](src)` 是否为纯徽章(架构图/截图等保留)
fn is_badge_image(alt: &str, src: &str) -> bool {
    // 域名命中(shields.io / badge.fury / GitHub Actions 等)
    let src_lower = src.to_lowercase();
    if BADGE_HOSTS.iter().any(|h| src_lower.contains(h)) {
        return true;
    }
    // 关键词命中(alt 或 src 含 build/version/license/coverage 等)
    let hay = format!("{alt} {src}").to_lowercase();
    if BADGE_KEYWORDS.iter().any(|k| hay.contains(k)) {
        return true;
    }
    false
}

/// 从 README 收集内容图(跳徽章),保留 markdown 原样引用
pub(crate) fn collect_images(content: &str) -> Vec<ImageRef> {
    let mut out = Vec::new();
    let mut seen = std::collections::HashSet::new();
    // 匹配 ![alt](src) —— 简单行级扫描,够用且无 regex 依赖
    for line in content.lines() {
        let mut rest = line;
        while let Some(start) = rest.find("![") {
            let after_bracket = &rest[start + 2..];
            let Some(alt_end) = after_bracket.find("](") else { break };
            let alt = after_bracket[..alt_end].to_string();
            let after_paren = &after_bracket[alt_end + 2..];
            let Some(paren_end) = after_paren.find(')') else { break };
            let src = after_paren[..paren_end].to_string();
            rest = &after_paren[paren_end + 1..];
            if is_badge_image(&alt, &src) {
                continue;
            }
            // 同 src 去重
            if seen.insert(src.clone()) {
                out.push(ImageRef { alt, src });
            }
            if out.len() >= 20 {
                return out;
            }
        }
    }
    out
}

/// 剥 README 噪音:frontmatter(YAML/TOML 块) / HTML 注释 / TOC(纯链接目录行) /
/// 纯徽章图片行 / 纯徽章 HTML 行。保留标题、正文、内容图 markdown 原样。
pub(crate) fn strip_readme_noise(content: &str) -> String {
    let mut out = Vec::new();
    let mut lines = content.lines().peekable();
    while let Some(line) = lines.next() {
        let trimmed = line.trim();
        // ── frontmatter 块(YAML `---` / TOML `+++`)──
        if trimmed == "---" || trimmed == "+++" {
            // 开头处的 frontmatter:跳到下一个匹配分隔符
            let delim = trimmed;
            for next in lines.by_ref() {
                if next.trim() == delim {
                    break;
                }
            }
            continue;
        }
        // ── HTML 注释块(<!-- ... -->,可能跨行)──
        if trimmed.starts_with("<!--") {
            if trimmed.contains("-->") {
                // 单行注释,整行跳过
                continue;
            }
            // 多行:吃到含 --> 的行
            for next in lines.by_ref() {
                if next.contains("-->") {
                    break;
                }
            }
            continue;
        }
        // ── 纯徽章行:整行只剩图片/HTML 徽章(可能多个 ![..](..) 连排)──
        if is_pure_badge_line(trimmed) {
            continue;
        }
        // ── TOC:纯目录行(整行只有 [..](#anchor) 锚点链接,或 markdown 列表项仅锚点)──
        if is_toc_line(trimmed) {
            continue;
        }
        out.push(line.to_string());
    }
    out.join("\n")
}

/// 判定整行是否为纯徽章行(仅含图片/HTML badge,无其它实质文本)。
/// 如 `![build](https://img.shields.io/x) ![license](...) <a href="..."><img.../></a>`
pub(crate) fn is_pure_badge_line(line: &str) -> bool {
    if line.is_empty() {
        return false;
    }
    // 仅含图片语法/HTML 标签/空白/[alt] 片段,且至少有一个图片或 HTML img 标签
    let mut content_found = false;
    let mut text_only: String = String::new();
    let mut rest = line;
    loop {
        // 找下一个 ![ 或 <img 或 <a
        // 注:HTML 标签为纯 ASCII,用 to_ascii_lowercase 做「不区分大小写」匹配即可;
        // 若用 to_lowercase(),非 ASCII 字符(如 İ→i̇、ß→ss)在小写化时改变字节长度,
        // 返回的索引会偏离原 rest 的字节边界,后续 &rest[..pos]/&rest[pos..] 切到错位/panic。
        // to_ascii_lowercase 仅改 ASCII 字母、保持非 ASCII 字节不变 → 字节长度不变 → 索引对齐。
        let img_md = rest.find("![");
        let img_html = rest.to_ascii_lowercase().find("<img");
        let anchor_html = rest.to_ascii_lowercase().find("<a ");
        let earliest = [img_md, img_html, anchor_html]
            .into_iter()
            .flatten()
            .min();
        let Some(pos) = earliest else {
            // 剩余纯文本
            text_only.push_str(rest);
            break;
        };
        // pos 之前的文本进 text_only
        text_only.push_str(&rest[..pos]);
        let suffix = &rest[pos..];
        let consumed = if suffix.starts_with("![") {
            // markdown 图片:吃到 )
            if let Some(end) = suffix.find(')') {
                let alt = &suffix[2..suffix.find("](").unwrap_or(end)];
                let src = &suffix[suffix.find("](").map(|i| i + 2).unwrap_or(end)..end];
                if !is_badge_image(alt, src) {
                    // 内容图混在行里 → 非纯徽章行
                    return false;
                }
                content_found = true;
                end + 1
            } else {
                // 不闭合,当普通文本
                return false;
            }
        } else {
            // HTML <img 或 <a 标签:视为徽章载体(HTML 行常见于此)
            // 找 > 闭合
            if let Some(end) = suffix.find('>') {
                content_found = true;
                end + 1
            } else {
                return false;
            }
        };
        rest = &suffix[consumed..];
    }
    // 剩余 text_only 必须为空或纯空白/标点
    let residual = text_only
        .chars()
        .filter(|c| !c.is_whitespace() && *c != '|' && *c != '-')
        .collect::<String>();
    content_found && residual.is_empty()
}

/// 判定 TOC 行:markdown 列表项仅含锚点链接 `- [..](#..)` 或整行仅锚点链接
fn is_toc_line(line: &str) -> bool {
    if line.is_empty() {
        return false;
    }
    let t = line.trim_start_matches(['-', '*', '+', ' ']);
    // 形如 [text](#anchor) 或 [text](#anchor "title")
    if !(t.starts_with('[') && t.contains("](")) {
        return false;
    }
    // 取 ]( 后到行尾或空格,须以 # 开头
    let Some(close) = t.find("](") else { return false };
    let after = &t[close + 2..];
    let target = after.trim_end_matches(')').split_whitespace().next().unwrap_or("");
    target.starts_with('#')
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn image_collection_skips_badges() {
        let content = "![build](https://img.shields.io/x)\n![arch](./a.png)\n![version](https://badge.fury.io/js/y)\n![screenshot](screens/b.png)\n";
        let imgs = collect_images(content);
        // 只有 arch + screenshot,build/version 跳
        assert_eq!(imgs.len(), 2);
        assert!(imgs.iter().any(|i| i.src == "./a.png"));
        assert!(imgs.iter().any(|i| i.src == "screens/b.png"));
        assert!(imgs.iter().all(|i| !i.src.contains("shields.io")));
        assert!(imgs.iter().all(|i| !i.src.contains("badge.fury")));
    }

    #[test]
    fn collect_images_filters_badge_by_keyword() {
        // 非徽章域名,但 alt/src 含关键词(build/license/coverage)应过滤
        let content = "![build status](https://example.com/build.svg)\n![my license](https://example.com/lic.svg)\n![coverage](https://example.com/cov.svg)\n![architecture](./arch.png)\n";
        let imgs = collect_images(content);
        assert!(imgs.iter().all(|i| !i.alt.contains("build")), "build 关键词徽章未过滤");
        assert!(imgs.iter().all(|i| !i.alt.contains("license")), "license 关键词徽章未过滤");
        assert!(imgs.iter().all(|i| !i.alt.contains("coverage")), "coverage 关键词徽章未过滤");
        assert_eq!(imgs.len(), 1, "应仅保留架构图: {imgs:?}");
        assert_eq!(imgs[0].src, "./arch.png");
    }

    #[test]
    fn collect_images_dedups_same_src() {
        // 同 src 不同 alt 去重,只留首个
        let content = "![first](./dup.png)\n![second](./dup.png)\n";
        let imgs = collect_images(content);
        assert_eq!(imgs.len(), 1, "同 src 应去重: {imgs:?}");
        assert_eq!(imgs[0].alt, "first");
    }

    #[test]
    fn collect_images_handles_multiple_in_one_line() {
        // 同行多图 + 徽章混排:徽章过滤、内容图保留
        let content = "![build](https://img.shields.io/x) ![shot](./shot.png) ![alt2](./two.png)\n";
        let imgs = collect_images(content);
        let srcs: Vec<_> = imgs.iter().map(|i| i.src.as_str()).collect();
        assert!(srcs.contains(&"./shot.png"), "同行内容图丢失: {srcs:?}");
        assert!(srcs.contains(&"./two.png"), "同行内容图丢失: {srcs:?}");
        assert!(!srcs.contains(&"https://img.shields.io/x"), "同行徽章未过滤: {srcs:?}");
    }

    #[test]
    fn collect_images_empty_and_malformed() {
        // 无图片 / 不闭合语法不 panic,返回空
        assert!(collect_images("plain text no images").is_empty());
        assert!(collect_images("![unclosed alt").is_empty());
        assert!(collect_images("![alt](no-close-paren").is_empty());
    }

    #[test]
    fn is_pure_badge_line_true_for_shields_row() {
        assert!(is_pure_badge_line(
            "![build](https://img.shields.io/badge/x-y-green) ![license](https://img.shields.io/badge/l-mit-blue)"
        ));
    }

    #[test]
    fn is_pure_badge_line_false_when_text_present() {
        // 行内含实质文本 → 非纯徽章行
        assert!(!is_pure_badge_line("![build](https://img.shields.io/x) and some real text"));
    }

    #[test]
    fn is_pure_badge_line_false_for_content_image_only() {
        // 纯内容图行(非徽章)不剥,保留 → 返回 false
        assert!(!is_pure_badge_line("![arch](./docs/arch.png)"));
    }

    #[test]
    fn strip_readme_noise_removes_frontmatter_htmlcomment_toc_badge() {
        let readme = "---\ntitle: Foo\n---\n\n# Foo\n\n<!-- hidden comment -->\n\n[Install](#install)\n\n- [Usage](#usage)\n\n![build](https://img.shields.io/badge/build-passing)\n\nThis is the real intro.\n";
        let cleaned = strip_readme_noise(readme);
        assert!(cleaned.contains("# Foo"), "标题应保留: {cleaned}");
        assert!(cleaned.contains("real intro"), "正文应保留: {cleaned}");
        assert!(!cleaned.contains("hidden comment"), "HTML 注释未剥: {cleaned}");
        assert!(!cleaned.contains("shields.io"), "徽章未剥: {cleaned}");
        assert!(!cleaned.contains("[Install](#install)"), "TOC 锚点未剥: {cleaned}");
        assert!(!cleaned.contains("[Usage](#usage)"), "TOC 列表项未剥: {cleaned}");
        assert!(!cleaned.contains("title: Foo"), "frontmatter 未剥: {cleaned}");
    }
}