diff --git a/Cargo.lock b/Cargo.lock index 2aa7c5f..cfe5bba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -859,6 +859,10 @@ dependencies = [ "tauri-plugin-window-state", "tokio", "tracing", + "tree-sitter", + "tree-sitter-javascript", + "tree-sitter-rust", + "tree-sitter-typescript", "uuid", ] @@ -3877,6 +3881,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -4140,6 +4145,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "string_cache" version = "0.9.0" @@ -5097,6 +5108,56 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tree-sitter" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/docs/02-架构设计/专项设计/AST符号解析-设计-2026-06-24.md b/docs/02-架构设计/专项设计/AST符号解析-设计-2026-06-24.md index a494114..6aae3c2 100644 --- a/docs/02-架构设计/专项设计/AST符号解析-设计-2026-06-24.md +++ b/docs/02-架构设计/专项设计/AST符号解析-设计-2026-06-24.md @@ -83,10 +83,30 @@ AST 是语法层(parse tree)。**语法层能**: ### 选型:tree-sitter - GitHub 出品,增量 AST 解析,几十种语言 grammar -- Rust 生态 `tree-sitter` + `tree-sitter-rust`/`tree-sitter-typescript` 等(编译期 crate) -- **按语言按需 enable grammar**(非全装,控体积) +- Rust 生态 `tree-sitter` + 各语言 grammar crate(编译期静态链接) - 增量解析:文件改只重解变更(<1ms) +### grammar 策略(2026-06-24 定稿) + +**静态编译 + 集中 lookup**(架构成本最低,详见 [插件机制设计](插件机制-设计-2026-06-24.md)): + +- **不做**动态加载(grammar 编共享库运行期 dlopen)、**不抽** trait(YAGNI,无第二实现不抽象) +- grammar 编译期静态链接进二进制(无 ABI 坑 / 无运行时状态 / 无分发负担) +- grammar 获取集中到一个普通函数 `grammar_for(ext) -> Language`(未来加动态只改这一点,不返工调用方) + +**Phase1 范围**(覆盖 devflow 源码 + 主流用户项目): + +| 文件类型 | grammar | 备注 | +|---|---|---| +| `.rs` | tree-sitter-rust | 后端主体 | +| `.ts`/`.tsx` | tree-sitter-typescript | 前端 script | +| `.js`/`.jsx` | tree-sitter-javascript | | +| `.vue` | **借 tree-sitter-typescript** | 切 `` 内容喂 tree-sitter-typescript: +/// - 省 1 个 grammar(tree-sitter-vue 不引,降依赖体积) +/// - 避 SFC 嵌套复杂度(tree-sitter-vue 解析 template/style/script 混合,符号提取只关心 script) +/// +/// 多个 script 段取首个(`")?; + Some((source[tag_close..tag_close + end].to_string(), tag_close)) +} + +/// read_symbol 三态读取(信息密度驱动)。 +/// +/// **不按物理行返、不截断**,按语义结构层级返(消解"截断多少行"争论): +/// - `full=false`(默认 骨架态):返签名 + kind + 行范围 + 内部调用列表 + 总行数(极小,高密度) +/// - `full=true`(全文态):返完整定义体(语法边界 100% 准) +/// +/// **不报错底线**:无 grammar / 解析失败 / symbol 未找到 / Vue 无 script → 返兜底提示(grep + read_file),不 panic。 +/// +/// 参数: +/// - `source`:文件全文(由 handler 读 + 算 hash 后传入,纯函数不碰 fs) +/// - `file_hash`:文件 hash(handler 算,回传 LLM 供后续定位 / 兜底回取) +/// - `path`:路径(取 ext + 兜底消息展示) +/// - `symbol`:目标符号名(精确匹配定义节点的 name field) +/// - `full`:true=完整定义体,false=骨架 +/// - `kind_hint`:可选 kind 过滤(如 "function"/"struct"),None 匹配任意定义 kind +pub fn read_symbol( + source: &str, + file_hash: &str, + path: &str, + symbol: &str, + full: bool, + kind_hint: Option<&str>, +) -> serde_json::Value { + let ext = path.rsplit('.').next().unwrap_or("").to_lowercase(); + let is_vue = ext == "vue"; + + // Vue 借 TS:切 script 段(Vue 逻辑在 script,template/style 不做符号) + let (parse_source, parse_ext, line_offset): (String, &str, usize) = if is_vue { + match extract_vue_script(source) { + Some((script, byte_off)) => { + // Vue 切段后节点行号是 script 内相对行,加原文件中 script 起始行偏移映射回原行号 + let off_lines = source[..byte_off].matches('\n').count(); + (script, "ts", off_lines) + } + None => return fallback(symbol, path, file_hash, "vue_no_script", + "Vue 文件无 + + +"#; + let r = read_symbol(vue, "h", "App.vue", "greet", false, None); + assert_eq!(r["fallback"], serde_json::Value::Null); + assert_eq!(r["symbol"], "greet"); + assert_eq!(r["kind"], "function_declaration"); + // Vue 切段后行号映射回原文件(greet 在原文件第 5 行附近) + let start = r["line_start"].as_u64().unwrap(); + assert!(start >= 4, "Vue 行号应映射回原文件(script 段偏移后),实际 {}", start); + let calls = r["calls"].as_array().unwrap(); + assert!(calls.iter().any(|c| c == "helper")); + } + + #[test] + fn test_read_symbol_vue_no_script_fallback() { + let vue = "\n"; + let r = read_symbol(vue, "h", "App.vue", "greet", false, None); + assert_eq!(r["fallback"], true); + assert_eq!(r["reason"], "vue_no_script"); + } + + #[test] + fn test_extract_vue_script() { + let vue = "a\nb\n"; + let (script, off) = extract_vue_script(vue).unwrap(); + assert_eq!(script, "let x = 1;"); + // script 在第 3 行(0-indexed offset 后),行偏移应 = 2(\n 数) + assert_eq!(vue[..off].matches('\n').count(), 2); + } + + #[test] + fn test_extract_signature() { + // function 带 body + let s = extract_signature("pub fn foo(x: i32) -> i32 {\n x + 1\n}"); + assert_eq!(s, "pub fn foo(x: i32) -> i32"); + // struct 带 body + let s = extract_signature("struct Config {\n name: String,\n}"); + assert_eq!(s, "struct Config"); + // 声明以 ; 结尾 + let s = extract_signature("fn foo();"); + assert_eq!(s, "fn foo()"); + } + + #[test] + fn test_collect_calls_dedup() { + let src = "fn a() { b(); b(); c(); }"; + let mut parser = Parser::new(); + parser.set_language(&Language::from(tree_sitter_rust::LANGUAGE)).unwrap(); + let tree = parser.parse(src, None).unwrap(); + let calls = collect_calls(tree.root_node(), src.as_bytes()); + // b 去重只出现一次 + assert!(calls.iter().any(|c| c == "b")); + assert_eq!(calls.iter().filter(|c| *c == "b").count(), 1); + assert!(calls.iter().any(|c| c == "c")); + } + + /// 多角度验证(用户要求:实现后看效果 + 多角度测试): + /// 解析 devflow 真实源码 code_intel.rs 自身,验骨架态密度提升(设计目标:降一个量级)。 + /// 对比 read_symbol 骨架(单符号签名+calls)vs read_file 全文(整个文件所有符号)的体积。 + #[test] + fn test_read_symbol_real_devflow_token_density() { + let manifest = env!("CARGO_MANIFEST_DIR"); // src-tauri/ + let path_str = format!("{}/src/commands/ai/code_intel.rs", manifest); + let source = std::fs::read_to_string(&path_str) + .expect("应能读到 devflow 自身 code_intel.rs"); + let full_bytes = source.len(); + + // 骨架态:解析自身 read_symbol 函数(只给该符号,非整个文件) + let skeleton = read_symbol(&source, "h0", "code_intel.rs", "read_symbol", false, None); + assert_ne!(skeleton["fallback"], true, "真实文件应解析出 read_symbol 符号"); + assert_eq!(skeleton["symbol"], "read_symbol"); + assert_eq!(skeleton["kind"], "function_item"); + let skeleton_str = serde_json::to_string(&skeleton).unwrap(); + let skeleton_bytes = skeleton_str.len(); + + // 密度断言:read_symbol 骨架(单符号)<< read_file 全文(整个文件)。 + // 正是治 prompt 爆的核心:LLM 要理解 read_symbol,工具只给该符号骨架,不给全文件。 + let ratio = full_bytes as f64 / skeleton_bytes.max(1) as f64; + eprintln!( + "[密度效果] code_intel.rs 全文 {} 字节 vs read_symbol 骨架 {} 字节 → 降 {:.1}x", + full_bytes, skeleton_bytes, ratio + ); + assert!(ratio > 5.0, "骨架密度应显著高于全文(降 >5x),实际 {:.1}x", ratio); + + // full 态:完整定义体(语法边界 100% 准) + let full_sym = read_symbol(&source, "h1", "code_intel.rs", "read_symbol", true, None); + let content = full_sym["content"].as_str().expect("full 态应返 content"); + assert!(content.contains("pub fn read_symbol")); + // calls 应含 read_symbol 内部调用的函数(grammar_for/extract_symbol/fallback 等) + let calls = skeleton["calls"].as_array().expect("骨架应含 calls"); + assert!(!calls.is_empty(), "应提取出内部调用"); + } +} diff --git a/src-tauri/src/commands/ai/mod.rs b/src-tauri/src/commands/ai/mod.rs index 7a9e323..69b6772 100644 --- a/src-tauri/src/commands/ai/mod.rs +++ b/src-tauri/src/commands/ai/mod.rs @@ -29,6 +29,7 @@ pub mod agentic; pub mod augmentation; pub mod audit; +pub mod code_intel; pub mod commands; pub mod compress; pub mod conversation; diff --git a/src-tauri/src/commands/ai/tool_registry.rs b/src-tauri/src/commands/ai/tool_registry.rs index 1a2a7f0..50195c5 100644 --- a/src-tauri/src/commands/ai/tool_registry.rs +++ b/src-tauri/src/commands/ai/tool_registry.rs @@ -1042,6 +1042,50 @@ fn register_file_tools( }) })}, ); + registry.register( + "read_symbol", "AST 符号解析(信息密度驱动,治 read_file 全文回灌 prompt 爆)。提取函数/结构体/类等符号:默认返回骨架(签名+行范围+内部调用,极小高密度);传 full=true 取完整定义体。不支持/未找到时回退提示用 grep+read_file。Phase1 支持 .rs/.ts/.tsx/.js/.jsx/.vue。", + df_ai::ai_tools::object_schema(vec![("path", "string", true), ("symbol", "string", true), ("full", "boolean", false), ("kind", "string", false)]), + RiskLevel::Low, + { let allowed_dirs = allowed_dirs.clone(); Box::new(move |args: serde_json::Value| { + let allowed_dirs = allowed_dirs.clone(); + Box::pin(async move { + let snap = allowed_dirs.read().await.clone(); + let resolved = resolve_workspace_path_with_allowed( + args["path"].as_str().ok_or_else(|| anyhow::anyhow!("缺少 path 参数"))?, + &snap, + )?; + let path = resolved.to_str().ok_or_else(|| anyhow::anyhow!("路径含非法字符"))?; + let symbol = args["symbol"].as_str().ok_or_else(|| anyhow::anyhow!("缺少 symbol 参数"))?; + let full = args["full"].as_bool().unwrap_or(false); + let kind_hint = args["kind"].as_str(); + use tokio::fs::File; + use tokio::io::AsyncReadExt; + let mut file = File::open(path).await + .map_err(|e| anyhow::anyhow!("无法访问文件 {}: {}", path, e))?; + let metadata = file.metadata().await + .map_err(|e| anyhow::anyhow!("读取元数据失败 {}: {}", path, e))?; + if metadata.len() > 1_048_576 { + anyhow::bail!("文件超过 1MB 限制 ({} 字节)", metadata.len()); + } + let file_hash = compute_file_hash(&metadata); + let mut content = String::new(); + if let Err(e) = file.read_to_string(&mut content).await { + if e.kind() == std::io::ErrorKind::InvalidData { + return Ok(serde_json::json!({ + "path": path, "binary": true, "size": metadata.len(), "file_hash": file_hash, + "fallback": true, "reason": "binary", + "suggestion": "文件非 UTF-8 文本,无法 AST 解析,用 grep 搜内容", + })); + } + anyhow::bail!("读取文件失败: {}", e); + } + // 调 code_intel 纯函数(三态 + 兜底,不 panic) + Ok(crate::commands::ai::code_intel::read_symbol( + &content, &file_hash, path, symbol, full, kind_hint, + )) + }) + })}, + ); registry.register( "list_directory", "列出目录内容,返回文件和子目录列表(名称、类型、大小)", df_ai::ai_tools::object_schema(vec![("path", "string", true), ("recursive", "boolean", false), ("skip_noise_dirs", "boolean", false), ("max_depth", "integer", false)]),