lxy/DevFlow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增: F-05多模态Phase2a后端(ContentPart+parts字段+provider图片块+truncate防撑爆)

Browse Source
This commit is contained in:
绝尘
2026-06-17 03:09:10 +08:00
parent 71718c1cd8
commit e3cd44802a
5 changed files with 496 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
crates/df-ai-core/src/provider.rs
View File

@@ -38,11 +38,69 @@ pub struct CompletionRequest {
pub tool_choice: Option<serde_json::Value>,
}
/// 多模态消息内容片。Text 片为字符串；Image 片可走 url 或 base64（二选一，base64 非空时 url 忽略）。
///
/// F-260614-05 Phase 2a 后端：ContentPart 作为 `ChatMessage.parts` 的元素类型。
/// 设计上 `content: String`（纯文本主载荷）保持不变，多模态片挂在 `parts`：
/// 这样未接入多模态的调用方（audit/title/commands/knowledge_inject 等读 content 当字符串）
/// 零回归，避免一次性改全仓。provider 转换层在 `has_image()` 为真时把 parts 透传给
/// vision 端点，否则只用 content 文本（保持纯文本端点兼容）。
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContentPart {
/// 文本片
Text { text: String },
/// 图片片。
/// - url：http(s) 可达 URL（provider 直接转发，不读字节）。
/// - base64：data URI 之外的纯 base64 字符串 + media_type（provider 内嵌转发）。
/// base64 非空时 url 忽略，便于前端上行无网络回拉的本地粘贴图。
Image {
url: Option<String>,
base64: Option<String>,
/// base64 模式必填（image/png | image/jpeg | image/webp | image/gif）；url 模式可空。
media_type: Option<String>,
/// 可选 alt（vision 模型/降级文本时用）
#[serde(skip_serializing_if = "Option::is_none")]
alt: Option<String>,
},
}
impl ContentPart {
pub fn text(text: impl Into<String>) -> Self {
ContentPart::Text { text: text.into() }
}
pub fn image_base64(media_type: impl Into<String>, data: impl Into<String>) -> Self {
ContentPart::Image {
url: None,
base64: Some(data.into()),
media_type: Some(media_type.into()),
alt: None,
}
}
pub fn image_url(url: impl Into<String>) -> Self {
ContentPart::Image { url: Some(url.into()), base64: None, media_type: None, alt: None }
}
/// 是否图片片
pub fn is_image(&self) -> bool {
matches!(self, ContentPart::Image { .. })
}
}
/// 聊天消息
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatMessage {
pub role: MessageRole,
pub content: String,
/// 多模态内容片（F-260614-05 Phase 2a）。
///
/// `None`/空 → 纯文本消息（绝大多数场景，content 即全部载荷）。
/// `Some(含 Image 片)` → 多模态消息，provider 在 `has_image()` 为真时把 parts
/// 连同 content（作为前置 Text 片）一起转成 OpenAI/Anthropic 的 content blocks。
/// content 字段始终保留人类可读文本（文本消息的 Text 片内容与 content 一致），
/// 保证 audit/title/export 等读 content 当字符串的调用方零回归。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parts: Option<Vec<ContentPart>>,
/// 工具调用 ID（role=Tool 时必填）
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
@@ -61,19 +119,50 @@ pub struct ChatMessage {
impl ChatMessage {
pub fn system(content: impl Into<String>) -> Self {
Self { role: MessageRole::System, content: content.into(), tool_call_id: None, tool_calls: None, model: None, status: None }
Self { role: MessageRole::System, content: content.into(), parts: None, tool_call_id: None, tool_calls: None, model: None, status: None }
}
pub fn user(content: impl Into<String>) -> Self {
Self { role: MessageRole::User, content: content.into(), tool_call_id: None, tool_calls: None, model: None, status: None }
Self { role: MessageRole::User, content: content.into(), parts: None, tool_call_id: None, tool_calls: None, model: None, status: None }
}
pub fn assistant(content: impl Into<String>) -> Self {
Self { role: MessageRole::Assistant, content: content.into(), tool_call_id: None, tool_calls: None, model: None, status: None }
Self { role: MessageRole::Assistant, content: content.into(), parts: None, tool_call_id: None, tool_calls: None, model: None, status: None }
}
pub fn assistant_with_tools(content: impl Into<String>, tool_calls: Vec<ToolCall>) -> Self {
Self { role: MessageRole::Assistant, content: content.into(), tool_call_id: None, tool_calls: Some(tool_calls), model: None, status: None }
Self { role: MessageRole::Assistant, content: content.into(), parts: None, tool_call_id: None, tool_calls: Some(tool_calls), model: None, status: None }
}
pub fn tool_result(call_id: impl Into<String>, content: impl Into<String>) -> Self {
Self { role: MessageRole::Tool, content: content.into(), tool_call_id: Some(call_id.into()), tool_calls: None, model: None, status: None }
Self { role: MessageRole::Tool, content: content.into(), parts: None, tool_call_id: Some(call_id.into()), tool_calls: None, model: None, status: None }
}
/// 多模态 user 消息：content 文本 + parts（含 Image 片）。
/// content 作为人类可读文本（也作非 vision 端点降级载荷）；parts 透传给 vision 端点。
pub fn user_parts(content: impl Into<String>, parts: Vec<ContentPart>) -> Self {
Self { role: MessageRole::User, content: content.into(), parts: Some(parts), tool_call_id: None, tool_calls: None, model: None, status: None }
}
/// 是否含图片片（供 provider 判定走多模态分支）。
pub fn has_image(&self) -> bool {
self.parts.as_ref().map(|ps| ps.iter().any(|p| p.is_image())).unwrap_or(false)
}
/// parts 若存在则返回引用，否则 None。
pub fn parts(&self) -> Option<&[ContentPart]> {
self.parts.as_deref()
}
/// 把 content + parts 拍平为有序 ContentPart 序列：先 content 作 Text 片（非空时），
/// 再追加 parts（若有）。供 provider 生成 content blocks（保证文本在前、图片在后）。
pub fn flattened_parts(&self) -> Vec<ContentPart> {
let mut out: Vec<ContentPart> = Vec::new();
if !self.content.is_empty() {
out.push(ContentPart::Text { text: self.content.clone() });
}
if let Some(ps) = &self.parts {
for p in ps {
out.push(p.clone());
}
}
out
}
/// 是否处于 active 态（status 为 None 或 "active"）。其余状态一律 false。
@@ -274,4 +363,94 @@ mod tests {
m.status = Some("compressed".to_string());
assert!(!m.is_active(), "compressed 应不 active（白名单隔离）");
}
// ---------- F-260614-05 Phase 2a ContentPart ----------
/// 老 JSON（无 parts 字段）反序列化时 parts 应为 None（向前兼容）
#[test]
fn contentpart_legacy_json_no_parts() {
let json = r#"{"role":"user","content":"hello"}"#;
let m: ChatMessage = serde_json::from_str(json).expect("老 JSON 应可反序列化");
assert_eq!(m.content, "hello");
assert!(m.parts.is_none(), "老 JSON 无 parts 字段 → None");
assert!(!m.has_image());
}
/// 新 JSON 带 parts（含 Image 片）反序列化 round-trip
#[test]
fn contentpart_with_image_roundtrip() {
let m = ChatMessage::user_parts(
"看这张图",
vec![
ContentPart::image_base64("image/png", "iVBORw0KGgo="),
ContentPart::text("说明"),
],
);
let json = serde_json::to_string(&m).expect("序列化");
// parts 顺序：[image_base64, text] → 首元素是 image
assert!(json.contains(r#""parts":[{"type":"image""#));
assert!(json.contains(r#""type":"text""#));
assert!(json.contains(r#""base64":"iVBORw0KGgo=""#));
assert!(json.contains(r#""media_type":"image/png""#));
let back: ChatMessage = serde_json::from_str(&json).expect("反序列化 round-trip");
assert_eq!(back.content, "看这张图");
assert!(back.has_image());
let parts = back.parts.expect("parts 应存在");
assert_eq!(parts.len(), 2);
assert!(parts[0].is_image());
}
/// has_image 判定
#[test]
fn contentpart_has_image_detection() {
assert!(!ChatMessage::user("纯文本").has_image());
let m = ChatMessage::user_parts("t", vec![ContentPart::text("只文本片")]);
assert!(!m.has_image(), "仅 Text 片不算 has_image");
let m = ChatMessage::user_parts(
"t",
vec![ContentPart::text("前缀"), ContentPart::image_url("https://x/a.png")],
);
assert!(m.has_image());
}
/// flattened_parts：content 非空 → 前置 Text 片 + parts 追加
#[test]
fn contentpart_flattened_parts() {
let m = ChatMessage::user("hi");
let flat = m.flattened_parts();
assert_eq!(flat.len(), 1);
assert_eq!(flat[0], ContentPart::Text { text: "hi".into() });
let m = ChatMessage::user_parts(
"cap",
vec![ContentPart::image_url("u"), ContentPart::text("尾")],
);
let flat = m.flattened_parts();
assert_eq!(flat.len(), 3);
assert_eq!(flat[0], ContentPart::Text { text: "cap".into() });
assert!(flat[1].is_image());
assert_eq!(flat[2], ContentPart::Text { text: "尾".into() });
let mut m = ChatMessage::user("");
m.parts = Some(vec![ContentPart::text("x")]);
let flat = m.flattened_parts();
assert_eq!(flat.len(), 1, "空 content 不应产生空 Text 片");
}
/// 向后兼容：旧代码 ChatMessage 字面量构造仍合法（audit/title/commands 零回归）
#[test]
fn contentpart_struct_literal_compat() {
let m = ChatMessage {
role: MessageRole::User,
content: "字面量构造".into(),
parts: None,
tool_call_id: None,
tool_calls: None,
model: None,
status: None,
};
assert_eq!(m.content, "字面量构造");
assert!(m.parts.is_none());
}
}