lxy/DevFlow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增: F-05多模态Phase2a后端(ContentPart+parts字段+provider图片块+truncate防撑爆)

Browse Source
This commit is contained in:
绝尘
2026-06-17 03:09:10 +08:00
parent 71718c1cd8
commit e3cd44802a
5 changed files with 496 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
crates/df-ai/src/anthropic_compat.rs
View File

@@ -18,6 +18,9 @@ use crate::provider::{
CompletionRequest, CompletionResponse, LlmProvider, MessageRole,
StreamChunk, StreamResult, TokenUsage, ToolCall, ToolCallDelta,
};
// ChatMessage 仅单测构造 CompletionRequest 用，避免非 test 构建的 unused import 警告。
#[cfg(test)]
use crate::provider::ChatMessage;
use crate::retry::{
retry_with_backoff, AttemptOutcome, is_reqwest_error_retryable, is_status_retryable,
};
@@ -330,7 +333,49 @@ impl AnthropicCompatProvider {
}
MessageRole::User => {
Self::flush_tool_results(&mut messages, &mut pending_tool_results);
messages.push(serde_json::json!({ "role": "user", "content": m.content }));
// F-260614-05 Phase 2a: 多模态 user 消息 → content blocks 数组（text/image）。
// 含图时把 content + parts 拍平成 blocks：Text 片 → {type:text}，
// Image 片 → {type:image, source:{type:base64, media_type, data}}。
// Anthropic 协议要求 image 必须内嵌 base64（不接受 URL 直传），
// url 模式应由 commands 层预拉字节回填 base64（provider 不发额外 HTTP）。
// 纯文本消息（无图）保持原字符串简写，与现有端点零回归。
if m.has_image() {
let blocks: Vec<serde_json::Value> = m
.flattened_parts()
.into_iter()
.map(|p| match p {
crate::provider::ContentPart::Text { text } => serde_json::json!({
"type": "text",
"text": text,
}),
crate::provider::ContentPart::Image { url, base64, media_type, alt: _ } => {
let mt = media_type.clone().unwrap_or_else(|| "image/png".into());
let data = base64.clone().unwrap_or_else(|| {
// url 模式无 base64 时 provider 层兜底（commands 层应已预拉）：
// 发空 data 会让 Anthropic 报错，warn 提示但不阻塞。
if url.is_some() {
warn!(
url = ?url,
"Anthropic user 消息含 Image(url) 但未预拉 base64，将发空 data（commands 层应回填 base64）"
);
}
String::new()
});
serde_json::json!({
"type": "image",
"source": {
"type": "base64",
"media_type": mt,
"data": data,
}
})
}
})
.collect();
messages.push(serde_json::json!({ "role": "user", "content": blocks }));
} else {
messages.push(serde_json::json!({ "role": "user", "content": m.content }));
}
}
MessageRole::Assistant => {
Self::flush_tool_results(&mut messages, &mut pending_tool_results);
@@ -742,4 +787,63 @@ mod tests {
assert_eq!(c.delta, "");
assert!(acc.is_none());
}
// ---------- F-260614-05 Phase 2a 多模态 convert_request ----------
/// 含图 user 消息 → content blocks（text + image source.base64）
#[test]
fn anthropic_convert_multimodal_user_blocks() {
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
let req = CompletionRequest {
model: "claude-3-5-sonnet".into(),
messages: vec![ChatMessage::user_parts(
"看图",
vec![crate::provider::ContentPart::image_base64("image/png", "iVBOR")],
)],
temperature: None,
max_tokens: None,
stream: false,
tools: None,
tool_choice: None,
};
let body = provider.convert_request(req);
// user 消息 content 应为数组形态
let user_msg = body
.messages
.iter()
.find(|m| m.get("role").and_then(|r| r.as_str()) == Some("user"))
.expect("应有 user 消息");
let content = user_msg.get("content").and_then(|c| c.as_array()).expect("user content 应为数组");
// 顺序：content "看图" → text 块，image 片 → image 块
assert_eq!(content.len(), 2);
assert_eq!(content[0]["type"], "text");
assert_eq!(content[0]["text"], "看图");
assert_eq!(content[1]["type"], "image");
assert_eq!(content[1]["source"]["type"], "base64");
assert_eq!(content[1]["source"]["media_type"], "image/png");
assert_eq!(content[1]["source"]["data"], "iVBOR");
}
/// 纯文本 user 消息 → content 仍是字符串简写（无图不数组化）
#[test]
fn anthropic_convert_text_only_user_remains_string() {
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
let req = CompletionRequest {
model: "claude-3-5-sonnet".into(),
messages: vec![ChatMessage::user("hello")],
temperature: None,
max_tokens: None,
stream: false,
tools: None,
tool_choice: None,
};
let body = provider.convert_request(req);
let user_msg = body
.messages
.iter()
.find(|m| m.get("role").and_then(|r| r.as_str()) == Some("user"))
.expect("应有 user 消息");
// 纯文本 → 字符串简写（非数组）
assert_eq!(user_msg.get("content").and_then(|c| c.as_str()), Some("hello"));
}
}

96
crates/df-ai/src/openai_compat.rs
View File

@@ -16,6 +16,9 @@ use crate::provider::{
CompletionRequest, CompletionResponse, LlmProvider, StreamChunk, StreamResult,
TokenUsage, ToolCall, ToolCallDelta,
};
// ChatMessage 仅单测构造 CompletionRequest 用，避免非 test 构建的 unused import 警告。
#[cfg(test)]
use crate::provider::ChatMessage;
use crate::retry::{
retry_with_backoff, AttemptOutcome, is_reqwest_error_retryable, is_status_retryable,
};
@@ -44,10 +47,13 @@ struct OpenAiRequest {
}
/// OpenAI 消息格式
///
/// `content` 为 `serde_json::Value`：纯文本消息走字符串简写（与老端点兼容），
/// 含图消息走 `[{type:"text",text},{type:"image_url",image_url:{url}}]` 数组。
#[derive(Debug, Clone, Serialize, Deserialize)]
struct OpenAiMessage {
role: String,
content: String,
content: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
tool_call_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -308,6 +314,38 @@ impl OpenAICompatProvider {
crate::provider::MessageRole::Assistant => "assistant",
crate::provider::MessageRole::Tool => "tool",
};
// F-260614-05 Phase 2a: 多模态 content（须在 move m.tool_calls 之前算，借用 m）。
// 含图消息走 content 数组（text/image_url）；纯文本走字符串简写
// （保持与现有纯文本端点零回归）。image_url 支持 data URI（base64）与 http(s) URL。
let content = if m.has_image() {
let parts: Vec<serde_json::Value> = m
.flattened_parts()
.into_iter()
.map(|p| match p {
crate::provider::ContentPart::Text { text } => serde_json::json!({
"type": "text",
"text": text,
}),
crate::provider::ContentPart::Image { url, base64, media_type, alt: _ } => {
let final_url = match (base64, url, media_type) {
(Some(b), _, Some(mt)) => {
format!("data:{};base64,{}", mt, b)
}
(None, Some(u), _) => u,
// 兜底：缺数据时退化为占位，避免发空 url 触发 400
_ => String::new(),
};
serde_json::json!({
"type": "image_url",
"image_url": { "url": final_url },
})
}
})
.collect();
serde_json::Value::Array(parts)
} else {
serde_json::Value::String(m.content.clone())
};
let tool_calls = m.tool_calls.map(|calls| {
calls
.into_iter()
@@ -325,7 +363,7 @@ impl OpenAICompatProvider {
});
OpenAiMessage {
role: role.to_string(),
content: m.content,
content,
tool_call_id: m.tool_call_id,
tool_calls,
}
@@ -691,4 +729,58 @@ mod tests {
assert_eq!(tcs[0].function_arguments.as_deref(), Some("{\"q\":"));
assert!(!c.finished);
}
// ---------- F-260614-05 Phase 2a 多模态 convert_request ----------
/// 含图消息 → content 数组（text + image_url data URI）；纯文本 → 字符串简写
#[test]
fn openai_convert_multimodal_content() {
let provider = OpenAICompatProvider::new("https://api.openai.com", "k", "gpt-4o");
let req = CompletionRequest {
model: "gpt-4o".into(),
messages: vec![ChatMessage::user_parts(
"看图",
vec![
crate::provider::ContentPart::image_base64("image/png", "iVBOR"),
crate::provider::ContentPart::image_url("https://x/a.png"),
],
)],
temperature: None,
max_tokens: None,
stream: false,
tools: None,
tool_choice: None,
};
let out = provider.convert_request(req);
let msg = &out.messages[0];
// content 是数组：[text "看图", image_url(data URI), image_url(http url)]
let arr = msg.content.as_array().expect("含图 → content 数组");
assert_eq!(arr.len(), 3);
assert_eq!(arr[0]["type"], "text");
assert_eq!(arr[0]["text"], "看图");
assert_eq!(arr[1]["type"], "image_url");
assert_eq!(
arr[1]["image_url"]["url"],
"data:image/png;base64,iVBOR"
);
assert_eq!(arr[2]["image_url"]["url"], "https://x/a.png");
}
/// 纯文本消息 → content 仍是字符串简写（无图不数组化，对齐纯文本端点兼容）
#[test]
fn openai_convert_text_only_remains_string() {
let provider = OpenAICompatProvider::new("https://api.openai.com", "k", "gpt-4o");
let req = CompletionRequest {
model: "gpt-4o".into(),
messages: vec![ChatMessage::user("hello")],
temperature: None,
max_tokens: None,
stream: false,
tools: None,
tool_choice: None,
};
let out = provider.convert_request(req);
let msg = &out.messages[0];
assert_eq!(msg.content, serde_json::Value::String("hello".into()));
}
}