新增: F-05多模态Phase2a后端(ContentPart+parts字段+provider图片块+truncate防撑爆)
This commit is contained in:
@@ -38,11 +38,69 @@ pub struct CompletionRequest {
|
|||||||
pub tool_choice: Option<serde_json::Value>,
|
pub tool_choice: Option<serde_json::Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 多模态消息内容片。Text 片为字符串;Image 片可走 url 或 base64(二选一,base64 非空时 url 忽略)。
|
||||||
|
///
|
||||||
|
/// F-260614-05 Phase 2a 后端:ContentPart 作为 `ChatMessage.parts` 的元素类型。
|
||||||
|
/// 设计上 `content: String`(纯文本主载荷)保持不变,多模态片挂在 `parts`:
|
||||||
|
/// 这样未接入多模态的调用方(audit/title/commands/knowledge_inject 等读 content 当字符串)
|
||||||
|
/// 零回归,避免一次性改全仓。provider 转换层在 `has_image()` 为真时把 parts 透传给
|
||||||
|
/// vision 端点,否则只用 content 文本(保持纯文本端点兼容)。
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
#[serde(tag = "type", rename_all = "snake_case")]
|
||||||
|
pub enum ContentPart {
|
||||||
|
/// 文本片
|
||||||
|
Text { text: String },
|
||||||
|
/// 图片片。
|
||||||
|
/// - url:http(s) 可达 URL(provider 直接转发,不读字节)。
|
||||||
|
/// - base64:data URI 之外的纯 base64 字符串 + media_type(provider 内嵌转发)。
|
||||||
|
/// base64 非空时 url 忽略,便于前端上行无网络回拉的本地粘贴图。
|
||||||
|
Image {
|
||||||
|
url: Option<String>,
|
||||||
|
base64: Option<String>,
|
||||||
|
/// base64 模式必填(image/png | image/jpeg | image/webp | image/gif);url 模式可空。
|
||||||
|
media_type: Option<String>,
|
||||||
|
/// 可选 alt(vision 模型/降级文本时用)
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
alt: Option<String>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ContentPart {
|
||||||
|
pub fn text(text: impl Into<String>) -> Self {
|
||||||
|
ContentPart::Text { text: text.into() }
|
||||||
|
}
|
||||||
|
pub fn image_base64(media_type: impl Into<String>, data: impl Into<String>) -> Self {
|
||||||
|
ContentPart::Image {
|
||||||
|
url: None,
|
||||||
|
base64: Some(data.into()),
|
||||||
|
media_type: Some(media_type.into()),
|
||||||
|
alt: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn image_url(url: impl Into<String>) -> Self {
|
||||||
|
ContentPart::Image { url: Some(url.into()), base64: None, media_type: None, alt: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 是否图片片
|
||||||
|
pub fn is_image(&self) -> bool {
|
||||||
|
matches!(self, ContentPart::Image { .. })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// 聊天消息
|
/// 聊天消息
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ChatMessage {
|
pub struct ChatMessage {
|
||||||
pub role: MessageRole,
|
pub role: MessageRole,
|
||||||
pub content: String,
|
pub content: String,
|
||||||
|
/// 多模态内容片(F-260614-05 Phase 2a)。
|
||||||
|
///
|
||||||
|
/// `None`/空 → 纯文本消息(绝大多数场景,content 即全部载荷)。
|
||||||
|
/// `Some(含 Image 片)` → 多模态消息,provider 在 `has_image()` 为真时把 parts
|
||||||
|
/// 连同 content(作为前置 Text 片)一起转成 OpenAI/Anthropic 的 content blocks。
|
||||||
|
/// content 字段始终保留人类可读文本(文本消息的 Text 片内容与 content 一致),
|
||||||
|
/// 保证 audit/title/export 等读 content 当字符串的调用方零回归。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub parts: Option<Vec<ContentPart>>,
|
||||||
/// 工具调用 ID(role=Tool 时必填)
|
/// 工具调用 ID(role=Tool 时必填)
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub tool_call_id: Option<String>,
|
pub tool_call_id: Option<String>,
|
||||||
@@ -61,19 +119,50 @@ pub struct ChatMessage {
|
|||||||
|
|
||||||
impl ChatMessage {
|
impl ChatMessage {
|
||||||
pub fn system(content: impl Into<String>) -> Self {
|
pub fn system(content: impl Into<String>) -> Self {
|
||||||
Self { role: MessageRole::System, content: content.into(), tool_call_id: None, tool_calls: None, model: None, status: None }
|
Self { role: MessageRole::System, content: content.into(), parts: None, tool_call_id: None, tool_calls: None, model: None, status: None }
|
||||||
}
|
}
|
||||||
pub fn user(content: impl Into<String>) -> Self {
|
pub fn user(content: impl Into<String>) -> Self {
|
||||||
Self { role: MessageRole::User, content: content.into(), tool_call_id: None, tool_calls: None, model: None, status: None }
|
Self { role: MessageRole::User, content: content.into(), parts: None, tool_call_id: None, tool_calls: None, model: None, status: None }
|
||||||
}
|
}
|
||||||
pub fn assistant(content: impl Into<String>) -> Self {
|
pub fn assistant(content: impl Into<String>) -> Self {
|
||||||
Self { role: MessageRole::Assistant, content: content.into(), tool_call_id: None, tool_calls: None, model: None, status: None }
|
Self { role: MessageRole::Assistant, content: content.into(), parts: None, tool_call_id: None, tool_calls: None, model: None, status: None }
|
||||||
}
|
}
|
||||||
pub fn assistant_with_tools(content: impl Into<String>, tool_calls: Vec<ToolCall>) -> Self {
|
pub fn assistant_with_tools(content: impl Into<String>, tool_calls: Vec<ToolCall>) -> Self {
|
||||||
Self { role: MessageRole::Assistant, content: content.into(), tool_call_id: None, tool_calls: Some(tool_calls), model: None, status: None }
|
Self { role: MessageRole::Assistant, content: content.into(), parts: None, tool_call_id: None, tool_calls: Some(tool_calls), model: None, status: None }
|
||||||
}
|
}
|
||||||
pub fn tool_result(call_id: impl Into<String>, content: impl Into<String>) -> Self {
|
pub fn tool_result(call_id: impl Into<String>, content: impl Into<String>) -> Self {
|
||||||
Self { role: MessageRole::Tool, content: content.into(), tool_call_id: Some(call_id.into()), tool_calls: None, model: None, status: None }
|
Self { role: MessageRole::Tool, content: content.into(), parts: None, tool_call_id: Some(call_id.into()), tool_calls: None, model: None, status: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 多模态 user 消息:content 文本 + parts(含 Image 片)。
|
||||||
|
/// content 作为人类可读文本(也作非 vision 端点降级载荷);parts 透传给 vision 端点。
|
||||||
|
pub fn user_parts(content: impl Into<String>, parts: Vec<ContentPart>) -> Self {
|
||||||
|
Self { role: MessageRole::User, content: content.into(), parts: Some(parts), tool_call_id: None, tool_calls: None, model: None, status: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 是否含图片片(供 provider 判定走多模态分支)。
|
||||||
|
pub fn has_image(&self) -> bool {
|
||||||
|
self.parts.as_ref().map(|ps| ps.iter().any(|p| p.is_image())).unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// parts 若存在则返回引用,否则 None。
|
||||||
|
pub fn parts(&self) -> Option<&[ContentPart]> {
|
||||||
|
self.parts.as_deref()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 把 content + parts 拍平为有序 ContentPart 序列:先 content 作 Text 片(非空时),
|
||||||
|
/// 再追加 parts(若有)。供 provider 生成 content blocks(保证文本在前、图片在后)。
|
||||||
|
pub fn flattened_parts(&self) -> Vec<ContentPart> {
|
||||||
|
let mut out: Vec<ContentPart> = Vec::new();
|
||||||
|
if !self.content.is_empty() {
|
||||||
|
out.push(ContentPart::Text { text: self.content.clone() });
|
||||||
|
}
|
||||||
|
if let Some(ps) = &self.parts {
|
||||||
|
for p in ps {
|
||||||
|
out.push(p.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 是否处于 active 态(status 为 None 或 "active")。其余状态一律 false。
|
/// 是否处于 active 态(status 为 None 或 "active")。其余状态一律 false。
|
||||||
@@ -274,4 +363,94 @@ mod tests {
|
|||||||
m.status = Some("compressed".to_string());
|
m.status = Some("compressed".to_string());
|
||||||
assert!(!m.is_active(), "compressed 应不 active(白名单隔离)");
|
assert!(!m.is_active(), "compressed 应不 active(白名单隔离)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------- F-260614-05 Phase 2a ContentPart ----------
|
||||||
|
|
||||||
|
/// 老 JSON(无 parts 字段)反序列化时 parts 应为 None(向前兼容)
|
||||||
|
#[test]
|
||||||
|
fn contentpart_legacy_json_no_parts() {
|
||||||
|
let json = r#"{"role":"user","content":"hello"}"#;
|
||||||
|
let m: ChatMessage = serde_json::from_str(json).expect("老 JSON 应可反序列化");
|
||||||
|
assert_eq!(m.content, "hello");
|
||||||
|
assert!(m.parts.is_none(), "老 JSON 无 parts 字段 → None");
|
||||||
|
assert!(!m.has_image());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 新 JSON 带 parts(含 Image 片)反序列化 round-trip
|
||||||
|
#[test]
|
||||||
|
fn contentpart_with_image_roundtrip() {
|
||||||
|
let m = ChatMessage::user_parts(
|
||||||
|
"看这张图",
|
||||||
|
vec![
|
||||||
|
ContentPart::image_base64("image/png", "iVBORw0KGgo="),
|
||||||
|
ContentPart::text("说明"),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let json = serde_json::to_string(&m).expect("序列化");
|
||||||
|
// parts 顺序:[image_base64, text] → 首元素是 image
|
||||||
|
assert!(json.contains(r#""parts":[{"type":"image""#));
|
||||||
|
assert!(json.contains(r#""type":"text""#));
|
||||||
|
assert!(json.contains(r#""base64":"iVBORw0KGgo=""#));
|
||||||
|
assert!(json.contains(r#""media_type":"image/png""#));
|
||||||
|
|
||||||
|
let back: ChatMessage = serde_json::from_str(&json).expect("反序列化 round-trip");
|
||||||
|
assert_eq!(back.content, "看这张图");
|
||||||
|
assert!(back.has_image());
|
||||||
|
let parts = back.parts.expect("parts 应存在");
|
||||||
|
assert_eq!(parts.len(), 2);
|
||||||
|
assert!(parts[0].is_image());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// has_image 判定
|
||||||
|
#[test]
|
||||||
|
fn contentpart_has_image_detection() {
|
||||||
|
assert!(!ChatMessage::user("纯文本").has_image());
|
||||||
|
let m = ChatMessage::user_parts("t", vec![ContentPart::text("只文本片")]);
|
||||||
|
assert!(!m.has_image(), "仅 Text 片不算 has_image");
|
||||||
|
let m = ChatMessage::user_parts(
|
||||||
|
"t",
|
||||||
|
vec![ContentPart::text("前缀"), ContentPart::image_url("https://x/a.png")],
|
||||||
|
);
|
||||||
|
assert!(m.has_image());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// flattened_parts:content 非空 → 前置 Text 片 + parts 追加
|
||||||
|
#[test]
|
||||||
|
fn contentpart_flattened_parts() {
|
||||||
|
let m = ChatMessage::user("hi");
|
||||||
|
let flat = m.flattened_parts();
|
||||||
|
assert_eq!(flat.len(), 1);
|
||||||
|
assert_eq!(flat[0], ContentPart::Text { text: "hi".into() });
|
||||||
|
|
||||||
|
let m = ChatMessage::user_parts(
|
||||||
|
"cap",
|
||||||
|
vec![ContentPart::image_url("u"), ContentPart::text("尾")],
|
||||||
|
);
|
||||||
|
let flat = m.flattened_parts();
|
||||||
|
assert_eq!(flat.len(), 3);
|
||||||
|
assert_eq!(flat[0], ContentPart::Text { text: "cap".into() });
|
||||||
|
assert!(flat[1].is_image());
|
||||||
|
assert_eq!(flat[2], ContentPart::Text { text: "尾".into() });
|
||||||
|
|
||||||
|
let mut m = ChatMessage::user("");
|
||||||
|
m.parts = Some(vec![ContentPart::text("x")]);
|
||||||
|
let flat = m.flattened_parts();
|
||||||
|
assert_eq!(flat.len(), 1, "空 content 不应产生空 Text 片");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 向后兼容:旧代码 ChatMessage 字面量构造仍合法(audit/title/commands 零回归)
|
||||||
|
#[test]
|
||||||
|
fn contentpart_struct_literal_compat() {
|
||||||
|
let m = ChatMessage {
|
||||||
|
role: MessageRole::User,
|
||||||
|
content: "字面量构造".into(),
|
||||||
|
parts: None,
|
||||||
|
tool_call_id: None,
|
||||||
|
tool_calls: None,
|
||||||
|
model: None,
|
||||||
|
status: None,
|
||||||
|
};
|
||||||
|
assert_eq!(m.content, "字面量构造");
|
||||||
|
assert!(m.parts.is_none());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,9 @@ use crate::provider::{
|
|||||||
CompletionRequest, CompletionResponse, LlmProvider, MessageRole,
|
CompletionRequest, CompletionResponse, LlmProvider, MessageRole,
|
||||||
StreamChunk, StreamResult, TokenUsage, ToolCall, ToolCallDelta,
|
StreamChunk, StreamResult, TokenUsage, ToolCall, ToolCallDelta,
|
||||||
};
|
};
|
||||||
|
// ChatMessage 仅单测构造 CompletionRequest 用,避免非 test 构建的 unused import 警告。
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::provider::ChatMessage;
|
||||||
use crate::retry::{
|
use crate::retry::{
|
||||||
retry_with_backoff, AttemptOutcome, is_reqwest_error_retryable, is_status_retryable,
|
retry_with_backoff, AttemptOutcome, is_reqwest_error_retryable, is_status_retryable,
|
||||||
};
|
};
|
||||||
@@ -330,7 +333,49 @@ impl AnthropicCompatProvider {
|
|||||||
}
|
}
|
||||||
MessageRole::User => {
|
MessageRole::User => {
|
||||||
Self::flush_tool_results(&mut messages, &mut pending_tool_results);
|
Self::flush_tool_results(&mut messages, &mut pending_tool_results);
|
||||||
messages.push(serde_json::json!({ "role": "user", "content": m.content }));
|
// F-260614-05 Phase 2a: 多模态 user 消息 → content blocks 数组(text/image)。
|
||||||
|
// 含图时把 content + parts 拍平成 blocks:Text 片 → {type:text},
|
||||||
|
// Image 片 → {type:image, source:{type:base64, media_type, data}}。
|
||||||
|
// Anthropic 协议要求 image 必须内嵌 base64(不接受 URL 直传),
|
||||||
|
// url 模式应由 commands 层预拉字节回填 base64(provider 不发额外 HTTP)。
|
||||||
|
// 纯文本消息(无图)保持原字符串简写,与现有端点零回归。
|
||||||
|
if m.has_image() {
|
||||||
|
let blocks: Vec<serde_json::Value> = m
|
||||||
|
.flattened_parts()
|
||||||
|
.into_iter()
|
||||||
|
.map(|p| match p {
|
||||||
|
crate::provider::ContentPart::Text { text } => serde_json::json!({
|
||||||
|
"type": "text",
|
||||||
|
"text": text,
|
||||||
|
}),
|
||||||
|
crate::provider::ContentPart::Image { url, base64, media_type, alt: _ } => {
|
||||||
|
let mt = media_type.clone().unwrap_or_else(|| "image/png".into());
|
||||||
|
let data = base64.clone().unwrap_or_else(|| {
|
||||||
|
// url 模式无 base64 时 provider 层兜底(commands 层应已预拉):
|
||||||
|
// 发空 data 会让 Anthropic 报错,warn 提示但不阻塞。
|
||||||
|
if url.is_some() {
|
||||||
|
warn!(
|
||||||
|
url = ?url,
|
||||||
|
"Anthropic user 消息含 Image(url) 但未预拉 base64,将发空 data(commands 层应回填 base64)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
String::new()
|
||||||
|
});
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "image",
|
||||||
|
"source": {
|
||||||
|
"type": "base64",
|
||||||
|
"media_type": mt,
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
messages.push(serde_json::json!({ "role": "user", "content": blocks }));
|
||||||
|
} else {
|
||||||
|
messages.push(serde_json::json!({ "role": "user", "content": m.content }));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
MessageRole::Assistant => {
|
MessageRole::Assistant => {
|
||||||
Self::flush_tool_results(&mut messages, &mut pending_tool_results);
|
Self::flush_tool_results(&mut messages, &mut pending_tool_results);
|
||||||
@@ -742,4 +787,63 @@ mod tests {
|
|||||||
assert_eq!(c.delta, "");
|
assert_eq!(c.delta, "");
|
||||||
assert!(acc.is_none());
|
assert!(acc.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------- F-260614-05 Phase 2a 多模态 convert_request ----------
|
||||||
|
|
||||||
|
/// 含图 user 消息 → content blocks(text + image source.base64)
|
||||||
|
#[test]
|
||||||
|
fn anthropic_convert_multimodal_user_blocks() {
|
||||||
|
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||||
|
let req = CompletionRequest {
|
||||||
|
model: "claude-3-5-sonnet".into(),
|
||||||
|
messages: vec![ChatMessage::user_parts(
|
||||||
|
"看图",
|
||||||
|
vec![crate::provider::ContentPart::image_base64("image/png", "iVBOR")],
|
||||||
|
)],
|
||||||
|
temperature: None,
|
||||||
|
max_tokens: None,
|
||||||
|
stream: false,
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
|
};
|
||||||
|
let body = provider.convert_request(req);
|
||||||
|
// user 消息 content 应为数组形态
|
||||||
|
let user_msg = body
|
||||||
|
.messages
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.get("role").and_then(|r| r.as_str()) == Some("user"))
|
||||||
|
.expect("应有 user 消息");
|
||||||
|
let content = user_msg.get("content").and_then(|c| c.as_array()).expect("user content 应为数组");
|
||||||
|
// 顺序:content "看图" → text 块,image 片 → image 块
|
||||||
|
assert_eq!(content.len(), 2);
|
||||||
|
assert_eq!(content[0]["type"], "text");
|
||||||
|
assert_eq!(content[0]["text"], "看图");
|
||||||
|
assert_eq!(content[1]["type"], "image");
|
||||||
|
assert_eq!(content[1]["source"]["type"], "base64");
|
||||||
|
assert_eq!(content[1]["source"]["media_type"], "image/png");
|
||||||
|
assert_eq!(content[1]["source"]["data"], "iVBOR");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 纯文本 user 消息 → content 仍是字符串简写(无图不数组化)
|
||||||
|
#[test]
|
||||||
|
fn anthropic_convert_text_only_user_remains_string() {
|
||||||
|
let provider = AnthropicCompatProvider::new("https://api.anthropic.com", "k", "claude-3-5-sonnet");
|
||||||
|
let req = CompletionRequest {
|
||||||
|
model: "claude-3-5-sonnet".into(),
|
||||||
|
messages: vec![ChatMessage::user("hello")],
|
||||||
|
temperature: None,
|
||||||
|
max_tokens: None,
|
||||||
|
stream: false,
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
|
};
|
||||||
|
let body = provider.convert_request(req);
|
||||||
|
let user_msg = body
|
||||||
|
.messages
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.get("role").and_then(|r| r.as_str()) == Some("user"))
|
||||||
|
.expect("应有 user 消息");
|
||||||
|
// 纯文本 → 字符串简写(非数组)
|
||||||
|
assert_eq!(user_msg.get("content").and_then(|c| c.as_str()), Some("hello"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ use crate::provider::{
|
|||||||
CompletionRequest, CompletionResponse, LlmProvider, StreamChunk, StreamResult,
|
CompletionRequest, CompletionResponse, LlmProvider, StreamChunk, StreamResult,
|
||||||
TokenUsage, ToolCall, ToolCallDelta,
|
TokenUsage, ToolCall, ToolCallDelta,
|
||||||
};
|
};
|
||||||
|
// ChatMessage 仅单测构造 CompletionRequest 用,避免非 test 构建的 unused import 警告。
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::provider::ChatMessage;
|
||||||
use crate::retry::{
|
use crate::retry::{
|
||||||
retry_with_backoff, AttemptOutcome, is_reqwest_error_retryable, is_status_retryable,
|
retry_with_backoff, AttemptOutcome, is_reqwest_error_retryable, is_status_retryable,
|
||||||
};
|
};
|
||||||
@@ -44,10 +47,13 @@ struct OpenAiRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// OpenAI 消息格式
|
/// OpenAI 消息格式
|
||||||
|
///
|
||||||
|
/// `content` 为 `serde_json::Value`:纯文本消息走字符串简写(与老端点兼容),
|
||||||
|
/// 含图消息走 `[{type:"text",text},{type:"image_url",image_url:{url}}]` 数组。
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
struct OpenAiMessage {
|
struct OpenAiMessage {
|
||||||
role: String,
|
role: String,
|
||||||
content: String,
|
content: serde_json::Value,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
tool_call_id: Option<String>,
|
tool_call_id: Option<String>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
@@ -308,6 +314,38 @@ impl OpenAICompatProvider {
|
|||||||
crate::provider::MessageRole::Assistant => "assistant",
|
crate::provider::MessageRole::Assistant => "assistant",
|
||||||
crate::provider::MessageRole::Tool => "tool",
|
crate::provider::MessageRole::Tool => "tool",
|
||||||
};
|
};
|
||||||
|
// F-260614-05 Phase 2a: 多模态 content(须在 move m.tool_calls 之前算,借用 m)。
|
||||||
|
// 含图消息走 content 数组(text/image_url);纯文本走字符串简写
|
||||||
|
// (保持与现有纯文本端点零回归)。image_url 支持 data URI(base64)与 http(s) URL。
|
||||||
|
let content = if m.has_image() {
|
||||||
|
let parts: Vec<serde_json::Value> = m
|
||||||
|
.flattened_parts()
|
||||||
|
.into_iter()
|
||||||
|
.map(|p| match p {
|
||||||
|
crate::provider::ContentPart::Text { text } => serde_json::json!({
|
||||||
|
"type": "text",
|
||||||
|
"text": text,
|
||||||
|
}),
|
||||||
|
crate::provider::ContentPart::Image { url, base64, media_type, alt: _ } => {
|
||||||
|
let final_url = match (base64, url, media_type) {
|
||||||
|
(Some(b), _, Some(mt)) => {
|
||||||
|
format!("data:{};base64,{}", mt, b)
|
||||||
|
}
|
||||||
|
(None, Some(u), _) => u,
|
||||||
|
// 兜底:缺数据时退化为占位,避免发空 url 触发 400
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": { "url": final_url },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
serde_json::Value::Array(parts)
|
||||||
|
} else {
|
||||||
|
serde_json::Value::String(m.content.clone())
|
||||||
|
};
|
||||||
let tool_calls = m.tool_calls.map(|calls| {
|
let tool_calls = m.tool_calls.map(|calls| {
|
||||||
calls
|
calls
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -325,7 +363,7 @@ impl OpenAICompatProvider {
|
|||||||
});
|
});
|
||||||
OpenAiMessage {
|
OpenAiMessage {
|
||||||
role: role.to_string(),
|
role: role.to_string(),
|
||||||
content: m.content,
|
content,
|
||||||
tool_call_id: m.tool_call_id,
|
tool_call_id: m.tool_call_id,
|
||||||
tool_calls,
|
tool_calls,
|
||||||
}
|
}
|
||||||
@@ -691,4 +729,58 @@ mod tests {
|
|||||||
assert_eq!(tcs[0].function_arguments.as_deref(), Some("{\"q\":"));
|
assert_eq!(tcs[0].function_arguments.as_deref(), Some("{\"q\":"));
|
||||||
assert!(!c.finished);
|
assert!(!c.finished);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------- F-260614-05 Phase 2a 多模态 convert_request ----------
|
||||||
|
|
||||||
|
/// 含图消息 → content 数组(text + image_url data URI);纯文本 → 字符串简写
|
||||||
|
#[test]
|
||||||
|
fn openai_convert_multimodal_content() {
|
||||||
|
let provider = OpenAICompatProvider::new("https://api.openai.com", "k", "gpt-4o");
|
||||||
|
let req = CompletionRequest {
|
||||||
|
model: "gpt-4o".into(),
|
||||||
|
messages: vec![ChatMessage::user_parts(
|
||||||
|
"看图",
|
||||||
|
vec![
|
||||||
|
crate::provider::ContentPart::image_base64("image/png", "iVBOR"),
|
||||||
|
crate::provider::ContentPart::image_url("https://x/a.png"),
|
||||||
|
],
|
||||||
|
)],
|
||||||
|
temperature: None,
|
||||||
|
max_tokens: None,
|
||||||
|
stream: false,
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
|
};
|
||||||
|
let out = provider.convert_request(req);
|
||||||
|
let msg = &out.messages[0];
|
||||||
|
// content 是数组:[text "看图", image_url(data URI), image_url(http url)]
|
||||||
|
let arr = msg.content.as_array().expect("含图 → content 数组");
|
||||||
|
assert_eq!(arr.len(), 3);
|
||||||
|
assert_eq!(arr[0]["type"], "text");
|
||||||
|
assert_eq!(arr[0]["text"], "看图");
|
||||||
|
assert_eq!(arr[1]["type"], "image_url");
|
||||||
|
assert_eq!(
|
||||||
|
arr[1]["image_url"]["url"],
|
||||||
|
"data:image/png;base64,iVBOR"
|
||||||
|
);
|
||||||
|
assert_eq!(arr[2]["image_url"]["url"], "https://x/a.png");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 纯文本消息 → content 仍是字符串简写(无图不数组化,对齐纯文本端点兼容)
|
||||||
|
#[test]
|
||||||
|
fn openai_convert_text_only_remains_string() {
|
||||||
|
let provider = OpenAICompatProvider::new("https://api.openai.com", "k", "gpt-4o");
|
||||||
|
let req = CompletionRequest {
|
||||||
|
model: "gpt-4o".into(),
|
||||||
|
messages: vec![ChatMessage::user("hello")],
|
||||||
|
temperature: None,
|
||||||
|
max_tokens: None,
|
||||||
|
stream: false,
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
|
};
|
||||||
|
let out = provider.convert_request(req);
|
||||||
|
let msg = &out.messages[0];
|
||||||
|
assert_eq!(msg.content, serde_json::Value::String("hello".into()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -74,6 +74,58 @@ pub(crate) fn truncate_for_persist(content: &str) -> String {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 落库前对 `ChatMessage.parts` 做截断(F-260614-05 Phase 2a)。
|
||||||
|
///
|
||||||
|
/// - `Text` 片:复用 `truncate_for_persist` 头尾截断语义(单 Text 片 > 50KB 才截)。
|
||||||
|
/// - `Image` 片:base64 通常已 50KB+,落库前替换为占位 `Text` 片
|
||||||
|
/// `<image: base64 已省略, 共 N 字节>`,避免大体量图把对话 JSON 撑爆
|
||||||
|
/// (一张 1MB PNG 的 base64 ≈ 1.3MB 字符串)。原 Image 片仅在内存真相源(ContextManager)
|
||||||
|
/// 保留,重发时仍带图——对齐现有「持久化视图不污染内存真相源」约定。
|
||||||
|
/// - `Image` url 模式(无 base64):url 本身短,原样保留。
|
||||||
|
///
|
||||||
|
/// 返回 None 表示 parts 无需保留(全空或全部被替换且原 parts 仅含图);调用方据此清空 parts。
|
||||||
|
pub(crate) fn truncate_parts_for_persist(parts: &[df_ai::provider::ContentPart]) -> Option<Vec<df_ai::provider::ContentPart>> {
|
||||||
|
use df_ai::provider::ContentPart;
|
||||||
|
let mut out: Vec<ContentPart> = Vec::with_capacity(parts.len());
|
||||||
|
let mut changed = false;
|
||||||
|
for p in parts {
|
||||||
|
match p {
|
||||||
|
ContentPart::Text { text } => {
|
||||||
|
let truncated = truncate_for_persist(text);
|
||||||
|
if truncated.len() != text.len() {
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
out.push(ContentPart::Text { text: truncated });
|
||||||
|
}
|
||||||
|
ContentPart::Image { url, base64, media_type, alt } => {
|
||||||
|
// base64 模式:体量大,替换占位 Text 片
|
||||||
|
if let Some(b) = base64 {
|
||||||
|
let bytes_approx = b.len(); // base64 字符数 ≈ 字节数 * 4/3,粗估
|
||||||
|
let placeholder = format!("<image: base64 已省略, 共约 {} 字符>", bytes_approx);
|
||||||
|
out.push(ContentPart::Text { text: placeholder });
|
||||||
|
changed = true;
|
||||||
|
} else {
|
||||||
|
// url 模式:url 短,原样保留(含 media_type/alt)
|
||||||
|
out.push(ContentPart::Image {
|
||||||
|
url: url.clone(),
|
||||||
|
base64: None,
|
||||||
|
media_type: media_type.clone(),
|
||||||
|
alt: alt.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if out.is_empty() {
|
||||||
|
None
|
||||||
|
} else if changed {
|
||||||
|
Some(out)
|
||||||
|
} else {
|
||||||
|
// 无变化:返回克隆的原 parts(保持引用语义一致)
|
||||||
|
Some(parts.to_vec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// 保存对话到数据库(按 conv_id 写库,不受 active_conversation_id 切换影响)
|
/// 保存对话到数据库(按 conv_id 写库,不受 active_conversation_id 切换影响)
|
||||||
///
|
///
|
||||||
/// 写 messages + updated_at + 累加 token 用量 + 首次落库的 model;标题由 ensure_conversation_title 单独生成。
|
/// 写 messages + updated_at + 累加 token 用量 + 首次落库的 model;标题由 ensure_conversation_title 单独生成。
|
||||||
@@ -95,6 +147,11 @@ pub(crate) async fn save_conversation(
|
|||||||
let mut msgs = session.messages.all_messages_clone();
|
let mut msgs = session.messages.all_messages_clone();
|
||||||
for m in &mut msgs {
|
for m in &mut msgs {
|
||||||
m.content = truncate_for_persist(&m.content);
|
m.content = truncate_for_persist(&m.content);
|
||||||
|
// F-260614-05 Phase 2a: parts(Image base64) 同样截断(替换占位 Text 片),
|
||||||
|
// 防大体量图把对话 JSON 撑爆。仅作用于持久化副本,不污染内存真相源。
|
||||||
|
if let Some(parts) = m.parts.as_ref() {
|
||||||
|
m.parts = truncate_parts_for_persist(parts);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
(
|
(
|
||||||
serde_json::to_string(&msgs).unwrap_or_else(|_| "[]".to_string()),
|
serde_json::to_string(&msgs).unwrap_or_else(|_| "[]".to_string()),
|
||||||
@@ -269,4 +326,59 @@ mod tests {
|
|||||||
assert!(result.ends_with('中'));
|
assert!(result.ends_with('中'));
|
||||||
assert!(result.contains("已截断"));
|
assert!(result.contains("已截断"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------- F-260614-05 Phase 2a truncate_parts_for_persist ----------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_parts_image_base64_replaced_with_placeholder() {
|
||||||
|
use df_ai::provider::ContentPart;
|
||||||
|
let parts = vec![
|
||||||
|
ContentPart::text("前缀文本"),
|
||||||
|
ContentPart::image_base64("image/png", &"A".repeat(60_000)),
|
||||||
|
ContentPart::text("后缀"),
|
||||||
|
];
|
||||||
|
let out = truncate_parts_for_persist(&parts).expect("应有结果");
|
||||||
|
// Image base64 被替换为占位 Text 片
|
||||||
|
assert!(out.iter().all(|p| !matches!(p, ContentPart::Image { base64: Some(_), .. })));
|
||||||
|
let placeholder = out.iter().find_map(|p| match p {
|
||||||
|
ContentPart::Text { text } if text.contains("base64 已省略") => Some(text.clone()),
|
||||||
|
_ => None,
|
||||||
|
}).expect("应含占位 Text 片");
|
||||||
|
assert!(placeholder.contains("60000"));
|
||||||
|
// 非 base64 的 Text 片保留原值
|
||||||
|
assert!(out.iter().any(|p| matches!(p, ContentPart::Text { text } if text == "前缀文本")));
|
||||||
|
assert!(out.iter().any(|p| matches!(p, ContentPart::Text { text } if text == "后缀")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_parts_image_url_preserved() {
|
||||||
|
use df_ai::provider::ContentPart;
|
||||||
|
// url 模式:url 短,原样保留(含 media_type)
|
||||||
|
let parts = vec![ContentPart::Image {
|
||||||
|
url: Some("https://x/a.png".into()),
|
||||||
|
base64: None,
|
||||||
|
media_type: None,
|
||||||
|
alt: None,
|
||||||
|
}];
|
||||||
|
let out = truncate_parts_for_persist(&parts).expect("应有结果");
|
||||||
|
assert!(matches!(out[0], ContentPart::Image { ref url, .. } if url.as_deref() == Some("https://x/a.png")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_parts_long_text_part_truncated() {
|
||||||
|
use df_ai::provider::ContentPart;
|
||||||
|
// 单 Text 片超阈值 → 头尾截断
|
||||||
|
let long: String = "x".repeat(TRUNCATE_THRESHOLD + 1000);
|
||||||
|
let parts = vec![ContentPart::Text { text: long }];
|
||||||
|
let out = truncate_parts_for_persist(&parts).expect("应有结果");
|
||||||
|
match &out[0] {
|
||||||
|
ContentPart::Text { text } => assert!(text.contains("已截断"), "超长 Text 片应被截断"),
|
||||||
|
_ => panic!("应仍是 Text 片"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_parts_empty_returns_none() {
|
||||||
|
assert_eq!(truncate_parts_for_persist(&[]), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ pub(crate) async fn ensure_conversation_title(
|
|||||||
.map(|m| ChatMessage {
|
.map(|m| ChatMessage {
|
||||||
role: m.role.clone(),
|
role: m.role.clone(),
|
||||||
content: m.content.clone(),
|
content: m.content.clone(),
|
||||||
|
parts: None,
|
||||||
tool_call_id: None,
|
tool_call_id: None,
|
||||||
tool_calls: None,
|
tool_calls: None,
|
||||||
model: None,
|
model: None,
|
||||||
|
|||||||
Reference in New Issue
Block a user