调整后台文字模型请求发送图片或者视频的方式base64，而不是现在的链接形式，增加后台配置

2026-07-16 13:36:37 +08:00
parent 38040dfb0d
commit 4d872036b2
10 changed files with 198 additions and 60 deletions
@@ -248,7 +248,63 @@ async def _call_openai_compatible(
            if ref_type == "video":
                video_urls.append(ref_url)

-    def _build_file_url_or_data_uri(file_url: str, fallback_mime: str) -> str:
+    async def _build_multimodal_content(
+        user_content: str,
+        image_urls: list[str],
+        video_urls: list[str],
+    ) -> tuple[dict, dict | None]:
+        """构建多模态 user_message。返回 (actual_message, log_message)。"""
+        from app.utils.media import media_to_base64
+
+        content_parts = [{"type": "text", "text": user_content}]
+
+        from app.utils.media import get_llm_media_as_base64
+        as_base64 = await get_llm_media_as_base64(db)
+        for img in image_urls:
+            if as_base64:
+                url = await media_to_base64(img, "image/png")
+            else:
+                url = _file_url_or_data_uri(img, "image/png")
+            content_parts.append({
+                "type": "image_url",
+                "image_url": {"url": url},
+            })
+
+        for video in video_urls:
+            if as_base64:
+                url = await media_to_base64(video, "video/mp4")
+            else:
+                url = _file_url_or_data_uri(video, "video/mp4")
+            content_parts.append({
+                "type": "video_url",
+                "video_url": {"url": url},
+            })
+
+        user_message = {
+            "role": "user",
+            "content": content_parts,
+        }
+
+        # Log-friendly version: keep original paths instead of base64
+        log_content_parts = [{"type": "text", "text": user_content}]
+        for img in image_urls:
+            log_content_parts.append({
+                "type": "image_url",
+                "image_url": {"url": img},
+            })
+        for video in video_urls:
+            log_content_parts.append({
+                "type": "video_url",
+                "video_url": {"url": video},
+            })
+
+        log_user_message = {
+            "role": "user",
+            "content": log_content_parts,
+        }
+        return user_message, log_user_message
+
+    def _file_url_or_data_uri(file_url: str, fallback_mime: str) -> str:
        """
        Convert local upload path to base64 data URI.
        Keep remote http/https/data URLs as-is.
@@ -268,54 +324,9 @@ async def _call_openai_compatible(
        return f"data:{mime};base64,{b64}"

    if image_urls or video_urls:
-        content_parts = [{"type": "text", "text": user_content}]
-
-        for img in image_urls:
-            url = _build_file_url_or_data_uri(img, "image/png")
-            content_parts.append({
-                "type": "image_url",
-                "image_url": {
-                    "url": url,
-                },
-            })
-
-        for video in video_urls:
-            url = _build_file_url_or_data_uri(video, "video/mp4")
-            content_parts.append({
-                "type": "video_url",
-                "video_url": {
-                    "url": url,
-                },
-            })
-
-        user_message = {
-            "role": "user",
-            "content": content_parts,
-        }
-
-        # Log-friendly version: keep original paths instead of base64
-        log_content_parts = [{"type": "text", "text": user_content}]
-
-        for img in image_urls:
-            log_content_parts.append({
-                "type": "image_url",
-                "image_url": {
-                    "url": img,
-                },
-            })
-
-        for video in video_urls:
-            log_content_parts.append({
-                "type": "video_url",
-                "video_url": {
-                    "url": video,
-                },
-            })
-
-        log_user_message = {
-            "role": "user",
-            "content": log_content_parts,
-        }
+        user_message, log_user_message = await _build_multimodal_content(
+            user_content, image_urls, video_urls
+        )
    else:
        user_message = {
            "role": "user",