Merge branch 'main' of https://gitee.com/wg123/video-gen

video-gen-api/app/services/generation/prompt_service.py

@@ -36,7 +36,9 @@ def _load_refs(record: ChatGenerationTask) -> list[dict]:
         return []
 
 
-def _build_user_content(record: ChatGenerationTask) -> list[dict[str, Any]]:
+async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | None = None) -> list[dict[str, Any]]:
+    from app.utils.media import media_to_base64
+
     if record.gen_type == "image":
         params = f"图片参数：分辨率档位={record.image_size or '2K'}，比例={record.image_proportion or '1:1'}，像素={record.image_px or '2048x2048'}"
     else:
@@ -54,7 +56,15 @@ def _build_user_content(record: ChatGenerationTask) -> list[dict[str, Any]]:
         ref_url = ref.get("url") or ""
         if not ref_url:
             continue
-        url = _absolute_url(ref_url)
+        if db and await get_llm_media_as_base64(db):
+            if ref_type == "image":
+                url = await media_to_base64(ref_url, "image/png")
+            elif ref_type == "video":
+                url = await media_to_base64(ref_url, "video/mp4")
+            else:
+                continue
+        else:
+            url = _absolute_url(ref_url)
         if ref_type == "image":
             parts.append({"type": "image_url", "image_url": {"url": url}})
         elif ref_type == "video":
@@ -94,7 +104,7 @@ async def build_prompt_with_chatapi(db: AsyncSession, record: ChatGenerationTask
         "model": config.model_name,
         "messages": [
             {"role": "system", "content": system_prompt},
-            {"role": "user", "content": _build_user_content(record)},
+            {"role": "user", "content": await _build_user_content(record, db)},
         ],
         "max_tokens": config.max_tokens,
         "temperature": config.temperature,

video-gen-api/app/services/hot_opening_video_prompt_service.py

@@ -1536,9 +1536,16 @@ async def optimize_hot_opening_video_prompt(
     trace_id: str | None = None,
 ) -> tuple[dict[str, Any], str, dict[str, Any]]:
     duration = int(video_config["duration"])
+    from app.utils.media import media_to_base64, get_llm_media_as_base64
+    if await get_llm_media_as_base64(db):
+        video_url_final = await media_to_base64(material_video_url, "video/mp4")
+        image_url_final = await media_to_base64(generated_image_url, "image/png")
+    else:
+        video_url_final = _build_file_url_or_data_uri(material_video_url)
+        image_url_final = _build_file_url_or_data_uri(generated_image_url)
     references = [
-        {"type": "video", "url": _build_file_url_or_data_uri(material_video_url)},
-        {"type": "image", "url": _build_file_url_or_data_uri(generated_image_url)},
+        {"type": "video", "url": video_url_final},
+        {"type": "image", "url": image_url_final},
     ]
     client_schema = build_dynamic_schema(video_config, schema_config_snapshot)
     reference_video_fps = int(video_config.get("reference_video_fps") or DEFAULT_REFERENCE_VIDEO_FPS)

video-gen-api/app/services/llm.py

@@ -248,7 +248,63 @@ async def _call_openai_compatible(
             if ref_type == "video":
                 video_urls.append(ref_url)
 
-    def _build_file_url_or_data_uri(file_url: str, fallback_mime: str) -> str:
+    async def _build_multimodal_content(
+        user_content: str,
+        image_urls: list[str],
+        video_urls: list[str],
+    ) -> tuple[dict, dict | None]:
+        """构建多模态 user_message。返回 (actual_message, log_message)。"""
+        from app.utils.media import media_to_base64
+
+        content_parts = [{"type": "text", "text": user_content}]
+
+        from app.utils.media import get_llm_media_as_base64
+        as_base64 = await get_llm_media_as_base64(db)
+        for img in image_urls:
+            if as_base64:
+                url = await media_to_base64(img, "image/png")
+            else:
+                url = _file_url_or_data_uri(img, "image/png")
+            content_parts.append({
+                "type": "image_url",
+                "image_url": {"url": url},
+            })
+
+        for video in video_urls:
+            if as_base64:
+                url = await media_to_base64(video, "video/mp4")
+            else:
+                url = _file_url_or_data_uri(video, "video/mp4")
+            content_parts.append({
+                "type": "video_url",
+                "video_url": {"url": url},
+            })
+
+        user_message = {
+            "role": "user",
+            "content": content_parts,
+        }
+
+        # Log-friendly version: keep original paths instead of base64
+        log_content_parts = [{"type": "text", "text": user_content}]
+        for img in image_urls:
+            log_content_parts.append({
+                "type": "image_url",
+                "image_url": {"url": img},
+            })
+        for video in video_urls:
+            log_content_parts.append({
+                "type": "video_url",
+                "video_url": {"url": video},
+            })
+
+        log_user_message = {
+            "role": "user",
+            "content": log_content_parts,
+        }
+        return user_message, log_user_message
+
+    def _file_url_or_data_uri(file_url: str, fallback_mime: str) -> str:
         """
         Convert local upload path to base64 data URI.
         Keep remote http/https/data URLs as-is.
@@ -268,54 +324,9 @@ async def _call_openai_compatible(
         return f"data:{mime};base64,{b64}"
 
     if image_urls or video_urls:
-        content_parts = [{"type": "text", "text": user_content}]
-
-        for img in image_urls:
-            url = _build_file_url_or_data_uri(img, "image/png")
-            content_parts.append({
-                "type": "image_url",
-                "image_url": {
-                    "url": url,
-                },
-            })
-
-        for video in video_urls:
-            url = _build_file_url_or_data_uri(video, "video/mp4")
-            content_parts.append({
-                "type": "video_url",
-                "video_url": {
-                    "url": url,
-                },
-            })
-
-        user_message = {
-            "role": "user",
-            "content": content_parts,
-        }
-
-        # Log-friendly version: keep original paths instead of base64
-        log_content_parts = [{"type": "text", "text": user_content}]
-
-        for img in image_urls:
-            log_content_parts.append({
-                "type": "image_url",
-                "image_url": {
-                    "url": img,
-                },
-            })
-
-        for video in video_urls:
-            log_content_parts.append({
-                "type": "video_url",
-                "video_url": {
-                    "url": video,
-                },
-            })
-
-        log_user_message = {
-            "role": "user",
-            "content": log_content_parts,
-        }
+        user_message, log_user_message = await _build_multimodal_content(
+            user_content, image_urls, video_urls
+        )
     else:
         user_message = {
             "role": "user",

video-gen-api/app/services/shot_video_analysis_service.py

@@ -77,8 +77,12 @@ def build_file_url_or_data_uri(file_url: str, fallback_mime: str = "video/mp4")
     # return f"data:{mime};base64,{b64}"
 
 
-def build_user_message(user_text: str, video_url: str) -> tuple[dict[str, Any], dict[str, Any], str]:
-    real_url = build_file_url_or_data_uri(video_url)
+async def build_user_message(user_text: str, video_url: str, db=None) -> tuple[dict[str, Any], dict[str, Any], str]:
+    from app.utils.media import media_to_base64, get_llm_media_as_base64
+    if await get_llm_media_as_base64(db):
+        real_url = await media_to_base64(video_url, "video/mp4")
+    else:
+        real_url = build_file_url_or_data_uri(video_url)
     content_parts = [
         {
             "type": "video_url",
@@ -543,7 +547,7 @@ async def analyze_video_for_shot_split(
 
     system_prompt = build_video_analysis_system_prompt(mode=mode)
     user_text = build_video_analysis_user_text(mode=mode)
-    user_message, log_user_message, real_video_url = build_user_message(user_text, video_url)
+    user_message, log_user_message, real_video_url = await build_user_message(user_text, video_url, db)
 
     request_data: dict[str, Any] = {
         "model": config.model_name,