diff --git a/video-gen-api/app/services/llm.py b/video-gen-api/app/services/llm.py index 79878493..fa70a529 100644 --- a/video-gen-api/app/services/llm.py +++ b/video-gen-api/app/services/llm.py @@ -230,32 +230,97 @@ async def _call_openai_compatible( # Build multimodal user message content when images are present image_urls = [] + video_urls = [] if references: for ref in references: - if ref.get("type") == "image" and ref.get("url"): - image_urls.append(ref["url"]) + if not isinstance(ref, dict): + continue - if image_urls: + ref_type = ref.get("type") + ref_url = ref.get("url") + + if not ref_url: + continue + + if ref_type == "image": + image_urls.append(ref_url) + + if ref_type == "video": + video_urls.append(ref_url) + + def _build_file_url_or_data_uri(file_url: str, fallback_mime: str) -> str: + """ + Convert local upload path to base64 data URI. + Keep remote http/https/data URLs as-is. + """ + if file_url.startswith(("http://", "https://", "data:")): + return file_url + + # Compatible with /uploads/xxx and plain relative paths + relative_path = file_url.replace("/uploads/", "", 1).lstrip("/") + file_path = os.path.join(settings.UPLOAD_LOCAL_PATH, relative_path) + + mime = mimetypes.guess_type(file_path)[0] or fallback_mime + + with open(file_path, "rb") as f: + b64 = base64.b64encode(f.read()).decode() + + return f"data:{mime};base64,{b64}" + + if image_urls or video_urls: content_parts = [{"type": "text", "text": user_content}] + for img in image_urls: - if img.startswith("http"): - url = img - else: - # Local file: read and encode as base64 data URI - file_path = os.path.join(settings.UPLOAD_LOCAL_PATH, img.replace("/uploads/", "")) - mime = mimetypes.guess_type(file_path)[0] or "image/png" - with open(file_path, "rb") as f: - b64 = base64.b64encode(f.read()).decode() - url = f"data:{mime};base64,{b64}" - content_parts.append({"type": "image_url", "image_url": {"url": url}}) - user_message = {"role": "user", "content": content_parts} + url = _build_file_url_or_data_uri(img, "image/png") + content_parts.append({ + "type": "image_url", + "image_url": { + "url": url, + }, + }) + + for video in video_urls: + url = _build_file_url_or_data_uri(video, "video/mp4") + content_parts.append({ + "type": "video_url", + "video_url": { + "url": url, + }, + }) + + user_message = { + "role": "user", + "content": content_parts, + } + # Log-friendly version: keep original paths instead of base64 - log_user_message = {"role": "user", "content": [ - {"type": "text", "text": user_content}, - *[{"type": "image_url", "image_url": {"url": img}} for img in image_urls], - ]} + log_content_parts = [{"type": "text", "text": user_content}] + + for img in image_urls: + log_content_parts.append({ + "type": "image_url", + "image_url": { + "url": img, + }, + }) + + for video in video_urls: + log_content_parts.append({ + "type": "video_url", + "video_url": { + "url": video, + }, + }) + + log_user_message = { + "role": "user", + "content": log_content_parts, + } else: - user_message = {"role": "user", "content": user_content} + user_message = { + "role": "user", + "content": user_content, + } log_user_message = None async with httpx.AsyncClient(timeout=120) as client: