mz/video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

音频文件上传 | AI创作模块兼容音频

Browse Source
This commit is contained in:
GinHa
2026-07-03 11:44:32 +08:00
parent 899732aa54
commit 28b1d24173
14 changed files with 209 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video-gen-admin/dist/assets/index-CoDurcdv.js → video-gen-admin/dist/assets/index-BhcMFup1.js
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
video-gen-admin/dist/index.html
Vendored
+1 -1
View File
@@ -28,7 +28,7 @@
} }
})(); })();
</script> </script>
<script type="module" crossorigin src="/assets/index-CoDurcdv.js"></script> <script type="module" crossorigin src="/assets/index-BhcMFup1.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css"> <link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css">
</head> </head>
<body> <body>
video-gen-admin/src/pages/AdminVideoEngines.tsx
+29 -1
View File
@@ -20,6 +20,7 @@ interface VideoEngine {
maxDuration: number; maxDuration: number;
maxImageCount: number; maxImageCount: number;
maxVideoCount: number; maxVideoCount: number;
maxAudioCount: number;
supportsFirstLastFrame: boolean; supportsFirstLastFrame: boolean;
supportsUniversalReference: boolean; supportsUniversalReference: boolean;
isActive: boolean; isActive: boolean;
@@ -74,6 +75,7 @@ const AdminVideoEngines: React.FC = () => {
max_duration: values.maxDuration ?? 15, max_duration: values.maxDuration ?? 15,
max_image_count: values.maxImageCount ?? 2, max_image_count: values.maxImageCount ?? 2,
max_video_count: values.maxVideoCount ?? 0, max_video_count: values.maxVideoCount ?? 0,
max_audio_count: values.maxAudioCount ?? 0,
supports_first_last_frame: values.supportsFirstLastFrame ?? false, supports_first_last_frame: values.supportsFirstLastFrame ?? false,
supports_universal_reference: values.supportsUniversalReference ?? true, supports_universal_reference: values.supportsUniversalReference ?? true,
is_active: values.isActive ?? true, is_active: values.isActive ?? true,
@@ -116,6 +118,7 @@ const AdminVideoEngines: React.FC = () => {
maxDuration: 15, maxDuration: 15,
maxImageCount: 2, maxImageCount: 2,
maxVideoCount: 0, maxVideoCount: 0,
maxAudioCount: 0,
supportsFirstLastFrame: false, supportsFirstLastFrame: false,
supportsUniversalReference: true, supportsUniversalReference: true,
supportedRatios: ['16:9', '4:3', '1:1', '3:4', '9:16', '21:9'], supportedRatios: ['16:9', '4:3', '1:1', '3:4', '9:16', '21:9'],
@@ -165,6 +168,10 @@ const AdminVideoEngines: React.FC = () => {
title: '最大视频', dataIndex: 'maxVideoCount', width: 100, title: '最大视频', dataIndex: 'maxVideoCount', width: 100,
render: (v: number) => <Tag color="cyan">{v} 个</Tag>, render: (v: number) => <Tag color="cyan">{v} 个</Tag>,
}, },
{
title: '最大音频', dataIndex: 'maxAudioCount', width: 100,
render: (v: number) => <Tag color={v > 0 ? 'geekblue' : 'default'}>{v || 0} 段</Tag>,
},
{ {
title: '首尾帧', dataIndex: 'supportsFirstLastFrame', width: 90, title: '首尾帧', dataIndex: 'supportsFirstLastFrame', width: 90,
render: (v: boolean) => <Tag color={v ? 'green' : 'default'}>{v ? '支持' : '不支持'}</Tag>, render: (v: boolean) => <Tag color={v ? 'green' : 'default'}>{v ? '支持' : '不支持'}</Tag>,
@@ -210,7 +217,7 @@ const AdminVideoEngines: React.FC = () => {
rowKey="id" rowKey="id"
loading={loading} loading={loading}
pagination={false} pagination={false}
scroll={{ x: 1100 }} scroll={{ x: 1200 }}
/> />
</Card> </Card>
@@ -276,6 +283,27 @@ const AdminVideoEngines: React.FC = () => {
<Input type="number" size="large" /> <Input type="number" size="large" />
</Form.Item> </Form.Item>
</div> </div>
<div style={{ display: 'flex', gap: 16 }}>
<Form.Item
name="maxAudioCount"
label="最大参考音频数"
style={{ flex: 1 }}
extra="0 表示不支持音频参考，最大 3 段"
rules={[
{
validator: (_, value) => {
const n = Number(value ?? 0);
if (!Number.isInteger(n) || n < 0 || n > 3) {
return Promise.reject(new Error('最大参考音频数必须为 0-3 的整数'));
}
return Promise.resolve();
},
},
]}
>
<Input type="number" size="large" min={0} max={3} />
</Form.Item>
</div>
<div style={{ display: 'flex', gap: 16 }}> <div style={{ display: 'flex', gap: 16 }}>
<Form.Item name="supportsFirstLastFrame" label="首尾帧模式" valuePropName="checked" style={{ paddingTop: 30, flex: 1 }}> <Form.Item name="supportsFirstLastFrame" label="首尾帧模式" valuePropName="checked" style={{ paddingTop: 30, flex: 1 }}>
<Switch checkedChildren="支持" unCheckedChildren="不支持" /> <Switch checkedChildren="支持" unCheckedChildren="不支持" />
video-gen-admin/src/types/index.ts
+10
View File
@@ -287,6 +287,11 @@ export interface GenerationAiVideoEngine {
supportedResolutions: string[]; supportedResolutions: string[];
supportedDurations: number[]; supportedDurations: number[];
maxDuration: number; maxDuration: number;
maxImageCount?: number;
maxVideoCount?: number;
maxAudioCount?: number;
supportsFirstLastFrame?: boolean;
supportsUniversalReference?: boolean;
priority: number; priority: number;
} }
@@ -309,6 +314,11 @@ export interface GenerationAiEngineOption {
supportedSizes?: Record<string, Record<string, string>>; supportedSizes?: Record<string, Record<string, string>>;
defaultSize?: string; defaultSize?: string;
maxDuration?: number; maxDuration?: number;
maxImageCount?: number;
maxVideoCount?: number;
maxAudioCount?: number;
supportsFirstLastFrame?: boolean;
supportsUniversalReference?: boolean;
priority: number; priority: number;
genType: GenerationAiGenType; genType: GenerationAiGenType;
} }
video-gen-api/alembic/versions/054735900c23_add_video_engine_max_audio_count.py
+29
View File
@@ -0,0 +1,29 @@
"""add video engine max audio count
Revision ID: 054735900c23
Revises: d4e5f6a7b8c9
Create Date: 2026-07-03 11:23:07.721724
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '054735900c23'
down_revision: Union[str, None] = 'd4e5f6a7b8c9'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('video_engines', sa.Column('max_audio_count', sa.Integer(), server_default='0', nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('video_engines', 'max_audio_count')
# ### end Alembic commands ###
video-gen-api/app/api/v1/generation.py
+43
View File
@@ -47,6 +47,11 @@ from app.services.generation_refund_service import mark_generation_record_failed
from app.services.media_token_usage_snapshot_service import sync_generation_record_media_token_snapshot from app.services.media_token_usage_snapshot_service import sync_generation_record_media_token_snapshot
from app.services.credit_record_meta_service import build_generation_record_prompt_meta from app.services.credit_record_meta_service import build_generation_record_prompt_meta
from app.services.video_cover_service import async_create_video_cover_for_local_video from app.services.video_cover_service import async_create_video_cover_for_local_video
from app.enums.audio_reference import (
AUDIO_ALLOWED_EXTENSIONS,
AUDIO_ALLOWED_MIME_TYPES,
AUDIO_MAX_FILE_SIZE_MB,
)
from app.utils.id_gen import generate_id from app.utils.id_gen import generate_id
from app.utils.exceptions import InsufficientCreditsError, RecordNotFoundError, InvalidStatusError from app.utils.exceptions import InsufficientCreditsError, RecordNotFoundError, InvalidStatusError
@@ -832,6 +837,44 @@ async def upload_video(
return {"url": url, "filename": file.filename or safe_name, "type": "video"} return {"url": url, "filename": file.filename or safe_name, "type": "video"}
@router.post("/upload-audio")
async def upload_audio(
file: UploadFile = File(...),
current_user: User = Depends(get_current_user),
):
"""Upload an audio file for AI creation reference."""
import os
import uuid
from app.config import settings
from datetime import datetime
ext = os.path.splitext(file.filename or "")[1].lower().lstrip(".")
if ext not in AUDIO_ALLOWED_EXTENSIONS:
raise HTTPException(status_code=400, detail="仅支持 mp3、wav 音频文件")
expected_mime = AUDIO_ALLOWED_MIME_TYPES.get(ext)
if not file.content_type or file.content_type != expected_mime:
raise HTTPException(status_code=400, detail=f"音频 MIME 类型错误，{ext} 必须为 {expected_mime}")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
safe_name = f"audio_ref_{current_user.id}_{timestamp}_{uuid.uuid4().hex[:8]}.{ext}"
date_dir = datetime.now().strftime("%Y/%m/%d")
dir_path = os.path.join(settings.UPLOAD_LOCAL_PATH, "audios", date_dir)
os.makedirs(dir_path, exist_ok=True)
file_path = os.path.join(dir_path, safe_name)
content = await file.read()
max_bytes = AUDIO_MAX_FILE_SIZE_MB * 1024 * 1024
if len(content) > max_bytes:
raise HTTPException(status_code=400, detail=f"音频大小不能超过{AUDIO_MAX_FILE_SIZE_MB}MB")
with open(file_path, "wb") as f:
f.write(content)
url = f"/uploads/audios/{date_dir}/{safe_name}"
return {"url": url, "filename": file.filename or safe_name, "type": "audio"}
@router.post("/delete-file") @router.post("/delete-file")
async def delete_upload( async def delete_upload(
url: str = Query(..., description="文件URL，如 /uploads/images/2024/01/01/video_img_xxx.png"), url: str = Query(..., description="文件URL，如 /uploads/images/2024/01/01/video_img_xxx.png"),
video-gen-api/app/api/v1/video_engines.py
+1
View File
@@ -51,6 +51,7 @@ async def list_active_engines(
"supported_durations": durations, "supported_durations": durations,
"max_image_count": e.max_image_count, "max_image_count": e.max_image_count,
"max_video_count": e.max_video_count, "max_video_count": e.max_video_count,
"max_audio_count": e.max_audio_count,
"supports_first_last_frame": e.supports_first_last_frame, "supports_first_last_frame": e.supports_first_last_frame,
"supports_universal_reference": e.supports_universal_reference, "supports_universal_reference": e.supports_universal_reference,
}) })
video-gen-api/app/enums/__init__.py
+1
View File
@@ -15,3 +15,4 @@ from app.enums.resource_capacity import *
from app.enums.team import * from app.enums.team import *
from app.enums.home_material import * from app.enums.home_material import *
from app.enums.celery_queue import * from app.enums.celery_queue import *
from app.enums.audio_reference import *
video-gen-api/app/enums/audio_reference.py
+33
View File
@@ -0,0 +1,33 @@
from enum import StrEnum
class AudioReferenceType(StrEnum):
"""参考素材类型。"""
audio = "audio"
class AudioReferenceFormat(StrEnum):
"""允许上传的参考音频格式。"""
mp3 = "mp3"
wav = "wav"
class AudioReferenceMimeType(StrEnum):
"""允许上传的参考音频 MIME。"""
mp3 = "audio/mpeg"
wav = "audio/wav"
AUDIO_ALLOWED_EXTENSIONS = {item.value for item in AudioReferenceFormat}
AUDIO_ALLOWED_MIME_TYPES = {
AudioReferenceFormat.mp3.value: AudioReferenceMimeType.mp3.value,
AudioReferenceFormat.wav.value: AudioReferenceMimeType.wav.value,
}
AUDIO_MAX_FILE_SIZE_MB = 15
AUDIO_MIN_DURATION_SECONDS = 2
AUDIO_MAX_DURATION_SECONDS = 15
AUDIO_MAX_TOTAL_DURATION_SECONDS = 15
AUDIO_MAX_COUNT_LIMIT = 3
video-gen-api/app/models/video_engine.py
+1
View File
@@ -19,6 +19,7 @@ class VideoEngine(Base, TimestampMixin):
max_duration: Mapped[int] = mapped_column(Integer, default=15) max_duration: Mapped[int] = mapped_column(Integer, default=15)
max_image_count: Mapped[int] = mapped_column(Integer, default=2) max_image_count: Mapped[int] = mapped_column(Integer, default=2)
max_video_count: Mapped[int] = mapped_column(Integer, default=0) max_video_count: Mapped[int] = mapped_column(Integer, default=0)
max_audio_count: Mapped[int] = mapped_column(Integer, default=0)
supports_first_last_frame: Mapped[bool] = mapped_column(Boolean, default=False) supports_first_last_frame: Mapped[bool] = mapped_column(Boolean, default=False)
supports_universal_reference: Mapped[bool] = mapped_column(Boolean, default=True) supports_universal_reference: Mapped[bool] = mapped_column(Boolean, default=True)
generate_url: Mapped[str | None] = mapped_column(String(512), nullable=True, default="") generate_url: Mapped[str | None] = mapped_column(String(512), nullable=True, default="")
video-gen-api/app/schemas/generation_ai.py
+6 -4
View File
@@ -22,12 +22,12 @@ class GenerationAIReference(BaseModel):
url: str = Field( url: str = Field(
..., ...,
description="参考素材地址，可以是图片地址或视频地址", description="参考素材地址，可以是图片地址、视频地址或音频地址",
examples=["https://example.com/reference.png"], examples=["https://example.com/reference.png"],
) )
type: str = Field( type: str = Field(
..., ...,
description="参考素材类型：image=图片，video=视频", description="参考素材类型：image=图片，video=视频，audio=音频",
examples=["image"], examples=["image"],
) )
name: str | None = Field( name: str | None = Field(
@@ -38,7 +38,7 @@ class GenerationAIReference(BaseModel):
duration: float | None = Field( duration: float | None = Field(
None, None,
ge=0, ge=0,
description="视频素材时长（秒）。type=video 时使用，用于视频素材计费和时长校验", description="参考素材时长（秒）。type=video/audio 时使用，用于视频/音频素材时长校验",
examples=[5.0], examples=[5.0],
) )
@@ -112,7 +112,7 @@ class GenerationAITaskCreate(BaseModel):
) )
media_references: list[GenerationAIReference] | None = Field( media_references: list[GenerationAIReference] | None = Field(
None, None,
description="参考素材列表。可以传图片/视频参考素材；为空表示不使用参考素材", description="参考素材列表。可以传图片/视频/音频参考素材；为空表示不使用参考素材",
) )
idempotency_key: str | None = Field( idempotency_key: str | None = Field(
None, None,
@@ -193,6 +193,7 @@ class GenerationAIVideoEngineOptionOut(BaseModel):
priority: int = Field(0, description="引擎优先级，数值越大越优先") priority: int = Field(0, description="引擎优先级，数值越大越优先")
max_image_count: int | None = Field(None, description="最大图片数量") max_image_count: int | None = Field(None, description="最大图片数量")
max_video_count: int | None = Field(None, description="最大视频数量") max_video_count: int | None = Field(None, description="最大视频数量")
max_audio_count: int | None = Field(None, description="最大参考音频数量，0 表示不支持音频参考")
supports_first_last_frame: bool = Field(False, description="是否支持首帧和最后一帧") supports_first_last_frame: bool = Field(False, description="是否支持首帧和最后一帧")
supports_universal_reference: bool = Field(False, description="是否支持通用参考") supports_universal_reference: bool = Field(False, description="是否支持通用参考")
@@ -248,6 +249,7 @@ class GenerationAIEngineOptionsOut(BaseModel):
"priority": 10, "priority": 10,
"max_image_count": 2, "max_image_count": 2,
"max_video_count": 0, "max_video_count": 0,
"max_audio_count": 0,
"supports_first_last_frame": False, "supports_first_last_frame": False,
"supports_universal_reference": False, "supports_universal_reference": False,
} }
video-gen-api/app/schemas/video_engine.py
+2
View File
@@ -15,6 +15,7 @@ class VideoEngineCreate(BaseModel):
max_duration: int = Field(default=15) max_duration: int = Field(default=15)
max_image_count: int = Field(default=2) max_image_count: int = Field(default=2)
max_video_count: int = Field(default=0) max_video_count: int = Field(default=0)
max_audio_count: int = Field(default=0, ge=0, le=3, description="最大参考音频数量，0 表示不支持音频参考")
supports_first_last_frame: bool = Field(default=False, description="是否支持首尾帧模式") supports_first_last_frame: bool = Field(default=False, description="是否支持首尾帧模式")
supports_universal_reference: bool = Field(default=True, description="是否支持全能参考模式") supports_universal_reference: bool = Field(default=True, description="是否支持全能参考模式")
generate_url: str = Field(default="", max_length=512) generate_url: str = Field(default="", max_length=512)
@@ -39,6 +40,7 @@ class VideoEnginePublic(BaseModel):
supported_durations: list[int] = [] supported_durations: list[int] = []
max_image_count: int = 2 max_image_count: int = 2
max_video_count: int = 0 max_video_count: int = 0
max_audio_count: int = 0
supports_first_last_frame: bool = False supports_first_last_frame: bool = False
supports_universal_reference: bool = True supports_universal_reference: bool = True
video-gen-api/app/services/generation_ai_service.py
+48 -1
View File
@@ -15,6 +15,13 @@ from app.models.project import Project
from app.models.image_engine import ImageEngine from app.models.image_engine import ImageEngine
from app.models.user import User from app.models.user import User
from app.models.video_engine import VideoEngine from app.models.video_engine import VideoEngine
from app.enums.audio_reference import (
AUDIO_ALLOWED_EXTENSIONS,
AUDIO_MAX_COUNT_LIMIT,
AUDIO_MAX_DURATION_SECONDS,
AUDIO_MAX_TOTAL_DURATION_SECONDS,
AUDIO_MIN_DURATION_SECONDS,
)
from app.enums.generation_history import ( from app.enums.generation_history import (
GenerationHistorySourceEnum, GenerationHistorySourceEnum,
get_generation_history_source_label, get_generation_history_source_label,
@@ -157,6 +164,7 @@ def _build_video_snapshot(engine: VideoEngine, ratio: str, resolution: str, dura
"supported_resolutions": _parse_list(engine.supported_resolutions, []), "supported_resolutions": _parse_list(engine.supported_resolutions, []),
"supported_durations": _parse_list(engine.supported_durations, []), "supported_durations": _parse_list(engine.supported_durations, []),
"max_duration": engine.max_duration, "max_duration": engine.max_duration,
"max_audio_count": engine.max_audio_count,
"selected_ratio": ratio, "selected_ratio": ratio,
"selected_resolution": resolution, "selected_resolution": resolution,
"selected_duration": duration, "selected_duration": duration,
@@ -203,6 +211,7 @@ async def list_generation_ai_engine_options(db: AsyncSession) -> GenerationAIEng
priority=engine.priority or 0, priority=engine.priority or 0,
max_image_count=engine.max_image_count, max_image_count=engine.max_image_count,
max_video_count=engine.max_video_count, max_video_count=engine.max_video_count,
max_audio_count=engine.max_audio_count,
supports_first_last_frame=engine.supports_first_last_frame, supports_first_last_frame=engine.supports_first_last_frame,
supports_universal_reference=engine.supports_universal_reference, supports_universal_reference=engine.supports_universal_reference,
) )
@@ -244,6 +253,8 @@ async def create_async_generation_task(db: AsyncSession, current_user: User, req
await assert_user_resource_capacity_available(db, current_user.id) await assert_user_resource_capacity_available(db, current_user.id)
if gen_type == "image": if gen_type == "image":
if any((r.get("type") or "").lower() == "audio" for r in refs):
raise HTTPException(status_code=400, detail="图片生成不支持音频参考素材")
engine = await _get_image_engine(db, req.engine_id) engine = await _get_image_engine(db, req.engine_id)
sizes = _image_supported_sizes(engine) sizes = _image_supported_sizes(engine)
size = req.image_size or engine.default_size or IMAGE_DEFAULT_SIZE size = req.image_size or engine.default_size or IMAGE_DEFAULT_SIZE
@@ -306,7 +317,7 @@ async def create_async_generation_task(db: AsyncSession, current_user: User, req
input_video_duration = 0.0 input_video_duration = 0.0
if refs: if refs:
video_refs = [r for r in refs if r.get("type") == "video"] video_refs = [r for r in refs if (r.get("type") or "").lower() == "video"]
for ref in video_refs: for ref in video_refs:
ref_duration = float(ref.get("duration") or 0) ref_duration = float(ref.get("duration") or 0)
if ref_duration < 2: if ref_duration < 2:
@@ -315,6 +326,42 @@ async def create_async_generation_task(db: AsyncSession, current_user: User, req
if input_video_duration > 15: if input_video_duration > 15:
raise HTTPException(status_code=400, detail=f"所有视频素材总时长不能超过 15 秒，当前 {input_video_duration:.1f} 秒") raise HTTPException(status_code=400, detail=f"所有视频素材总时长不能超过 15 秒，当前 {input_video_duration:.1f} 秒")
audio_refs = [r for r in refs if (r.get("type") or "").lower() == "audio"]
if audio_refs:
max_audio_count = int(engine.max_audio_count or 0)
if max_audio_count <= 0:
raise HTTPException(status_code=400, detail="当前视频引擎不支持音频参考素材")
if max_audio_count > AUDIO_MAX_COUNT_LIMIT:
max_audio_count = AUDIO_MAX_COUNT_LIMIT
if len(audio_refs) > max_audio_count:
raise HTTPException(
status_code=400,
detail=f"参考音频最多可传 {max_audio_count} 段，当前 {len(audio_refs)} 段",
)
input_audio_duration = 0.0
for ref in audio_refs:
raw_duration = ref.get("duration")
if raw_duration is None:
raw_duration = 0.0
try:
ref_duration = float(raw_duration)
except (TypeError, ValueError):
ref_duration = 0.0
if ref_duration < AUDIO_MIN_DURATION_SECONDS or ref_duration > AUDIO_MAX_DURATION_SECONDS:
raise HTTPException(
status_code=400,
detail=f"单段参考音频时长必须在 {AUDIO_MIN_DURATION_SECONDS}-{AUDIO_MAX_DURATION_SECONDS} 秒之间",
)
input_audio_duration += ref_duration
if input_audio_duration > AUDIO_MAX_TOTAL_DURATION_SECONDS:
raise HTTPException(
status_code=400,
detail=f"所有参考音频总时长不能超过 {AUDIO_MAX_TOTAL_DURATION_SECONDS} 秒，当前 {input_audio_duration:.1f} 秒",
)
media_billing = await charge_generation_media_by_params( media_billing = await charge_generation_media_by_params(
db, db,
user_id=current_user.id, user_id=current_user.id,
video-gen-api/app/services/video_gen.py
+4
View File
@@ -135,6 +135,10 @@ async def submit_video_task(
resolved = _resolve_url(ref_url) resolved = _resolve_url(ref_url)
role = ref_role if ref_role else "reference_video" role = ref_role if ref_role else "reference_video"
content.append({"type": "video_url", "video_url": {"url": resolved}, "role": role}) content.append({"type": "video_url", "video_url": {"url": resolved}, "role": role})
elif ref_type == "audio" and ref_url:
resolved = _resolve_url(ref_url)
role = ref_role if ref_role else "reference_audio"
content.append({"type": "audio_url", "audio_url": {"url": resolved}, "role": role})
except (json.JSONDecodeError, TypeError): except (json.JSONDecodeError, TypeError):
pass pass