mz/video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

拆镜复刻开发完成

Browse Source
This commit is contained in:
GinHa
2026-06-11 17:54:40 +08:00
parent a6b6e5f822
commit 74266a126c
34 changed files with 7105 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video-gen-api/alembic/versions/dbb11c0b5a0a_add_shot_replicate_tables.py
+161
View File
@@ -0,0 +1,161 @@
"""add shot replicate tables
Revision ID: dbb11c0b5a0a
Revises: 9ac2212e1b8e
Create Date: 2026-06-11 14:59:26.871902
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = 'dbb11c0b5a0a'
down_revision: Union[str, None] = '9ac2212e1b8e'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('shot_replicate_task_sets',
sa.Column('id', sa.String(length=32), nullable=False),
sa.Column('user_id', sa.String(length=32), nullable=False),
sa.Column('title', sa.String(length=160), nullable=True),
sa.Column('video_url', sa.String(length=512), nullable=False),
sa.Column('video_path', sa.String(length=512), nullable=False),
sa.Column('video_duration_seconds', sa.Float(), nullable=False),
sa.Column('status', sa.String(length=32), nullable=False),
sa.Column('analysis_status', sa.String(length=32), nullable=False),
sa.Column('split_status', sa.String(length=32), nullable=False),
sa.Column('original_video_content', sa.Text(), nullable=True),
sa.Column('original_video_category', sa.String(length=160), nullable=True),
sa.Column('original_video_audience', sa.Text(), nullable=True),
sa.Column('ai_suggestion_json', sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), nullable=True),
sa.Column('analysis_raw_json', sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), nullable=True),
sa.Column('analysis_result_json', sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), nullable=True),
sa.Column('segment_count', sa.Integer(), nullable=False),
sa.Column('completed_segment_count', sa.Integer(), nullable=False),
sa.Column('failed_segment_count', sa.Integer(), nullable=False),
sa.Column('analysis_error_message', sa.Text(), nullable=True),
sa.Column('split_error_message', sa.Text(), nullable=True),
sa.Column('idempotency_key', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index('idx_shot_replicate_task_sets_analysis_status', 'shot_replicate_task_sets', ['analysis_status'], unique=False)
op.create_index('idx_shot_replicate_task_sets_split_status', 'shot_replicate_task_sets', ['split_status'], unique=False)
op.create_index('idx_shot_replicate_task_sets_status', 'shot_replicate_task_sets', ['status'], unique=False)
op.create_index('idx_shot_replicate_task_sets_user_created', 'shot_replicate_task_sets', ['user_id', 'created_at'], unique=False)
op.create_index(op.f('ix_shot_replicate_task_sets_analysis_status'), 'shot_replicate_task_sets', ['analysis_status'], unique=False)
op.create_index(op.f('ix_shot_replicate_task_sets_deleted_at'), 'shot_replicate_task_sets', ['deleted_at'], unique=False)
op.create_index(op.f('ix_shot_replicate_task_sets_idempotency_key'), 'shot_replicate_task_sets', ['idempotency_key'], unique=False)
op.create_index(op.f('ix_shot_replicate_task_sets_split_status'), 'shot_replicate_task_sets', ['split_status'], unique=False)
op.create_index(op.f('ix_shot_replicate_task_sets_status'), 'shot_replicate_task_sets', ['status'], unique=False)
op.create_index(op.f('ix_shot_replicate_task_sets_user_id'), 'shot_replicate_task_sets', ['user_id'], unique=False)
op.create_index('uq_shot_replicate_task_sets_user_idempotency', 'shot_replicate_task_sets', ['user_id', 'idempotency_key'], unique=True, postgresql_where=sa.text('deleted_at IS NULL AND idempotency_key IS NOT NULL'))
op.create_table('shot_replicate_segments',
sa.Column('id', sa.String(length=32), nullable=False),
sa.Column('task_set_id', sa.String(length=32), nullable=False),
sa.Column('user_id', sa.String(length=32), nullable=False),
sa.Column('segment_index', sa.Integer(), nullable=False),
sa.Column('source_mode', sa.String(length=32), nullable=False),
sa.Column('start_second', sa.Float(), nullable=False),
sa.Column('end_second', sa.Float(), nullable=False),
sa.Column('duration_seconds', sa.Float(), nullable=False),
sa.Column('time_node', sa.String(length=64), nullable=False),
sa.Column('split_status', sa.String(length=32), nullable=False),
sa.Column('analysis_status', sa.String(length=32), nullable=False),
sa.Column('replicate_status', sa.String(length=32), nullable=False),
sa.Column('segment_video_url', sa.String(length=512), nullable=True),
sa.Column('segment_video_path', sa.String(length=512), nullable=True),
sa.Column('original_video_content', sa.Text(), nullable=True),
sa.Column('original_video_category', sa.String(length=160), nullable=True),
sa.Column('original_video_audience', sa.Text(), nullable=True),
sa.Column('segment_content', sa.Text(), nullable=True),
sa.Column('segment_category', sa.String(length=160), nullable=True),
sa.Column('segment_audience', sa.Text(), nullable=True),
sa.Column('analysis_json', sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), nullable=True),
sa.Column('ai_suggestion_json', sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), nullable=True),
sa.Column('module_project_id', sa.String(length=32), nullable=True),
sa.Column('split_celery_task_id', sa.String(length=160), nullable=True),
sa.Column('split_enqueued_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('split_started_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('split_lease_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('split_next_retry_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('split_retry_count', sa.Integer(), nullable=False),
sa.Column('split_last_error', sa.Text(), nullable=True),
sa.Column('split_completed_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('analysis_error_message', sa.Text(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['task_set_id'], ['shot_replicate_task_sets.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index('idx_shot_replicate_segments_analysis_status', 'shot_replicate_segments', ['analysis_status'], unique=False)
op.create_index('idx_shot_replicate_segments_project', 'shot_replicate_segments', ['module_project_id'], unique=False)
op.create_index('idx_shot_replicate_segments_replicate_status', 'shot_replicate_segments', ['replicate_status'], unique=False)
op.create_index('idx_shot_replicate_segments_source_mode', 'shot_replicate_segments', ['source_mode'], unique=False)
op.create_index('idx_shot_replicate_segments_split_status', 'shot_replicate_segments', ['split_status'], unique=False)
op.create_index('idx_shot_replicate_segments_task_set', 'shot_replicate_segments', ['task_set_id', 'segment_index'], unique=False)
op.create_index('idx_shot_replicate_segments_user_created', 'shot_replicate_segments', ['user_id', 'created_at'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_analysis_status'), 'shot_replicate_segments', ['analysis_status'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_deleted_at'), 'shot_replicate_segments', ['deleted_at'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_module_project_id'), 'shot_replicate_segments', ['module_project_id'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_replicate_status'), 'shot_replicate_segments', ['replicate_status'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_segment_index'), 'shot_replicate_segments', ['segment_index'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_source_mode'), 'shot_replicate_segments', ['source_mode'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_split_celery_task_id'), 'shot_replicate_segments', ['split_celery_task_id'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_split_lease_until'), 'shot_replicate_segments', ['split_lease_until'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_split_next_retry_at'), 'shot_replicate_segments', ['split_next_retry_at'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_split_status'), 'shot_replicate_segments', ['split_status'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_task_set_id'), 'shot_replicate_segments', ['task_set_id'], unique=False)
op.create_index(op.f('ix_shot_replicate_segments_user_id'), 'shot_replicate_segments', ['user_id'], unique=False)
op.create_index('uq_shot_replicate_segments_task_set_index_active', 'shot_replicate_segments', ['task_set_id', 'segment_index'], unique=True, postgresql_where=sa.text('deleted_at IS NULL'))
op.drop_column('user_oauth_app', 'count')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('user_oauth_app', sa.Column('count', sa.BIGINT(), autoincrement=False, nullable=False, comment='应用最大可以授权多少个用户'))
op.drop_index('uq_shot_replicate_segments_task_set_index_active', table_name='shot_replicate_segments', postgresql_where=sa.text('deleted_at IS NULL'))
op.drop_index(op.f('ix_shot_replicate_segments_user_id'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_task_set_id'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_split_status'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_split_next_retry_at'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_split_lease_until'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_split_celery_task_id'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_source_mode'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_segment_index'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_replicate_status'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_module_project_id'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_deleted_at'), table_name='shot_replicate_segments')
op.drop_index(op.f('ix_shot_replicate_segments_analysis_status'), table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_user_created', table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_task_set', table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_split_status', table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_source_mode', table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_replicate_status', table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_project', table_name='shot_replicate_segments')
op.drop_index('idx_shot_replicate_segments_analysis_status', table_name='shot_replicate_segments')
op.drop_table('shot_replicate_segments')
op.drop_index('uq_shot_replicate_task_sets_user_idempotency', table_name='shot_replicate_task_sets', postgresql_where=sa.text('deleted_at IS NULL AND idempotency_key IS NOT NULL'))
op.drop_index(op.f('ix_shot_replicate_task_sets_user_id'), table_name='shot_replicate_task_sets')
op.drop_index(op.f('ix_shot_replicate_task_sets_status'), table_name='shot_replicate_task_sets')
op.drop_index(op.f('ix_shot_replicate_task_sets_split_status'), table_name='shot_replicate_task_sets')
op.drop_index(op.f('ix_shot_replicate_task_sets_idempotency_key'), table_name='shot_replicate_task_sets')
op.drop_index(op.f('ix_shot_replicate_task_sets_deleted_at'), table_name='shot_replicate_task_sets')
op.drop_index(op.f('ix_shot_replicate_task_sets_analysis_status'), table_name='shot_replicate_task_sets')
op.drop_index('idx_shot_replicate_task_sets_user_created', table_name='shot_replicate_task_sets')
op.drop_index('idx_shot_replicate_task_sets_status', table_name='shot_replicate_task_sets')
op.drop_index('idx_shot_replicate_task_sets_split_status', table_name='shot_replicate_task_sets')
op.drop_index('idx_shot_replicate_task_sets_analysis_status', table_name='shot_replicate_task_sets')
op.drop_table('shot_replicate_task_sets')
# ### end Alembic commands ###
video-gen-api/app/api/v1/__init__.py
+2
View File
@@ -16,6 +16,7 @@ from app.api.v1.video_engines import router as video_engines_router
from app.api.v1.image_engines import router as image_engines_router
from app.api.v1.generation_ai import router as generation_ai_router
from app.api.v1.hot_opening_replicate import router as hot_opening_replicate_router
from app.api.v1.shot_replicate import router as shot_replicate_router
from app.api.v1.test import router as test_router
from app.api.v1.user_oauth import router as user_oauth_router
from app.api.v1.user_oauth_app import router as user_oauth_app_router
@@ -37,6 +38,7 @@ api_router.include_router(video_engines_router)
api_router.include_router(image_engines_router)
api_router.include_router(generation_ai_router)
api_router.include_router(hot_opening_replicate_router)
api_router.include_router(shot_replicate_router)
api_router.include_router(test_router)
api_router.include_router(user_oauth_router)
api_router.include_router(user_oauth_app_router)
video-gen-api/app/api/v1/hot_opening_replicate.py
+145 -3
View File
@@ -1,10 +1,14 @@
from __future__ import annotations
from types import SimpleNamespace
from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query
from sqlalchemy import inspect as sa_inspect
from sqlalchemy.ext.asyncio import AsyncSession
from app.dependencies import get_current_user, get_db
from app.models.user import User
from app.enums.hot_opening_replicate import ModuleCodeEnum
from app.schemas.hot_opening_replicate import (
HotOpeningActionOut,
HotOpeningDeleteOut,
@@ -35,14 +39,94 @@ from app.services.hot_opening_replicate_service import (
update_hot_opening_material_input,
update_hot_opening_video_prompt_schema,
)
from app.services.module_generation_log_service import log_module_error
from app.tasks.celery_app import celery_app
MODULE = ModuleCodeEnum.HOT_OPENING_REPLICATE.value
def _safe_user_id(user: object | None) -> str | None:
"""从 ORM 对象中安全取用户ID，避免 rollback/commit 后访问过期属性触发 MissingGreenlet。"""
if user is None:
return None
try:
value = getattr(user, "__dict__", {}).get("id")
if value is not None:
return str(value)
except Exception:
pass
try:
identity = sa_inspect(user).identity
if identity:
return str(identity[0])
except Exception:
pass
return None
def _safe_user_is_admin(user: object | None) -> bool:
"""安全判断管理员身份；如果对象属性已过期，保守按普通用户处理。"""
if user is None:
return False
try:
data = getattr(user, "__dict__", {})
if "is_admin" in data:
return bool(data.get("is_admin"))
except Exception:
pass
return False
def _user_context(user: object | None) -> SimpleNamespace:
return SimpleNamespace(id=_safe_user_id(user), is_admin=_safe_user_is_admin(user))
router = APIRouter(
prefix="/hot-opening-replications",
tags=["hot-opening-replications"],
)
def _log_api_error(
*,
event_type: str,
current_user: User | None = None,
project_id: str | None = None,
step_id: str | None = None,
message: str | None = None,
exc: BaseException | None = None,
detail: dict | None = None,
) -> None:
log_module_error(
module=MODULE,
event_type=event_type,
project_id=project_id,
step_id=step_id,
user_id=_safe_user_id(current_user),
message=message,
detail=detail,
exc=exc,
)
def _log_api_exception_from_locals(exc: BaseException, local_values: dict, message: str) -> None:
current_user = local_values.get("current_user")
project_id = local_values.get("project_id_value") or local_values.get("project_id")
step_id = local_values.get("step_id_value") or local_values.get("step_id")
req = local_values.get("req")
detail = {"request": req.model_dump() if hasattr(req, "model_dump") else str(req) if req is not None else None}
_log_api_error(
event_type="API_REQUEST_FAILED",
current_user=current_user if isinstance(current_user, User) else None,
project_id=str(project_id) if project_id else None,
step_id=str(step_id) if step_id else None,
message=message,
detail=detail,
exc=exc,
)
async def _reload_project_detail(
db: AsyncSession,
current_user: User,
@@ -52,7 +136,7 @@ async def _reload_project_detail(
project = await _get_project_for_user(
db,
project_id=project_id,
user=current_user,
user=_user_context(current_user),
for_update=False,
populate_existing=True,
)
@@ -72,14 +156,33 @@ async def _mark_dispatch_failed_and_raise(
try:
await mark_hot_opening_step_dispatch_failed(
db,
current_user=current_user,
current_user=_user_context(current_user),
project_id=project_id,
step_id=step_id,
error_message=message,
)
await db.commit()
except Exception:
except Exception as exc:
await db.rollback()
_log_api_error(
event_type="CELERY_DISPATCH_MARK_FAILED",
current_user=current_user,
project_id=project_id,
step_id=step_id,
message="Celery 投递失败后标记步骤失败也失败",
detail={"dispatch_error": message},
exc=exc,
)
log_module_error(
module=MODULE,
event_type="CELERY_DISPATCH_FAILED",
project_id=project_id,
step_id=step_id,
user_id=_safe_user_id(current_user),
message=message,
detail={"reason": "celery_dispatch_failed"},
error=message,
)
raise HTTPException(status_code=503, detail=message)
@@ -118,6 +221,7 @@ async def create_task(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"创建爆款开头复刻项目失败: {exc}")
raise HTTPException(status_code=500, detail=f"创建爆款开头复刻项目失败: {exc}")
return await _reload_project_detail(db, current_user, project_id_value)
@@ -185,6 +289,7 @@ async def update_material(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"修改素材输入失败: {exc}")
raise HTTPException(status_code=500, detail=f"修改素材输入失败: {exc}")
return HotOpeningActionOut(
@@ -228,6 +333,7 @@ async def update_image_prompt(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"修改图片 AI 提词失败: {exc}")
raise HTTPException(status_code=500, detail=f"修改图片 AI 提词失败: {exc}")
return HotOpeningActionOut(
@@ -300,6 +406,14 @@ async def generate_image_prompt(
):
_ = req
if celery_app is None:
_log_api_error(
event_type="CELERY_DISABLED",
current_user=current_user,
project_id=project_id,
step_id=step_id,
message="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker",
detail={"api": "hot_opening_replicate"},
)
raise HTTPException(status_code=503, detail="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker")
try:
@@ -312,6 +426,7 @@ async def generate_image_prompt(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"图片提词任务创建失败: {exc}")
raise HTTPException(status_code=500, detail=f"图片提词任务创建失败: {exc}")
from app.tasks.hot_opening_replicate_tasks import start_image_prompt_optimize
@@ -354,6 +469,14 @@ async def generate_image(
db: AsyncSession = Depends(get_db),
):
if celery_app is None:
_log_api_error(
event_type="CELERY_DISABLED",
current_user=current_user,
project_id=project_id,
step_id=step_id,
message="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker",
detail={"api": "hot_opening_replicate"},
)
raise HTTPException(status_code=503, detail="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker")
try:
@@ -369,6 +492,7 @@ async def generate_image(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"图片生成任务创建失败: {exc}")
raise HTTPException(status_code=500, detail=f"图片生成任务创建失败: {exc}")
from app.tasks.generation_create_tasks import chatapi_create_generation_task
@@ -411,6 +535,14 @@ async def generate_video_prompt(
db: AsyncSession = Depends(get_db),
):
if celery_app is None:
_log_api_error(
event_type="CELERY_DISABLED",
current_user=current_user,
project_id=project_id,
step_id=step_id,
message="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker",
detail={"api": "hot_opening_replicate"},
)
raise HTTPException(status_code=503, detail="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker")
try:
@@ -423,6 +555,7 @@ async def generate_video_prompt(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"视频提词任务创建失败: {exc}")
raise HTTPException(status_code=500, detail=f"视频提词任务创建失败: {exc}")
from app.tasks.hot_opening_replicate_tasks import start_video_prompt_optimize
@@ -466,6 +599,14 @@ async def generate_video(
db: AsyncSession = Depends(get_db),
):
if celery_app is None:
_log_api_error(
event_type="CELERY_DISABLED",
current_user=current_user,
project_id=project_id,
step_id=step_id,
message="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker",
detail={"api": "hot_opening_replicate"},
)
raise HTTPException(status_code=503, detail="Celery未启用：请配置 REDIS_URL 或 CELERY_BROKER_URL 后启动 worker")
try:
@@ -481,6 +622,7 @@ async def generate_video(
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"视频生成任务创建失败: {exc}")
raise HTTPException(status_code=500, detail=f"视频生成任务创建失败: {exc}")
from app.tasks.generation_create_tasks import chatapi_create_generation_task
video-gen-api/app/api/v1/shot_replicate.py
+668
View File
@@ -0,0 +1,668 @@
from __future__ import annotations
from types import SimpleNamespace
from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query
from sqlalchemy import inspect as sa_inspect
from sqlalchemy.ext.asyncio import AsyncSession
from app.dependencies import get_current_user, get_db
from app.models.user import User
from app.enums.shot_replicate import ModuleCodeEnum
from app.schemas.shot_replicate import (
ShotReplicateActionOut,
ShotReplicateDeleteOut,
ShotReplicateGenerateImagePromptRequest,
ShotReplicateGenerateImageRequest,
ShotReplicateGenerateVideoPromptRequest,
ShotReplicateGenerateVideoRequest,
ShotReplicateImagePromptUpdateRequest,
ShotReplicateMaterialUpdateRequest,
ShotReplicateSpecOut,
ShotReplicateTaskDetailOut,
ShotReplicateVideoPromptSchemaUpdateRequest,
ShotSegmentDetailOut,
ShotSegmentListOut,
ShotSegmentReplicationCreateRequest,
ShotSplitByAIOut,
ShotSplitByAIRequest,
ShotSplitCustomOut,
ShotSplitCustomRequest,
ShotTaskSetCreate,
ShotTaskSetDetailOut,
ShotTaskSetListOut,
)
from app.services.shot_replicate_flow_service import (
_get_project_for_user,
create_shot_replicate_project_from_segment,
delete_shot_replicate_project,
generate_image_from_prompt,
generate_video_from_prompt,
mark_shot_replicate_step_dispatch_failed,
project_to_detail_out,
submit_image_prompt_optimize,
submit_video_prompt_optimize,
update_shot_replicate_image_prompt,
update_shot_replicate_material_input,
update_shot_replicate_video_prompt_schema,
)
from app.services.shot_replicate_taskset_service import (
create_custom_segment,
create_segments_by_ai,
create_task_set,
get_segment_for_user,
list_segments,
list_task_sets,
segment_detail,
task_set_detail,
)
from app.services.module_generation_log_service import log_module_error, log_module_event_file
from app.tasks.celery_app import celery_app
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
def _safe_user_id(user: object | None) -> str | None:
"""从 ORM 对象中安全取用户ID，避免 rollback/commit 后访问过期属性触发 MissingGreenlet。"""
if user is None:
return None
try:
value = getattr(user, "__dict__", {}).get("id")
if value is not None:
return str(value)
except Exception:
pass
try:
identity = sa_inspect(user).identity
if identity:
return str(identity[0])
except Exception:
pass
return None
def _safe_user_is_admin(user: object | None) -> bool:
"""安全判断管理员身份；如果对象属性已过期，保守按普通用户处理。"""
if user is None:
return False
try:
data = getattr(user, "__dict__", {})
if "is_admin" in data:
return bool(data.get("is_admin"))
except Exception:
pass
return False
def _user_context(user: object | None) -> SimpleNamespace:
return SimpleNamespace(id=_safe_user_id(user), is_admin=_safe_user_is_admin(user))
router = APIRouter(prefix="/shot-replications", tags=["shot-replications"])
def _log_api_error(
*,
event_type: str,
current_user: User | None = None,
project_id: str | None = None,
step_id: str | None = None,
message: str | None = None,
exc: BaseException | None = None,
detail: dict | None = None,
) -> None:
log_module_error(
module=MODULE,
event_type=event_type,
project_id=project_id,
step_id=step_id,
user_id=_safe_user_id(current_user),
message=message,
detail=detail,
exc=exc,
)
def _log_api_exception_from_locals(exc: BaseException, local_values: dict, message: str) -> None:
current_user = local_values.get("current_user")
project_id = local_values.get("project_id_value") or local_values.get("project_id") or local_values.get("task_set_id") or local_values.get("task_set_id_value")
step_id = local_values.get("step_id_value") or local_values.get("step_id") or local_values.get("segment_id") or local_values.get("segment_id_value")
req = local_values.get("req")
detail = {"api": local_values.get("__name__"), "request": req.model_dump() if hasattr(req, "model_dump") else str(req) if req is not None else None}
_log_api_error(
event_type="API_REQUEST_FAILED",
current_user=current_user if isinstance(current_user, User) else None,
project_id=str(project_id) if project_id else None,
step_id=str(step_id) if step_id else None,
message=message,
detail=detail,
exc=exc,
)
async def _reload_project_detail(db: AsyncSession, current_user: User, project_id: str) -> ShotReplicateTaskDetailOut:
project = await _get_project_for_user(
db,
project_id=project_id,
user=_user_context(current_user),
for_update=False,
populate_existing=True,
)
return await project_to_detail_out(db, project)
async def _mark_dispatch_failed_and_raise(
db: AsyncSession,
*,
current_user: User,
project_id: str,
step_id: str | None,
message: str,
) -> None:
if step_id:
try:
await mark_shot_replicate_step_dispatch_failed(
db,
current_user=_user_context(current_user),
project_id=project_id,
step_id=step_id,
error_message=message,
)
await db.commit()
except Exception as exc:
await db.rollback()
_log_api_error(
event_type="CELERY_DISPATCH_MARK_FAILED",
current_user=current_user,
project_id=project_id,
step_id=step_id,
message="Celery 投递失败后标记步骤失败也失败",
detail={"dispatch_error": message},
exc=exc,
)
log_module_error(
module=MODULE,
event_type="CELERY_DISPATCH_FAILED",
project_id=project_id,
step_id=step_id,
user_id=_safe_user_id(current_user),
message=message,
detail={"reason": "celery_dispatch_failed"},
error=message,
)
raise HTTPException(status_code=503, detail=message)
@router.get(
"/spec",
response_model=ShotReplicateSpecOut,
summary="查询拆镜复刻模块状态枚举和步骤 JSON 结构说明",
)
async def get_spec():
return ShotReplicateSpecOut()
@router.post(
"/task-sets",
response_model=ShotTaskSetDetailOut,
summary="创建拆镜总任务集并异步分析原视频",
)
async def create_shot_task_set(
req: ShotTaskSetCreate = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
task_set = await create_task_set(db, current_user=current_user, req=req)
task_set_id = task_set.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"创建拆镜总任务集失败: {exc}")
raise HTTPException(status_code=500, detail=f"创建拆镜总任务集失败: {exc}")
if celery_app:
try:
from app.tasks.shot_replicate_tasks import analyze_original_video
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
except Exception as exc:
# 分析任务投递失败时保留总任务，前端可稍后通过恢复/重试处理。
_log_api_error(
event_type="CELERY_DISPATCH_FAILED",
current_user=current_user,
project_id=task_set_id,
message=f"拆镜分析任务投递失败: {exc}",
detail={"task_set_id": task_set_id, "task": "analyze_original_video"},
exc=exc,
)
raise HTTPException(status_code=503, detail=f"拆镜分析任务投递失败: {exc}")
return await task_set_detail(db, current_user=_user_context(current_user), task_set_id=task_set_id)
@router.get(
"/task-sets",
response_model=ShotTaskSetListOut,
summary="查询拆镜总任务集列表",
)
async def list_shot_task_sets(
status: str | None = Query(None, description="总任务状态，见 ShotTaskSetStatusEnum"),
analysis_status: str | None = Query(None, description="分析状态，见 ShotAnalysisStatusEnum"),
split_status: str | None = Query(None, description="拆镜状态，见 ShotSplitStatusEnum"),
keyword: str | None = Query(None, description="标题/内容关键词"),
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await list_task_sets(
db,
current_user=current_user,
status=status,
analysis_status=analysis_status,
split_status=split_status,
keyword=keyword,
page=page,
page_size=page_size,
)
@router.get(
"/task-sets/{task_set_id}",
response_model=ShotTaskSetDetailOut,
summary="获取拆镜总任务集详情",
)
async def get_shot_task_set(
task_set_id: str = Path(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await task_set_detail(db, current_user=_user_context(current_user), task_set_id=task_set_id)
@router.post(
"/task-sets/{task_set_id}/split-by-ai",
response_model=ShotSplitByAIOut,
summary="按 AI 建议方案异步拆镜",
)
async def split_by_ai(
task_set_id: str = Path(...),
req: ShotSplitByAIRequest = Body(default_factory=ShotSplitByAIRequest),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
out = await create_segments_by_ai(db, current_user=current_user, task_set_id=task_set_id, req=req)
segment_ids = [item.id for item in out.segments]
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"按 AI 建议拆镜失败: {exc}")
raise HTTPException(status_code=500, detail=f"按 AI 建议拆镜失败: {exc}")
if celery_app:
from app.tasks.shot_replicate_tasks import split_one_segment
for segment_id in segment_ids:
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
return out
@router.post(
"/task-sets/{task_set_id}/split-custom",
response_model=ShotSplitCustomOut,
summary="按用户自定义开始/结束秒异步拆单条片段",
)
async def split_custom(
task_set_id: str = Path(...),
req: ShotSplitCustomRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
out = await create_custom_segment(db, current_user=current_user, task_set_id=task_set_id, req=req)
segment_id = out.segment.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"自定义拆镜失败: {exc}")
raise HTTPException(status_code=500, detail=f"自定义拆镜失败: {exc}")
if celery_app:
from app.tasks.shot_replicate_tasks import split_one_segment
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
return out
@router.get(
"/task-sets/{task_set_id}/segments",
response_model=ShotSegmentListOut,
summary="查询拆镜片段列表",
)
async def list_task_set_segments(
task_set_id: str = Path(...),
source_mode: str | None = Query(None, description="ai_suggestion/custom"),
split_status: str | None = Query(None, description="拆镜状态"),
analysis_status: str | None = Query(None, description="片段分析状态"),
replicate_status: str | None = Query(None, description="复刻状态"),
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await list_segments(
db,
current_user=current_user,
task_set_id=task_set_id,
source_mode=source_mode,
split_status=split_status,
analysis_status=analysis_status,
replicate_status=replicate_status,
page=page,
page_size=page_size,
)
@router.get(
"/segments/{segment_id}",
response_model=ShotSegmentDetailOut,
summary="获取拆镜片段详情",
)
async def get_segment(
segment_id: str = Path(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await segment_detail(db, current_user=current_user, segment_id=segment_id)
@router.post(
"/segments/{segment_id}/replication-projects",
response_model=ShotReplicateActionOut,
summary="将拆镜片段创建为拆镜复刻项目",
)
async def create_replication_project_from_segment(
segment_id: str = Path(...),
req: ShotSegmentReplicationCreateRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
segment = await get_segment_for_user(db, segment_id=segment_id, user=current_user, for_update=True)
project = await create_shot_replicate_project_from_segment(db, current_user=current_user, segment=segment, req=req)
project_id = project.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"创建拆镜复刻项目失败: {exc}")
raise HTTPException(status_code=500, detail=f"创建拆镜复刻项目失败: {exc}")
return ShotReplicateActionOut(
message="已从拆镜片段创建复刻项目，素材视频已锁定",
project_id=project_id,
step_id=None,
detail=await _reload_project_detail(db, current_user, project_id),
)
@router.get(
"/projects/{project_id}",
response_model=ShotReplicateTaskDetailOut,
summary="获取拆镜复刻项目详情",
)
async def get_project(
project_id: str = Path(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await _reload_project_detail(db, current_user, project_id)
@router.put(
"/projects/{project_id}/material",
response_model=ShotReplicateActionOut,
summary="修改拆镜复刻素材信息，素材视频不允许修改",
)
async def update_material(
project_id: str = Path(...),
req: ShotReplicateMaterialUpdateRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project_id_value, step_id_value = await update_shot_replicate_material_input(db, current_user=current_user, project_id=project_id, req=req)
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"修改素材输入失败: {exc}")
raise HTTPException(status_code=500, detail=f"修改素材输入失败: {exc}")
return ShotReplicateActionOut(message="素材输入已修改，素材视频保持锁定", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.put(
"/projects/{project_id}/steps/{step_id}/image-prompt",
response_model=ShotReplicateActionOut,
summary="直接修改图片 AI 优化提词",
)
async def update_image_prompt(
project_id: str = Path(...),
step_id: str = Path(...),
req: ShotReplicateImagePromptUpdateRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project, step = await update_shot_replicate_image_prompt(db, current_user=current_user, project_id=project_id, step_id=step_id, req=req)
project_id_value, step_id_value = project.id, step.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"修改图片 AI 提词失败: {exc}")
raise HTTPException(status_code=500, detail=f"修改图片 AI 提词失败: {exc}")
return ShotReplicateActionOut(message="图片 AI 提词已修改，后续步骤已软删除", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.put(
"/projects/{project_id}/steps/{step_id}/video-prompt-schema",
response_model=ShotReplicateActionOut,
summary="修改视频 AI 提词 JSON schema",
)
async def update_video_prompt_schema(
project_id: str = Path(...),
step_id: str = Path(...),
req: ShotReplicateVideoPromptSchemaUpdateRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project, step = await update_shot_replicate_video_prompt_schema(db, current_user=current_user, project_id=project_id, step_id=step_id, req=req)
project_id_value, step_id_value = project.id, step.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"修改视频 AI 提词失败: {exc}")
raise HTTPException(status_code=500, detail=f"修改视频 AI 提词失败: {exc}")
return ShotReplicateActionOut(message="视频 AI 提词 schema 已修改，第5步视频生成已软删除", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.post(
"/projects/{project_id}/generate-image-prompt",
response_model=ShotReplicateActionOut,
summary="生成图片 AI 提词",
)
async def generate_image_prompt(
project_id: str = Path(...),
req: ShotReplicateGenerateImagePromptRequest = Body(default_factory=ShotReplicateGenerateImagePromptRequest),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project, step = await submit_image_prompt_optimize(db, current_user=current_user, project_id=project_id, req=req)
project_id_value, step_id_value = project.id, step.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"提交图片 AI 提词失败: {exc}")
raise HTTPException(status_code=500, detail=f"提交图片 AI 提词失败: {exc}")
try:
if celery_app:
from app.tasks.shot_replicate_flow_tasks import start_image_prompt_optimize
start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0)
except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片 AI 提词任务投递失败: {exc}")
return ShotReplicateActionOut(message="图片 AI 提词任务已提交", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.post(
"/projects/{project_id}/generate-image",
response_model=ShotReplicateActionOut,
summary="根据图片 AI 提词生成图片",
)
async def generate_image(
project_id: str = Path(...),
req: ShotReplicateGenerateImageRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project, step, chat_task = await generate_image_from_prompt(db, current_user=current_user, project_id=project_id, req=req)
project_id_value, step_id_value, chat_task_id_value = project.id, step.id, chat_task.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"提交图片生成失败: {exc}")
raise HTTPException(status_code=500, detail=f"提交图片生成失败: {exc}")
try:
if celery_app:
from app.tasks.generation_create_tasks import chatapi_create_generation_task
chatapi_create_generation_task.apply_async(args=[chat_task_id_value], queue="gen_chatapi_create", countdown=0)
except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片生成任务投递失败: {exc}")
return ShotReplicateActionOut(message="图片生成任务已提交", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.post(
"/projects/{project_id}/generate-video-prompt",
response_model=ShotReplicateActionOut,
summary="生成视频 AI 提词 JSON schema",
)
async def generate_video_prompt(
project_id: str = Path(...),
req: ShotReplicateGenerateVideoPromptRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project, step = await submit_video_prompt_optimize(db, current_user=current_user, project_id=project_id, req=req)
project_id_value, step_id_value = project.id, step.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"提交视频 AI 提词失败: {exc}")
raise HTTPException(status_code=500, detail=f"提交视频 AI 提词失败: {exc}")
try:
if celery_app:
from app.tasks.shot_replicate_flow_tasks import start_video_prompt_optimize
start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0)
except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频 AI 提词任务投递失败: {exc}")
return ShotReplicateActionOut(message="视频 AI 提词任务已提交", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.post(
"/projects/{project_id}/generate-video",
response_model=ShotReplicateActionOut,
summary="根据视频 AI 提词生成视频",
)
async def generate_video(
project_id: str = Path(...),
req: ShotReplicateGenerateVideoRequest = Body(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
project, step, chat_task = await generate_video_from_prompt(db, current_user=current_user, project_id=project_id, req=req)
project_id_value, step_id_value, chat_task_id_value = project.id, step.id, chat_task.id
await db.commit()
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"提交视频生成失败: {exc}")
raise HTTPException(status_code=500, detail=f"提交视频生成失败: {exc}")
try:
if celery_app:
from app.tasks.generation_create_tasks import chatapi_create_generation_task
chatapi_create_generation_task.apply_async(args=[chat_task_id_value], queue="gen_chatapi_create", countdown=0)
except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频生成任务投递失败: {exc}")
return ShotReplicateActionOut(message="视频生成任务已提交", project_id=project_id_value, step_id=step_id_value, detail=await _reload_project_detail(db, current_user, project_id_value))
@router.delete(
"/projects/{project_id}",
response_model=ShotReplicateDeleteOut,
summary="软删除拆镜复刻项目",
)
async def delete_project(
project_id: str = Path(...),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
out = await delete_shot_replicate_project(db, current_user=current_user, project_id=project_id)
await db.commit()
return out
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
_log_api_exception_from_locals(exc, locals(), f"删除拆镜复刻项目失败: {exc}")
raise HTTPException(status_code=500, detail=f"删除拆镜复刻项目失败: {exc}")
video-gen-api/app/config.py
+56
View File
@@ -87,6 +87,13 @@ class Settings(BaseSettings):
# ChatAPI async generation pipeline settings
CELERY_BROKER_URL: str = ""
CELERY_RESULT_BACKEND: str = ""
# Celery async 兼容配置。
# single_loop：每个 Celery 子进程一个专用 event loop，推荐线上/本地统一使用。
# direct：旧版线程本地 loop 降级模式，建议配合 CELERY_DB_USE_NULLPOOL=true。
CELERY_ASYNC_RUNNER_MODE: str = "single_loop"
CELERY_DB_USE_NULLPOOL: bool = False
CELERY_STARTUP_RECOVERY_ENABLED: bool = True
CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180
CHATAPI_VIDEO_FPS: float = 0.5
CHATAPI_ASYNC_MAX_RETRIES: int = 3
@@ -123,6 +130,22 @@ class Settings(BaseSettings):
DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active"
DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index"
# Celery 生成链路 / provider poll 容灾配置。
# 说明：
# - 不新增 Celery worker；恢复任务仍投递到 gen_result_download。
# - worker_ready 每个 worker 都会尝试抢启动恢复锁，只有抢到锁的 worker 投递恢复任务。
# - poll active 使用独立 Redis key，避免影响稳定的下载 active 注册表。
GENERATION_RECOVERY_BATCH_SIZE: int = 100
GENERATION_RECOVERY_MAX_ROUNDS: int = 5
POLL_RECOVERY_BATCH_SIZE: int = 100
POLL_TASK_LEASE_SECONDS: int = 5 * 60
POLL_TASK_QUEUE_TIMEOUT_SECONDS: int = 2 * 60
POLL_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:poll:active"
POLL_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:poll:active_index"
CELERY_STARTUP_RECOVERY_LOCK_KEY: str = "vg:celery:startup_recovery_lock"
CELERY_STARTUP_RECOVERY_LOCK_TTL_SECONDS: int = 120
RESOURCE_SIGN_SECRET: str = "resource-signature-secret-key-for-API-authentication"
RESOURCE_SIGN_EXPIRE_SECONDS: int = 60
RESOURCE_SIGN_ARG_EXPIRE: str = "exp"
@@ -134,5 +157,38 @@ class Settings(BaseSettings):
HOT_OPENING_DEFAULT_VIDEO_RESOLUTION: str = "480p"
HOT_OPENING_DEFAULT_TARGET_PLATFORM: str = "抖音"
# 拆镜复刻配置。
# 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。
SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 180
SHOT_ANALYSIS_TEMPERATURE: float = 0.1
SHOT_ANALYSIS_MAX_TOKENS: int = 5000
SHOT_ANALYSIS_VIDEO_FPS: float = 1.0
SHOT_ANALYSIS_MAX_LOCAL_VIDEO_MB: int = 45
SHOT_SEGMENT_LOCAL_PATH: str = "./storage/uploads/shot_segments"
SHOT_SEGMENT_URL_PREFIX: str = "/uploads/shot_segments"
SHOT_SPLIT_MIN_SECONDS: float = 2
SHOT_SPLIT_MAX_SECONDS: float = 15
SHOT_SPLIT_END_TOLERANCE_SECONDS: float = 0.5
SHOT_DURATION_TOLERANCE_SECONDS: float = 1.0
SHOT_FFMPEG_TIMEOUT_SECONDS: int = 120
SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20
FFPROBE_BIN: str = ""
# 继续复用 gen_result_download 队列，但限制 ffmpeg 并发，避免拖慢 Chat 下载。
SHOT_SPLIT_MAX_CONCURRENT: int = 1
SHOT_SPLIT_MAX_RETRY_COUNT: int = 3
SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30
SHOT_SPLIT_LEASE_SECONDS: int = 10 * 60
SHOT_SPLIT_PENDING_TIMEOUT_SECONDS: int = 5 * 60
SHOT_SPLIT_RECOVERY_BATCH_SIZE: int = 50
SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock"
SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore"
SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4
SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16"
SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p"
SHOT_REPLICATE_DEFAULT_TARGET_PLATFORM: str = "抖音"
settings = Settings()
video-gen-api/app/enums/__init__.py
+1
View File
@@ -1,3 +1,4 @@
from app.enums.common import *
from app.enums.hot_opening_replicate import *
from app.enums.video_prompt_schema import *
from app.enums.shot_replicate import *
video-gen-api/app/enums/shot_replicate.py
+92
View File
@@ -0,0 +1,92 @@
from __future__ import annotations
from enum import StrEnum
class ModuleCodeEnum(StrEnum):
"""拆镜复刻模块编码。"""
SHOT_REPLICATE = "shot_replicate"
class ShotReplicateStepCodeEnum(StrEnum):
"""拆镜复刻复用通用模块步骤编码。"""
MATERIAL_INPUT = "material_input"
IMAGE_PROMPT_OPTIMIZE = "image_prompt_optimize"
IMAGE_GENERATE = "image_generate"
VIDEO_PROMPT_OPTIMIZE = "video_prompt_optimize"
VIDEO_GENERATE = "video_generate"
class ShotReplicateGenerationModeEnum(StrEnum):
"""复用 ChatGenerationTask 时使用的 generation_mode。"""
SHOT_REPLICATE = "shot_replicate"
class ShotReplicateStepIOSchemaVersionEnum(StrEnum):
"""拆镜复刻子任务 input_json/output_json 结构版本。"""
V1 = "shot_replicate_step_io_v1"
class ShotTaskSetStatusEnum(StrEnum):
"""拆镜总任务集状态。"""
PENDING_ANALYSIS = "pending_analysis"
ANALYZING = "analyzing"
ANALYSIS_COMPLETED = "analysis_completed"
ANALYSIS_FAILED = "analysis_failed"
SPLITTING = "splitting"
SPLIT_COMPLETED = "split_completed"
PARTIAL_FAILED = "partial_failed"
FAILED = "failed"
DELETED = "deleted"
class ShotAnalysisStatusEnum(StrEnum):
"""原视频/片段视频分析状态。"""
PENDING = "pending"
PROCESSING = "processing"
COMPLETED = "completed"
FAILED = "failed"
class ShotSplitStatusEnum(StrEnum):
"""ffmpeg 拆镜状态。"""
NONE = "none"
PENDING = "pending"
PROCESSING = "processing"
COMPLETED = "completed"
FAILED = "failed"
RETRY_WAITING = "retry_waiting"
class ShotSegmentSourceModeEnum(StrEnum):
"""拆镜片段来源。"""
AI_SUGGESTION = "ai_suggestion"
CUSTOM = "custom"
class ShotSegmentAnalysisStatusEnum(StrEnum):
"""拆镜片段分析状态。"""
NOT_REQUIRED = "not_required"
PENDING = "pending"
PROCESSING = "processing"
COMPLETED = "completed"
FAILED = "failed"
class ShotSegmentReplicateStatusEnum(StrEnum):
"""拆镜片段进入复刻流程后的状态。"""
NOT_STARTED = "not_started"
PROJECT_CREATED = "project_created"
PROCESSING = "processing"
COMPLETED = "completed"
FAILED = "failed"
video-gen-api/app/models/__init__.py
+3
View File
@@ -23,6 +23,8 @@ from app.models.user_resource_month_stat import UserResourceMonthStat
from app.models.user_resource_total_stat import UserResourceTotalStat
from app.models.module_generation_project import ModuleGenerationProject
from app.models.module_generation_step import ModuleGenerationStep
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.user_oauth import UserOAuth
from app.models.user_oauth_account import UserOAuthAccount
from app.models.user_oauth_app import UserOAuthApp
@@ -37,5 +39,6 @@ __all__ = [
"ChatGenerationTask", "ChatGenerationTaskEvent", "ChatProviderCallLog",
"GeneratedResource", "UserResourceMonthStat", "UserResourceTotalStat",
"ModuleGenerationProject", "ModuleGenerationStep",
"ShotReplicateTaskSet", "ShotReplicateSegment",
"UserOAuth", "UserOAuthAccount", "UserOAuthApp",
]
video-gen-api/app/models/base.py
+12 -7
View File
@@ -4,6 +4,7 @@ import os
from sqlalchemy import DateTime, func
from sqlalchemy.ext.asyncio import AsyncAttrs, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.pool import NullPool
from app.config import settings
@@ -18,14 +19,18 @@ engine_kwargs = {
"pool_pre_ping": True,
}
# SQLite 本地调试时不要乱塞 pool_size/max_overflow，PostgreSQL/asyncpg 才建议配置
# SQLite 本地调试时不要乱塞 pool_size/max_overflow，PostgreSQL/asyncpg 才建议配置。
# 默认保持 Celery 连接池复用；只有显式开启 CELERY_DB_USE_NULLPOOL=true 时才降级 NullPool。
if _is_celery_process() and not settings.DATABASE_URL.startswith("sqlite"):
engine_kwargs.update(
pool_size=settings.CELERY_DB_POOL_SIZE,
max_overflow=settings.CELERY_DB_MAX_OVERFLOW,
pool_timeout=settings.CELERY_DB_POOL_TIMEOUT,
pool_recycle=settings.CELERY_DB_POOL_RECYCLE,
)
if bool(getattr(settings, "CELERY_DB_USE_NULLPOOL", False)):
engine_kwargs.update(poolclass=NullPool)
else:
engine_kwargs.update(
pool_size=settings.CELERY_DB_POOL_SIZE,
max_overflow=settings.CELERY_DB_MAX_OVERFLOW,
pool_timeout=settings.CELERY_DB_POOL_TIMEOUT,
pool_recycle=settings.CELERY_DB_POOL_RECYCLE,
)
engine = create_async_engine(settings.DATABASE_URL, **engine_kwargs)
video-gen-api/app/models/shot_replicate_segment.py
+80
View File
@@ -0,0 +1,80 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, JSON, String, Text, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, SoftDeleteMixin, TimestampMixin
_JSON_TYPE = JSON().with_variant(JSONB, "postgresql")
class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
"""拆镜复刻片段表。"""
__tablename__ = "shot_replicate_segments"
__table_args__ = (
Index("idx_shot_replicate_segments_task_set", "task_set_id", "segment_index"),
Index("idx_shot_replicate_segments_user_created", "user_id", "created_at"),
Index("idx_shot_replicate_segments_source_mode", "source_mode"),
Index("idx_shot_replicate_segments_split_status", "split_status"),
Index("idx_shot_replicate_segments_analysis_status", "analysis_status"),
Index("idx_shot_replicate_segments_replicate_status", "replicate_status"),
Index("idx_shot_replicate_segments_project", "module_project_id"),
Index(
"uq_shot_replicate_segments_task_set_index_active",
"task_set_id",
"segment_index",
unique=True,
postgresql_where=text("deleted_at IS NULL"),
),
)
id: Mapped[str] = mapped_column(String(32), primary_key=True)
task_set_id: Mapped[str] = mapped_column(
String(32), ForeignKey("shot_replicate_task_sets.id", ondelete="CASCADE"), index=True, nullable=False
)
user_id: Mapped[str] = mapped_column(
String(32), ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
segment_index: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
source_mode: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
start_second: Mapped[float] = mapped_column(Float, nullable=False)
end_second: Mapped[float] = mapped_column(Float, nullable=False)
duration_seconds: Mapped[float] = mapped_column(Float, nullable=False)
time_node: Mapped[str] = mapped_column(String(64), nullable=False)
split_status: Mapped[str] = mapped_column(String(32), default="pending", index=True, nullable=False)
analysis_status: Mapped[str] = mapped_column(String(32), default="pending", index=True, nullable=False)
replicate_status: Mapped[str] = mapped_column(String(32), default="not_started", index=True, nullable=False)
segment_video_url: Mapped[str | None] = mapped_column(String(512), nullable=True)
segment_video_path: Mapped[str | None] = mapped_column(String(512), nullable=True)
original_video_content: Mapped[str | None] = mapped_column(Text, nullable=True)
original_video_category: Mapped[str | None] = mapped_column(String(160), nullable=True)
original_video_audience: Mapped[str | None] = mapped_column(Text, nullable=True)
segment_content: Mapped[str | None] = mapped_column(Text, nullable=True)
segment_category: Mapped[str | None] = mapped_column(String(160), nullable=True)
segment_audience: Mapped[str | None] = mapped_column(Text, nullable=True)
analysis_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
ai_suggestion_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
module_project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
split_celery_task_id: Mapped[str | None] = mapped_column(String(160), nullable=True, index=True)
split_enqueued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
split_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
split_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
split_next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
split_retry_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
split_last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
split_completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
video-gen-api/app/models/shot_replicate_task_set.py
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
from typing import Any
from sqlalchemy import Float, ForeignKey, Index, Integer, JSON, String, Text, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, SoftDeleteMixin, TimestampMixin
_JSON_TYPE = JSON().with_variant(JSONB, "postgresql")
class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
"""拆镜复刻总任务集。"""
__tablename__ = "shot_replicate_task_sets"
__table_args__ = (
Index("idx_shot_replicate_task_sets_user_created", "user_id", "created_at"),
Index("idx_shot_replicate_task_sets_status", "status"),
Index("idx_shot_replicate_task_sets_analysis_status", "analysis_status"),
Index("idx_shot_replicate_task_sets_split_status", "split_status"),
Index(
"uq_shot_replicate_task_sets_user_idempotency",
"user_id",
"idempotency_key",
unique=True,
postgresql_where=text("deleted_at IS NULL AND idempotency_key IS NOT NULL"),
),
)
id: Mapped[str] = mapped_column(String(32), primary_key=True)
user_id: Mapped[str] = mapped_column(
String(32), ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
title: Mapped[str | None] = mapped_column(String(160), nullable=True)
video_url: Mapped[str] = mapped_column(String(512), nullable=False)
video_path: Mapped[str] = mapped_column(String(512), nullable=False)
video_duration_seconds: Mapped[float] = mapped_column(Float, nullable=False)
status: Mapped[str] = mapped_column(String(32), default="pending_analysis", index=True, nullable=False)
analysis_status: Mapped[str] = mapped_column(String(32), default="pending", index=True, nullable=False)
split_status: Mapped[str] = mapped_column(String(32), default="none", index=True, nullable=False)
original_video_content: Mapped[str | None] = mapped_column(Text, nullable=True)
original_video_category: Mapped[str | None] = mapped_column(String(160), nullable=True)
original_video_audience: Mapped[str | None] = mapped_column(Text, nullable=True)
ai_suggestion_json: Mapped[list[Any] | dict[str, Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
analysis_raw_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
analysis_result_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
completed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
failed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
split_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
idempotency_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
video-gen-api/app/schemas/shot_replicate.py
+734
View File
@@ -0,0 +1,734 @@
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from app.schemas.common import NaiveDatetimeOptional
SHOT_REPLICATE_PROJECT_STATUS_DESCRIPTIONS: dict[str, str] = {
"pending": "已创建但未进入流程",
"waiting_user": "等待用户手动触发下一步",
"processing": "当前有步骤处理中",
"completed": "总任务完成",
"failed": "总任务失败",
"cancelled": "总任务取消",
}
SHOT_REPLICATE_STEP_STATUS_DESCRIPTIONS: dict[str, str] = {
"pending": "子任务待处理",
"waiting_user": "等待用户确认或触发",
"processing": "子任务处理中",
"completed": "子任务完成",
"failed": "子任务失败",
"cancelled": "子任务取消",
}
SHOT_REPLICATE_STEP_DESCRIPTIONS: list[dict[str, Any]] = [
{"step_index": 1, "step_code": "material_input", "name": "素材输入"},
{"step_index": 2, "step_code": "image_prompt_optimize", "name": "图片 AI 提词"},
{"step_index": 3, "step_code": "image_generate", "name": "图片生成"},
{"step_index": 4, "step_code": "video_prompt_optimize", "name": "视频 AI 提词 JSON schema"},
{"step_index": 5, "step_code": "video_generate", "name": "视频生成"},
]
SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION = "shot_replicate_step_io_v1"
SHOT_REPLICATE_STEP_IO_EXAMPLES: dict[str, dict[str, Any]] = {
"material_input": {
"input_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "material_input",
"source": {"source_step_id": None, "parent_step_id": None},
"payload": {
"material_video_url": "https://example.com/source.mp4",
"material_image_url": "https://example.com/product.png",
"source_project_name": "参考素材项目名称",
"target_project_name": "新项目名称",
"core_content_point": "50字以内核心内容点",
},
"context": {},
},
"output_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "material_input",
"status": "completed",
"payload": {},
"result": {"accepted": True, "message": "素材输入已提交", "next_step_code": "image_prompt_optimize"},
"usage": {},
"error": {},
},
},
"image_prompt_optimize": {
"input_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "image_prompt_optimize",
"source": {"source_step_id": "第1步素材输入ID", "parent_step_id": "第1步素材输入ID"},
"payload": {"source_step_id": "第1步素材输入ID"},
"context": {},
},
"output_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "image_prompt_optimize",
"status": "completed",
"payload": {
"optimized_prompt": "图片生成提示词",
"prompt": "兼容字段，同 optimized_prompt",
"original_prompt": "后端拼接的图片提词原始需求",
"references": [{"type": "video|image", "url": "...", "name": "..."}],
},
"result": {},
"usage": {
"input_tokens": 0,
"output_tokens": 0,
"total_tokens": 0,
"text_credits_cost": 0,
"credit_biz_key": "module_generation_step:{step_id}:attempt:1:text_prompt:charge",
},
"error": {},
},
},
"image_generate": {
"input_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "image_generate",
"source": {"source_step_id": "第2步图片提词ID", "parent_step_id": "第2步图片提词ID"},
"payload": {
"engine_id": "图片引擎ID",
"params": {"image_size": "2K", "image_proportion": "1:1", "image_px": "2048x2048"},
"prompt": "图片生成提示词",
"media_references": [{"type": "image", "url": "新产品图片", "name": "新产品图片"}],
},
"context": {},
},
"output_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "image_generate",
"status": "completed",
"payload": {},
"result": {"result_image_url": "/generate/images/xxx.png", "chat_task_id": "ChatGenerationTask ID"},
"usage": {},
"error": {},
},
},
"video_prompt_optimize": {
"input_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "video_prompt_optimize",
"source": {"source_step_id": "第3步图片生成ID", "parent_step_id": "第3步图片生成ID"},
"payload": {
"source_step_id": "第3步图片生成ID",
"video_config": {"engine_id": "视频引擎ID", "duration": 8, "aspect_ratio": "9:16", "resolution": "1080p"},
"target_platform": "抖音",
},
"context": {},
},
"output_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "video_prompt_optimize",
"status": "completed",
"payload": {
"prompt_schema": {"任务基础信息": {}, "最终提示词": {}},
"final_prompt": "展示用最终视频提示词",
"params_used_for_prompt": {"duration": 8, "aspect_ratio": "9:16", "resolution": "1080p"},
"target_platform": "抖音",
},
"result": {},
"usage": {
"input_tokens": 0,
"output_tokens": 0,
"total_tokens": 0,
"text_credits_cost": 0,
"credit_biz_key": "module_generation_step:{step_id}:attempt:1:text_prompt:charge",
},
"error": {},
},
},
"video_generate": {
"input_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "video_generate",
"source": {"source_step_id": "第4步视频提词ID", "parent_step_id": "第4步视频提词ID"},
"payload": {
"engine_id": "视频引擎ID",
"params": {"duration": 8, "aspect_ratio": "9:16", "resolution": "1080p"},
"prompt_schema": {"任务基础信息": {}, "最终提示词": {}},
"final_prompt": "展示用最终提示词",
"media_references": [{"type": "image", "url": "第3步生成图片", "name": "新项目图片"}],
},
"context": {},
},
"output_json": {
"schema_version": SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION,
"step_code": "video_generate",
"status": "completed",
"payload": {},
"result": {
"result_video_url": "/generate/videos/xxx.mp4",
"result_video_cover_url": "/generate/covers/xxx.jpg",
"chat_task_id": "ChatGenerationTask ID",
},
"usage": {},
"error": {},
},
},
}
class ShotReplicateTaskCreate(BaseModel):
"""创建拆镜复刻总任务项目请求体。"""
model_config = ConfigDict(
json_schema_extra={
"example": {
"material_video_url": "https://example.com/source.mp4",
"material_image_url": "https://example.com/product.png",
"source_project_name": "参考素材项目名称",
"target_project_name": "新项目名称",
"core_content_point": "突出产品能帮助用户认识附近新朋友",
"idempotency_key": "frontend-submit-uuid-001",
}
}
)
material_video_url: str = Field(..., min_length=1, description="素材视频链接，参考素材，1份。由项目已有上传接口返回，本接口不负责上传，不做后端素材校验")
material_image_url: str = Field(..., min_length=1, description="素材图片链接，新产品图片，1份。由项目已有上传接口返回，本接口不负责上传，不做后端素材校验")
source_project_name: str = Field(..., min_length=1, max_length=20, description="视频素材内容项目名称")
target_project_name: str = Field(..., min_length=1, max_length=20, description="生成项目名称")
core_content_point: str = Field(..., min_length=1, max_length=50, description="生成的项目核心内容点，最多50字")
idempotency_key: str | None = Field(None, max_length=64, description="创建总任务幂等键。只用于 module_generation_projects，不用于 ChatGenerationTask")
@field_validator("material_video_url", "material_image_url", "source_project_name", "target_project_name", "core_content_point")
@classmethod
def _strip_required(cls, value: str) -> str:
value = str(value or "").strip()
if not value:
raise ValueError("字段不能为空")
return value
class ShotReplicateMaterialUpdateRequest(BaseModel):
"""修改拆镜复刻第1步素材输入请求体。"""
model_config = ConfigDict(
json_schema_extra={
"example": {
"material_video_url": "https://example.com/new-source.mp4",
"material_image_url": "https://example.com/new-product.png",
"source_project_name": "新的参考素材项目名称",
"target_project_name": "新的生成项目名称",
"core_content_point": "新的50字以内核心内容点",
}
}
)
material_video_url: str | None = Field(None, min_length=1, description="素材视频链接，未传则沿用旧值")
material_image_url: str | None = Field(None, min_length=1, description="素材图片链接，未传则沿用旧值")
source_project_name: str | None = Field(None, min_length=1, max_length=20, description="视频素材内容项目名称，未传则沿用旧值")
target_project_name: str | None = Field(None, min_length=1, max_length=20, description="生成项目名称，未传则沿用旧值")
core_content_point: str | None = Field(None, min_length=1, max_length=50, description="生成项目核心内容点，最多50字，未传则沿用旧值")
@field_validator("material_video_url", "material_image_url", "source_project_name", "target_project_name", "core_content_point", mode="before")
@classmethod
def _strip_optional(cls, value: str | None) -> str | None:
if value is None:
return None
value = str(value).strip()
if not value:
raise ValueError("字段不能为空字符串")
return value
@model_validator(mode="after")
def _require_at_least_one(self) -> "ShotReplicateMaterialUpdateRequest":
if not any(getattr(self, field) is not None for field in ("material_video_url", "material_image_url", "source_project_name", "target_project_name", "core_content_point")):
raise ValueError("至少需要传入一个需要修改的字段")
return self
class ShotReplicateStepUpdate(BaseModel):
"""修改拆镜复刻子任务请求体。"""
material_video_url: str | None = Field(None, description="修改第1步素材视频链接")
material_image_url: str | None = Field(None, description="修改第1步素材图片链接")
source_project_name: str | None = Field(None, max_length=20, description="修改第1步视频素材内容项目名称")
target_project_name: str | None = Field(None, max_length=20, description="修改第1步生成项目名称")
core_content_point: str | None = Field(None, max_length=50, description="修改第1步生成项目核心内容点，最多50字")
prompt: str | None = Field(None, description="修改第2步图片提词或第4步视频最终提词")
prompt_schema: dict[str, Any] | None = Field(None, description="修改第4步视频提词 JSON schema。只对视频提词步骤有意义")
input_json: dict[str, Any] | None = Field(None, description="高级用法：合并修改当前步骤 input_json.payload")
output_json: dict[str, Any] | None = Field(None, description="高级用法：合并修改当前步骤 output_json.payload")
class ShotReplicateImagePromptUpdateRequest(BaseModel):
"""直接修改第2步图片 AI 优化提词请求体。
本接口不调用 AI、不扣积分；保存后会软删除第3、4、5步当前有效任务。
"""
model_config = ConfigDict(
extra="ignore",
json_schema_extra={"example": {"prompt": "用户手动修改后的图片生成提示词"}},
)
prompt: str = Field(..., min_length=1, description="用户手动修改后的图片生成提示词，不能为空")
@field_validator("prompt", mode="before")
@classmethod
def _strip_prompt(cls, value: str) -> str:
value = str(value or "").strip()
if not value:
raise ValueError("图片提示词不能为空")
return value
class ShotReplicateVideoPromptSchemaUpdateRequest(BaseModel):
"""修改第4步视频 AI 提词 JSON schema 请求体。
前端提交的 prompt_schema 只作为 patch：服务端会锁定视频时长、比例、清晰度、帧率、推荐分辨率、
动作/镜头/动态时间规划数组长度和时间段、输出规格限制、质量控制、合规控制、schema_version、schema_usage。
最终提示词允许修改，但保存前会清洗视频参数。
"""
model_config = ConfigDict(
extra="ignore",
json_schema_extra={
"example": {
"prompt_schema": {
"业务属性": {"产品名称": "脱单交友APP", "行动引导": "立即下载"},
"最终提示词": {"主提示词": "脱单交友APP推广短视频，突出认识附近新朋友和高效匹配"},
}
}
},
)
prompt_schema: dict[str, Any] = Field(..., description="前端修改后的视频提词 JSON schema。后端只按白名单回填允许修改字段")
@field_validator("prompt_schema")
@classmethod
def _validate_schema(cls, value: dict[str, Any]) -> dict[str, Any]:
if not isinstance(value, dict) or not value:
raise ValueError("prompt_schema 必须是非空 JSON 对象")
return value
class ShotReplicateGenerateImagePromptRequest(BaseModel):
"""手动生成第2步图片 AI 提词请求体。当前无需请求参数。"""
model_config = ConfigDict(extra="ignore")
class ShotReplicateGenerateImageRequest(BaseModel):
"""根据图片提词生成新项目图片请求体。
ChatGenerationTask.idempotency_key 由后端自动生成，接口不再接收前端幂等键。
"""
model_config = ConfigDict(
extra="ignore",
json_schema_extra={"example": {"engine_id": "image_engine_xxx", "image_size": "2K", "image_proportion": "1:1", "image_px": "2048x2048"}},
)
engine_id: str | None = Field(None, description="图片生成引擎ID。为空则使用当前启用且优先级最高的图片引擎")
image_size: str | None = Field(None, description="图片分辨率档位，例如 1K、2K。为空使用引擎默认值")
image_proportion: str | None = Field(None, description="图片比例，例如 1:1、16:9、9:16。为空使用默认值")
image_px: str | None = Field(None, description="图片像素尺寸，例如 2048x2048。为空时按引擎支持尺寸自动匹配")
class ShotReplicateGenerateVideoPromptRequest(BaseModel):
"""手动生成第4步视频 AI 提词请求体。
视频时长、比例、分辨率集中在本步骤确定；第5步生成视频只选择视频引擎。
"""
model_config = ConfigDict(
extra="ignore",
json_schema_extra={"example": {"engine_id": "video_engine_xxx", "duration": 8, "aspect_ratio": "9:16", "resolution": "1080p", "target_platform": "抖音"}},
)
engine_id: str | None = Field(None, description="视频引擎ID。用于读取该引擎支持的视频时长、比例、分辨率配置；为空使用最高优先级启用引擎")
duration: int | None = Field(None, ge=1, description="希望用于视频提词规划的视频时长，单位秒。为空时优先使用 SHOT_REPLICATE_DEFAULT_VIDEO_DURATION")
aspect_ratio: str | None = Field(None, description="希望用于视频提词规划的视频比例。为空时优先使用 SHOT_REPLICATE_DEFAULT_VIDEO_RATIO")
resolution: str | None = Field(None, description="希望用于视频提词规划的视频分辨率。为空时优先使用 SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION")
target_platform: str | None = Field(None, max_length=64, description="目标平台，例如抖音/快手/小红书。为空时使用 SHOT_REPLICATE_DEFAULT_TARGET_PLATFORM")
class ShotReplicateGenerateVideoRequest(BaseModel):
"""根据视频提词生成最终视频请求体。
只选择视频生成引擎。duration / aspect_ratio / resolution 从第4步视频提词优化结果读取。
ChatGenerationTask.original_prompt / optimized_prompt 都写入第4步生成的 prompt_schema JSON 字符串。
"""
model_config = ConfigDict(extra="ignore", json_schema_extra={"example": {"engine_id": "video_engine_xxx"}})
engine_id: str | None = Field(None, description="视频生成引擎ID。为空优先使用第4步视频提词时选择的 engine_id，再为空使用最高优先级启用视频引擎")
class ShotReplicateStepOut(BaseModel):
id: str = Field(..., description="子任务ID")
project_id: str = Field(..., description="总任务项目ID，即 module_generation_projects.id")
module: str = Field(..., description="模块标识，例如 shot_replicate")
step_index: int = Field(..., description="步骤序号：1素材输入、2图片提词、3图片生成、4视频提词、5视频生成")
step_code: str = Field(..., description="步骤编码：material_input/image_prompt_optimize/image_generate/video_prompt_optimize/video_generate")
status: str = Field(..., description="步骤状态：pending/waiting_user/processing/completed/failed/cancelled")
version: int = Field(..., description="步骤版本号。重新生成或修改上游步骤后 version+1")
is_current: bool = Field(..., description="是否当前有效步骤。旧步骤会软删除且 is_current=false")
parent_step_id: str | None = Field(None, description="上一个步骤ID")
source_step_id: str | None = Field(None, description="当前步骤基于哪个上游步骤生成")
chat_task_id: str | None = Field(None, description="关联的 ChatGenerationTask ID。第3步图片生成、第5步视频生成有值")
input: dict[str, Any] | None = Field(None, description=f"步骤输入 JSON，统一 schema_version={SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION}")
output: dict[str, Any] | None = Field(None, description=f"步骤输出 JSON，统一 schema_version={SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION}")
error_message: str | None = Field(None, description="步骤错误信息")
created_at: NaiveDatetimeOptional = Field(None, description="创建时间")
updated_at: NaiveDatetimeOptional = Field(None, description="更新时间")
completed_at: NaiveDatetimeOptional = Field(None, description="完成时间")
class ShotReplicateMaterialOut(BaseModel):
material_step_id: str | None = Field(None, description="第1步素材输入子任务ID")
material_video_url: str | None = Field(None, description="素材视频链接")
material_image_url: str | None = Field(None, description="素材图片链接")
source_project_name: str | None = Field(None, description="视频素材内容项目名称")
target_project_name: str | None = Field(None, description="生成项目名称")
core_content_point: str | None = Field(None, description="生成项目核心内容点")
class ShotReplicateImageGenerationOut(BaseModel):
prompt_step_id: str | None = Field(None, description="第2步图片 AI 提词子任务ID")
generate_step_id: str | None = Field(None, description="第3步图片生成子任务ID")
prompt: str | None = Field(None, description="图片优化提词")
engine_id: str | None = Field(None, description="图片生成引擎ID")
engine_name: str | None = Field(None, description="图片生成引擎名称")
params: dict[str, Any] | None = Field(None, description="图片生成参数")
chat_task_id: str | None = Field(None, description="图片生成 ChatGenerationTask ID")
status: str | None = Field(None, description="图片生成状态")
result_image_url: str | None = Field(None, description="新项目图片 URL")
error_message: str | None = Field(None, description="图片生成错误信息")
class ShotReplicateVideoGenerationOut(BaseModel):
prompt_step_id: str | None = Field(None, description="第4步视频 AI 提词子任务ID")
generate_step_id: str | None = Field(None, description="第5步视频生成子任务ID")
prompt_schema: dict[str, Any] | None = Field(None, description="视频提词 JSON schema。第5步 ChatGenerationTask 原始提词会使用该 JSON 字符串")
final_prompt: str | None = Field(None, description="视频最终提词，仅用于前端展示")
prompt_params: dict[str, Any] | None = Field(None, description="第4步生成视频提词时使用的视频配置，例如 duration、aspect_ratio、resolution")
engine_id: str | None = Field(None, description="视频生成引擎ID")
engine_name: str | None = Field(None, description="视频生成引擎名称")
params: dict[str, Any] | None = Field(None, description="视频生成实际参数。第5步只传 engine_id，其它参数继承第4步")
chat_task_id: str | None = Field(None, description="视频生成 ChatGenerationTask ID")
status: str | None = Field(None, description="视频生成状态")
result_video_url: str | None = Field(None, description="最终视频 URL")
result_video_cover_url: str | None = Field(None, description="最终视频封面 URL")
error_message: str | None = Field(None, description="视频生成错误信息")
class ShotReplicateTaskDetailOut(BaseModel):
id: str = Field(..., description="总任务项目ID。这个ID就是前端项目ID")
project_id: str = Field(..., description="兼容前端命名，等同于 id")
module: str = Field(..., description="模块标识，拆镜复刻固定为 shot_replicate")
title: str | None = Field(None, description="项目标题，默认取生成项目名称")
status: str = Field(..., description="总任务状态：pending/waiting_user/processing/completed/failed/cancelled")
current_step_code: str | None = Field(None, description="当前所处步骤编码")
final_image_url: str | None = Field(None, description="最终新项目图片 URL")
final_video_url: str | None = Field(None, description="最终视频 URL")
final_video_cover_url: str | None = Field(None, description="最终视频封面 URL")
error_message: str | None = Field(None, description="总任务错误信息")
material: ShotReplicateMaterialOut = Field(default_factory=ShotReplicateMaterialOut, description="素材和项目描述信息")
image_generation: ShotReplicateImageGenerationOut = Field(default_factory=ShotReplicateImageGenerationOut, description="图片提词、图片引擎参数和图片结果")
video_generation: ShotReplicateVideoGenerationOut = Field(default_factory=ShotReplicateVideoGenerationOut, description="视频提词、视频引擎参数和视频结果")
steps: list[ShotReplicateStepOut] = Field(default_factory=list, description="当前有效子任务列表")
created_at: NaiveDatetimeOptional = Field(None, description="创建时间")
updated_at: NaiveDatetimeOptional = Field(None, description="更新时间")
completed_at: NaiveDatetimeOptional = Field(None, description="完成时间")
class ShotReplicateTaskListItemOut(BaseModel):
id: str = Field(..., description="总任务项目ID。这个ID就是前端项目ID")
project_id: str = Field(..., description="兼容前端命名，等同于 id")
module: str = Field(..., description="模块标识")
title: str | None = Field(None, description="项目标题")
status: str = Field(..., description="总任务状态")
current_step_code: str | None = Field(None, description="当前步骤")
target_project_name: str | None = Field(None, description="生成项目名称，来源于第1步素材输入")
final_image_url: str | None = Field(None, description="最终图片 URL")
final_video_url: str | None = Field(None, description="最终视频 URL")
error_message: str | None = Field(None, description="错误信息")
created_at: NaiveDatetimeOptional = Field(None, description="创建时间")
updated_at: NaiveDatetimeOptional = Field(None, description="更新时间")
completed_at: NaiveDatetimeOptional = Field(None, description="完成时间")
class ShotReplicateTaskListOut(BaseModel):
total: int = Field(..., description="总数量")
items: list[ShotReplicateTaskListItemOut] = Field(default_factory=list, description="列表数据")
class ShotReplicateActionOut(BaseModel):
message: str = Field(..., description="操作结果提示")
project_id: str = Field(..., description="总任务项目ID")
step_id: str | None = Field(None, description="本次创建或修改的子任务ID")
next_step_id: str | None = Field(None, description="兼容字段：当前接口不自动生成下下个任务，一般为空")
detail: ShotReplicateTaskDetailOut | None = Field(None, description="操作后的总任务详情")
class ShotReplicateDeleteOut(BaseModel):
message: str = Field(..., description="删除结果提示")
project_id: str = Field(..., description="被软删除的总任务项目ID")
deleted: bool = Field(..., description="是否已软删除")
class ShotReplicateSpecOut(BaseModel):
project_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_REPLICATE_PROJECT_STATUS_DESCRIPTIONS, description="总任务状态说明")
step_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_REPLICATE_STEP_STATUS_DESCRIPTIONS, description="子任务状态说明")
steps: list[dict[str, Any]] = Field(default_factory=lambda: SHOT_REPLICATE_STEP_DESCRIPTIONS, description="5个固定步骤说明")
step_io_schema_version: str = Field(default=SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION, description="步骤 input_json/output_json 结构版本")
step_io_examples: dict[str, dict[str, Any]] = Field(default_factory=lambda: SHOT_REPLICATE_STEP_IO_EXAMPLES, description="每个步骤 input_json/output_json 示例")
# ========================
# 拆镜总任务集 / 片段 API Schema
# ========================
SHOT_TASK_SET_STATUS_DESCRIPTIONS: dict[str, str] = {
"pending_analysis": "已创建，等待原视频分析",
"analyzing": "原视频分析中",
"analysis_completed": "原视频分析完成",
"analysis_failed": "原视频分析失败",
"splitting": "拆镜处理中",
"split_completed": "拆镜全部完成",
"partial_failed": "部分片段失败",
"failed": "总任务失败",
"deleted": "已软删",
}
SHOT_ANALYSIS_STATUS_DESCRIPTIONS: dict[str, str] = {
"pending": "待分析",
"processing": "分析中",
"completed": "分析完成",
"failed": "分析失败",
}
SHOT_SPLIT_STATUS_DESCRIPTIONS: dict[str, str] = {
"none": "尚未拆镜",
"pending": "待拆镜",
"processing": "拆镜中",
"completed": "拆镜完成",
"failed": "拆镜失败",
"retry_waiting": "等待恢复重试",
}
SHOT_SEGMENT_SOURCE_MODE_DESCRIPTIONS: dict[str, str] = {
"ai_suggestion": "AI 建议拆镜",
"custom": "用户自定义拆镜",
}
SHOT_SEGMENT_ANALYSIS_STATUS_DESCRIPTIONS: dict[str, str] = {
"not_required": "不需要单独分析，通常用于 AI 建议拆镜",
"pending": "等待片段分析",
"processing": "片段分析中",
"completed": "片段分析完成",
"failed": "片段分析失败",
}
SHOT_SEGMENT_REPLICATE_STATUS_DESCRIPTIONS: dict[str, str] = {
"not_started": "未进入复刻流程",
"project_created": "已创建复刻项目",
"processing": "复刻流程处理中",
"completed": "复刻流程完成",
"failed": "复刻流程失败",
}
class ShotAISuggestionOut(BaseModel):
index: int = Field(..., description="AI 建议序号，从1开始")
start_second: float = Field(..., description="拆镜开始秒")
end_second: float = Field(..., description="拆镜结束秒")
duration_seconds: float = Field(..., description="片段时长")
time_node: str = Field(..., description="拆镜时间节点，例如 0-15秒")
content: str = Field(..., description="对应时间节点内的内容")
category: str = Field(..., description="片段分类")
audience: str = Field(..., description="片段受众人群")
class ShotTaskSetCreate(BaseModel):
model_config = ConfigDict(
json_schema_extra={
"example": {
"video_url": "/uploads/2026/06/11/demo.mp4",
"video_duration_seconds": 31.42,
"title": "游戏视频拆镜",
"idempotency_key": "frontend-shot-task-001",
}
}
)
video_url: str = Field(..., min_length=1, description="已有上传接口返回的视频链接。必须能反解到 storage/uploads 下文件")
video_duration_seconds: float = Field(..., gt=0, description="前端获取的视频时长秒数，允许浮点；后端会用 ffprobe 校验并以后端真实时长为准")
title: str | None = Field(None, max_length=160, description="拆镜总任务标题")
idempotency_key: str | None = Field(None, max_length=64, description="创建总任务幂等键")
@field_validator("video_url", "title", "idempotency_key", mode="before")
@classmethod
def _strip_optional_text(cls, value: str | None) -> str | None:
if value is None:
return None
value = str(value).strip()
if not value:
return None
return value
class ShotTaskSetListQuery(BaseModel):
status: str | None = Field(None, description="总任务状态筛选，见 ShotTaskSetStatusEnum")
analysis_status: str | None = Field(None, description="分析状态筛选，见 ShotAnalysisStatusEnum")
split_status: str | None = Field(None, description="拆镜状态筛选，见 ShotSplitStatusEnum")
keyword: str | None = Field(None, description="标题/内容关键词")
page: int = Field(1, ge=1, description="页码")
page_size: int = Field(20, ge=1, le=100, description="每页数量")
class ShotTaskSetOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
title: str | None = None
video_url: str
video_duration_seconds: float
status: str
analysis_status: str
split_status: str
original_video_content: str | None = None
original_video_category: str | None = None
original_video_audience: str | None = None
segment_count: int = 0
completed_segment_count: int = 0
failed_segment_count: int = 0
analysis_error_message: str | None = None
split_error_message: str | None = None
created_at: NaiveDatetimeOptional = None
updated_at: NaiveDatetimeOptional = None
class ShotTaskSetListOut(BaseModel):
total: int
page: int
page_size: int
items: list[ShotTaskSetOut]
class ShotTaskSetDetailOut(ShotTaskSetOut):
ai_suggestions: list[ShotAISuggestionOut] = Field(default_factory=list)
analysis_result_json: dict[str, Any] | list[Any] | None = None
class ShotSplitByAIRequest(BaseModel):
selected_indices: list[int] | None = Field(None, description="指定 AI 建议序号。不传则全部拆")
replace_existing: bool = Field(False, description="是否软删旧 AI 建议片段后重新拆")
class ShotSplitCustomRequest(BaseModel):
start_second: float = Field(..., ge=0, description="自定义拆镜开始秒，允许浮点")
end_second: float = Field(..., gt=0, description="自定义拆镜结束秒，允许浮点，必须大于 start_second")
@model_validator(mode="after")
def _check_range(self) -> "ShotSplitCustomRequest":
if self.end_second <= self.start_second:
raise ValueError("end_second 必须大于 start_second")
return self
class ShotSegmentOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
task_set_id: str
segment_index: int
segment_name: str | None = None
source_mode: str
start_second: float
end_second: float
duration_seconds: float
time_node: str
split_status: str
analysis_status: str
replicate_status: str
segment_video_url: str | None = None
original_video_content: str | None = None
original_video_category: str | None = None
original_video_audience: str | None = None
segment_content: str | None = None
segment_category: str | None = None
segment_audience: str | None = None
split_retry_count: int = 0
split_last_error: str | None = None
analysis_error_message: str | None = None
module_project_id: str | None = None
created_at: NaiveDatetimeOptional = None
updated_at: NaiveDatetimeOptional = None
class ShotSegmentDetailOut(ShotSegmentOut):
analysis_json: dict[str, Any] | list[Any] | None = None
ai_suggestion_json: dict[str, Any] | list[Any] | None = None
class ShotSegmentListOut(BaseModel):
total: int
page: int
page_size: int
items: list[ShotSegmentOut]
class ShotSplitByAIOut(BaseModel):
task_set_id: str
status: str
split_status: str
created_segment_count: int
segments: list[ShotSegmentOut]
class ShotSplitCustomOut(BaseModel):
task_set_id: str
segment: ShotSegmentOut
class ShotSegmentReplicationCreateRequest(BaseModel):
model_config = ConfigDict(
json_schema_extra={
"example": {
"target_project_name": "新产品推广视频",
"core_content_point": "突出产品附近交友和快速脱单",
"material_image_url": "/uploads/2026/06/11/product.png",
"idempotency_key": "optional-key",
}
}
)
target_project_name: str = Field(..., min_length=1, max_length=20, description="生成项目名称")
core_content_point: str = Field(..., min_length=1, max_length=50, description="生成项目核心内容点，最多50字")
material_image_url: str = Field(..., min_length=1, description="新产品/目标素材图片链接，来自已有上传接口")
idempotency_key: str | None = Field(None, max_length=64, description="创建 ModuleGenerationProject 幂等键")
@field_validator("target_project_name", "core_content_point", "material_image_url", "idempotency_key", mode="before")
@classmethod
def _strip_text(cls, value: str | None) -> str | None:
if value is None:
return None
value = str(value).strip()
if not value:
return None
return value
class ShotReplicateSpecOut(BaseModel):
project_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_REPLICATE_PROJECT_STATUS_DESCRIPTIONS)
step_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_REPLICATE_STEP_STATUS_DESCRIPTIONS)
steps: list[dict[str, Any]] = Field(default_factory=lambda: SHOT_REPLICATE_STEP_DESCRIPTIONS)
step_io_schema_version: str = SHOT_REPLICATE_STEP_IO_SCHEMA_VERSION
step_io_examples: dict[str, dict[str, Any]] = Field(default_factory=lambda: SHOT_REPLICATE_STEP_IO_EXAMPLES)
task_set_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_TASK_SET_STATUS_DESCRIPTIONS)
analysis_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_ANALYSIS_STATUS_DESCRIPTIONS)
split_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_SPLIT_STATUS_DESCRIPTIONS)
segment_source_modes: dict[str, str] = Field(default_factory=lambda: SHOT_SEGMENT_SOURCE_MODE_DESCRIPTIONS)
segment_analysis_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_SEGMENT_ANALYSIS_STATUS_DESCRIPTIONS)
segment_replicate_statuses: dict[str, str] = Field(default_factory=lambda: SHOT_SEGMENT_REPLICATE_STATUS_DESCRIPTIONS)
video-gen-api/app/services/celery_download_recovery_service.py
+57 -262
View File
@@ -1,103 +1,28 @@
# app/services/celery_download_recovery_service.py
from __future__ import annotations
import inspect
import json
import logging
from datetime import datetime, timezone
from datetime import datetime
from typing import Any, Dict, Iterable, List, Optional, Union
from app.config import settings
try:
from redis.exceptions import RedisError
except ImportError:
RedisError = RuntimeError # type: ignore[assignment]
from app.services.redis_registry_service import (
close_registry_redis,
datetime_to_epoch,
ensure_aware_utc,
get_registry_redis,
redis_get_due_registry_ids,
redis_get_registry_payloads,
redis_postpone_registry_item,
redis_remove_registry_item,
redis_upsert_registry_item,
utc_now,
)
logger = logging.getLogger("video_gen")
_redis_client: Optional[Any] = None
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def ensure_aware_utc(value: Optional[datetime]) -> Optional[datetime]:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def datetime_to_epoch(value: Optional[datetime]) -> int:
checked_value = ensure_aware_utc(value) or utc_now()
return int(checked_value.timestamp())
def _registry_redis_url() -> str:
return settings.CELERY_BROKER_URL or settings.REDIS_URL or ""
async def get_registry_redis() -> Optional[Any]:
global _redis_client
if _redis_client is not None:
return _redis_client
redis_url = _registry_redis_url()
if not redis_url:
return None
try:
from redis.asyncio import Redis
except ImportError as exc:
logger.warning(
"下载容灾 Redis 注册表不可用，redis 依赖未安装。error=%s",
exc,
)
return None
try:
redis_client = Redis.from_url(redis_url, decode_responses=True)
await redis_client.ping()
_redis_client = redis_client
return _redis_client
except (RedisError, OSError, RuntimeError) as exc:
logger.warning(
"下载容灾 Redis 注册表不可用，降级为仅 DB 容灾。error=%s",
exc,
)
_redis_client = None
return None
async def close_registry_redis() -> None:
global _redis_client
client = _redis_client
_redis_client = None
if client is None:
return
try:
close_method = getattr(client, "close", None)
if close_method is None:
return
close_result = close_method()
if inspect.isawaitable(close_result):
await close_result
except (RedisError, OSError, RuntimeError) as exc:
logger.debug(
"关闭下载容灾 Redis 注册表连接失败。error=%s",
exc,
)
# 说明：
# - 本文件保留旧函数名，作为下载容灾兼容层。
# - 底层 Redis Hash/ZSet 操作已迁移到 redis_registry_service.py。
# - 下载 active key、Redis URL 选择逻辑不变，避免影响已稳定下载模块。
def build_download_active_payload(
@@ -131,32 +56,12 @@ def build_download_active_payload(
"attempt": int(attempt or 0),
"queue": queue,
"priority": priority,
"enqueue_at": (
datetime_to_epoch(checked_enqueue_at)
if checked_enqueue_at
else None
),
"started_at": (
datetime_to_epoch(checked_started_at)
if checked_started_at
else None
),
"enqueue_at": datetime_to_epoch(checked_enqueue_at) if checked_enqueue_at else None,
"started_at": datetime_to_epoch(checked_started_at) if checked_started_at else None,
"updated_at": datetime_to_epoch(checked_updated_at),
"lease_until": (
datetime_to_epoch(checked_lease_until)
if checked_lease_until
else None
),
"next_retry_at": (
datetime_to_epoch(checked_next_retry_at)
if checked_next_retry_at
else None
),
"check_at": (
datetime_to_epoch(checked_check_at)
if checked_check_at
else None
),
"lease_until": datetime_to_epoch(checked_lease_until) if checked_lease_until else None,
"next_retry_at": datetime_to_epoch(checked_next_retry_at) if checked_next_retry_at else None,
"check_at": datetime_to_epoch(checked_check_at) if checked_check_at else None,
"reason": reason,
}
@@ -167,56 +72,23 @@ async def upsert_download_active(
payload: Dict[str, Any],
check_at: Optional[Union[datetime, int, float]],
) -> None:
redis = await get_registry_redis()
if redis is None:
return
if isinstance(check_at, datetime):
score = datetime_to_epoch(check_at)
elif check_at is None:
score = datetime_to_epoch(utc_now())
else:
score = int(float(check_at))
updated_payload = dict(payload)
updated_payload["check_at"] = score
try:
pipe: Any = redis.pipeline(transaction=True)
pipe.hset(
settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
record_id,
json.dumps(updated_payload, ensure_ascii=False, default=str),
)
pipe.zadd(
settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
{record_id: score},
)
await pipe.execute()
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"写入下载容灾 Redis 注册表失败。record_id=%s, error=%s",
record_id,
exc,
)
await redis_upsert_registry_item(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
item_id=record_id,
payload=payload,
check_at=check_at,
log_context="download_active",
)
async def remove_download_active(record_id: str) -> None:
redis = await get_registry_redis()
if redis is None:
return
try:
pipe: Any = redis.pipeline(transaction=True)
pipe.hdel(settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, record_id)
pipe.zrem(settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY, record_id)
await pipe.execute()
except (RedisError, OSError, RuntimeError) as exc:
logger.warning(
"删除下载容灾 Redis 注册表失败。record_id=%s, error=%s",
record_id,
exc,
)
await redis_remove_registry_item(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
item_id=record_id,
log_context="download_active",
)
async def get_due_download_record_ids(
@@ -224,68 +96,22 @@ async def get_due_download_record_ids(
limit: Optional[int] = None,
now: Optional[datetime] = None,
) -> List[str]:
redis = await get_registry_redis()
if redis is None:
return []
batch_limit = int(limit or settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100)
score = datetime_to_epoch(now or utc_now())
try:
result = await redis.zrangebyscore(
settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
min="-inf",
max=score,
start=0,
num=batch_limit,
)
return [str(item) for item in result]
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"扫描下载容灾 Redis ZSet 失败。error=%s",
exc,
)
return []
return await redis_get_due_registry_ids(
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
limit=limit or int(settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100),
now=now,
log_context="download_active",
)
async def get_download_active_payloads(
record_ids: Iterable[str],
) -> Dict[str, Dict[str, Any]]:
cleaned_record_ids = [str(item) for item in record_ids if item]
if not cleaned_record_ids:
return {}
redis = await get_registry_redis()
if redis is None:
return {}
try:
raw_values = await redis.hmget(
settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
cleaned_record_ids,
)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"读取下载容灾 Redis Hash 失败。error=%s",
exc,
)
return {}
result: Dict[str, Dict[str, Any]] = {}
for record_id, raw in zip(cleaned_record_ids, raw_values):
if not raw:
continue
try:
value = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
continue
if isinstance(value, dict):
result[record_id] = value
return result
return await redis_get_registry_payloads(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
item_ids=record_ids,
log_context="download_active",
)
async def postpone_download_active_check(
@@ -294,45 +120,14 @@ async def postpone_download_active_check(
payload: Optional[Dict[str, Any]] = None,
check_at: Optional[Union[datetime, int, float]] = None,
) -> None:
redis = await get_registry_redis()
if redis is None:
return
if check_at is None:
check_at = utc_now().timestamp() + int(settings.DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS or 300)
if isinstance(check_at, datetime):
score = datetime_to_epoch(check_at)
elif check_at is None:
score = datetime_to_epoch(utc_now()) + int(
settings.DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS or 300
)
else:
score = int(float(check_at))
try:
pipe: Any = redis.pipeline(transaction=True)
pipe.zadd(
settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
{record_id: score},
)
if payload is not None:
updated_payload = dict(payload)
updated_payload["check_at"] = score
updated_payload["updated_at"] = datetime_to_epoch(utc_now())
pipe.hset(
settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
record_id,
json.dumps(
updated_payload,
ensure_ascii=False,
default=str,
),
)
await pipe.execute()
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"刷新下载容灾 Redis 检查时间失败。record_id=%s, error=%s",
record_id,
exc,
)
await redis_postpone_registry_item(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
item_id=record_id,
payload=payload,
check_at=check_at,
log_context="download_active",
)
video-gen-api/app/services/generation_module_hook_service.py
+12
View File
@@ -23,3 +23,15 @@ async def notify_chat_generation_task_finished(db: AsyncSession, task: ChatGener
await handle_chat_generation_task_completed(db, task)
elif task.status == "failed":
await handle_chat_generation_task_failed(db, task)
return
if task.generation_mode == "shot_replicate":
from app.services.shot_replicate_flow_service import (
handle_chat_generation_task_completed,
handle_chat_generation_task_failed,
)
if task.status == "completed":
await handle_chat_generation_task_completed(db, task)
elif task.status == "failed":
await handle_chat_generation_task_failed(db, task)
return
video-gen-api/app/services/generation_recovery_service.py
+389 -105
View File
@@ -1,10 +1,11 @@
from __future__ import annotations
import json
import logging
from datetime import datetime, timedelta, timezone
from typing import Any, Dict
from typing import Any
from sqlalchemy import or_, select
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
@@ -16,13 +17,21 @@ from app.services.celery_download_recovery_service import (
postpone_download_active_check,
remove_download_active,
)
from app.services.generation_log_service import log_task_event
from app.services.generation_log_service import log_provider_call, log_task_event
from app.services.generation_module_hook_service import notify_chat_generation_task_finished
from app.services.generation_provider_service import poll_provider_task
from app.services.generation_refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.redis_registry_service import (
redis_get_due_registry_ids,
redis_get_registry_payloads,
redis_postpone_registry_item,
redis_remove_registry_item,
)
logger = logging.getLogger("video_gen")
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate"}
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate", "shot_replicate"}
POLL_QUEUE = "gen_provider_poll"
def _now() -> datetime:
@@ -58,6 +67,52 @@ def _is_final_task_state(task: ChatGenerationTask) -> bool:
)
def _is_success(status: str | None) -> bool:
return str(status or "").lower() in ("succeeded", "success", "completed", "done")
def _is_failed(status: str | None) -> bool:
return str(status or "").lower() in ("failed", "error", "canceled", "cancelled")
def _engine_snapshot(task: ChatGenerationTask) -> dict[str, Any]:
try:
value = json.loads(task.engine_snapshot_json or "{}")
return value if isinstance(value, dict) else {}
except Exception:
return {}
def _poll_queue_timeout_at(now: datetime | None = None) -> datetime:
current_time = now or _now()
return current_time + timedelta(seconds=int(settings.POLL_TASK_QUEUE_TIMEOUT_SECONDS or 120))
async def _remove_poll_active(task_id: str) -> None:
await redis_remove_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=task_id,
log_context="poll_active",
)
async def _postpone_poll_active(
*,
task_id: str,
payload: dict[str, Any] | None = None,
check_at: datetime | int | float | None = None,
) -> None:
await redis_postpone_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=task_id,
payload=payload,
check_at=check_at or _poll_queue_timeout_at(),
log_context="poll_active",
)
async def recover_one_download_task(
db: AsyncSession,
task: ChatGenerationTask,
@@ -221,7 +276,7 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
select(ChatGenerationTask)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate"]),
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate", "shot_replicate"]),
ChatGenerationTask.status == "generating",
ChatGenerationTask.remote_result_url.is_not(None),
ChatGenerationTask.pipeline_stage.in_(
@@ -249,134 +304,363 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
return {"checked": len(checked_ids), "results": results}
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""启动时生成链路容灾扫描。
async def _mark_timeout(
db: AsyncSession,
task: ChatGenerationTask,
*,
error_message: str = "任务超时",
) -> str:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message=error_message,
pipeline_stage="timeout",
)
await notify_chat_generation_task_finished(db, task)
await db.commit()
await _remove_poll_active(task.id)
await log_task_event(
task,
event_type="TASK_TIMEOUT",
to_status="failed",
to_stage="timeout",
)
return "mark_timeout"
只在 Celery worker 启动时跑一次，不引入 beat，不新增第四条启动命令。
用于把 queued/creating/waiting_remote/polling/result_ready 等中间态重新投递到现有三个队列。
async def _mark_failed(
db: AsyncSession,
task: ChatGenerationTask,
*,
error_message: str,
event_type: str = "POLL_FAILED",
detail: Any = None,
) -> str:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message=error_message,
pipeline_stage="failed",
)
await notify_chat_generation_task_finished(db, task)
await db.commit()
await _remove_poll_active(task.id)
await log_task_event(task, event_type=event_type, message=task.error_message, detail=detail)
return "mark_failed"
async def _try_final_poll_before_timeout(db: AsyncSession, task: ChatGenerationTask) -> str:
"""超时前最后查一次供应商，避免 Celery 中断导致本地假超时。
如果供应商已经成功，继续进入下载；如果仍 running 或查询失败，再按超时处理。
"""
from app.tasks.generation_download_tasks import enqueue_download_task
if not (task.provider_task_id or task.seedance_task_id):
return await _mark_timeout(db, task)
try:
poll_result = await poll_provider_task(db, task)
status = poll_result.get("status")
response_data = poll_result.get("response_data")
except Exception as exc:
await log_task_event(
task,
event_type="FINAL_POLL_BEFORE_TIMEOUT_ERROR",
message=str(exc),
)
return await _mark_timeout(db, task)
try:
provider_response = json.loads(response_data or "{}")
except Exception:
provider_response = {"raw": response_data}
snapshot = _engine_snapshot(task)
await log_provider_call(
task,
provider=snapshot.get("provider") or "ark",
api_type=f"{task.gen_type}_final_poll_before_timeout",
model=snapshot.get("model_name"),
engine_id=task.engine_id,
status="success",
provider_task_id=task.seedance_task_id or task.provider_task_id,
response_data=provider_response,
)
if _is_success(status):
if task.gen_type == "image":
task.remote_result_url = poll_result.get("image_url")
task.image_tokens_used = poll_result.get("image_tokens", 0) or 0
else:
task.remote_result_url = poll_result.get("video_url")
task.video_tokens_used = poll_result.get("video_tokens", 0) or 0
task.provider_response_json = response_data
if not task.remote_result_url:
return await _mark_failed(
db,
task,
error_message="供应商任务成功但未返回结果URL",
detail=poll_result,
)
task.pipeline_stage = "result_ready"
task.retry_count = 0
await db.commit()
await _remove_poll_active(task.id)
await log_task_event(
task,
event_type="POLL_SUCCESS_AFTER_TIMEOUT_RECOVERY",
to_stage="result_ready",
detail=poll_result,
)
await enqueue_download_task(db, task, recover=True, reason="final_poll_before_timeout_success")
return "recover_timeout_success_to_download"
if _is_failed(status):
task.provider_response_json = response_data
return await _mark_failed(
db,
task,
error_message=poll_result.get("error") or f"供应商任务失败: {status}",
detail=poll_result,
)
await log_task_event(
task,
event_type="FINAL_POLL_BEFORE_TIMEOUT_PENDING",
message=f"status={status}",
detail=poll_result,
)
return await _mark_timeout(db, task)
async def recover_one_generation_task(
db: AsyncSession,
task: ChatGenerationTask,
*,
payload: dict[str, Any] | None = None,
source: str = "startup_db",
) -> str:
from app.tasks.generation_create_tasks import chatapi_create_generation_task
from app.tasks.generation_download_tasks import enqueue_download_task
from app.tasks.generation_poll_tasks import poll_generation_task
from app.tasks.generation_poll_tasks import poll_generation_task, register_poll_active
current_time = _now()
results: dict[str, int] = {}
redis_payload = payload or {}
query_result = await db.execute(
select(ChatGenerationTask)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate"]),
ChatGenerationTask.status == "generating",
ChatGenerationTask.pipeline_stage.in_(
[
"queued",
"preparing",
"creating_provider_task",
"waiting_remote",
"polling",
"result_ready",
]
),
)
.order_by(ChatGenerationTask.updated_at.asc())
.limit(int(settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100))
.with_for_update(skip_locked=True)
)
tasks = query_result.scalars().all()
if not task:
return "skip_missing_task"
if task.generation_mode not in ALLOWED_GENERATION_MODES:
await _remove_poll_active(task.id)
return "clean_invalid_mode"
if _is_final_task_state(task):
await _remove_poll_active(task.id)
return "clean_final_state"
if task.status != "generating":
await _remove_poll_active(task.id)
return "clean_not_generating"
for task in tasks:
if task.deadline_at and _is_expired(task.deadline_at, current_time):
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message="任务超时",
pipeline_stage="timeout",
)
await notify_chat_generation_task_finished(db, task)
if task.deadline_at and _is_expired(task.deadline_at, current_time):
if task.pipeline_stage in ("waiting_remote", "polling"):
return await _try_final_poll_before_timeout(db, task)
return await _mark_timeout(db, task)
if task.pipeline_stage in ("queued", "preparing", "creating_provider_task"):
if task.provider_task_id or task.seedance_task_id:
task.pipeline_stage = "waiting_remote"
await db.commit()
await log_task_event(
task,
event_type="TASK_TIMEOUT",
to_status="failed",
to_stage="timeout",
event_type="GENERATION_RECOVERY_ENQUEUE",
message=f"{source} 发现创建阶段已存在供应商任务ID，恢复投递轮询队列",
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
)
action = "mark_timeout"
poll_generation_task.apply_async(args=[task.id], queue=POLL_QUEUE, countdown=0)
await register_poll_active(
task,
check_at=_poll_queue_timeout_at(),
reason=f"{source}_create_stage_has_provider_id",
)
return "recover_poll_from_create_stage"
elif task.pipeline_stage in ("queued", "preparing", "creating_provider_task"):
await log_task_event(
task,
event_type="GENERATION_RECOVERY_ENQUEUE",
message=f"{source} 发现创建阶段任务未完成，恢复投递创建队列",
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
)
chatapi_create_generation_task.apply_async(
args=[task.id],
queue="gen_chatapi_create",
countdown=0,
)
return "recover_create"
if task.pipeline_stage in ("waiting_remote", "polling"):
if task.remote_result_url:
await _remove_poll_active(task.id)
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_waiting_remote_has_result",
)
return "recover_waiting_has_result"
if task.provider_task_id or task.seedance_task_id:
await log_task_event(
task,
event_type="GENERATION_RECOVERY_ENQUEUE",
message="启动时发现创建阶段任务未完成，恢复投递创建队列",
detail={"pipeline_stage": task.pipeline_stage},
message=f"{source} 发现远程等待/轮询阶段任务未完成，恢复投递轮询队列",
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
)
chatapi_create_generation_task.apply_async(
task.pipeline_stage = "waiting_remote"
await db.commit()
poll_generation_task.apply_async(
args=[task.id],
queue="gen_chatapi_create",
queue=POLL_QUEUE,
countdown=0,
)
action = "recover_create"
await register_poll_active(
task,
check_at=_poll_queue_timeout_at(),
reason=f"{source}_recover_poll",
)
return "recover_poll"
elif task.pipeline_stage in ("waiting_remote", "polling"):
if task.remote_result_url:
await enqueue_download_task(
db,
task,
recover=True,
reason="startup_waiting_remote_has_result",
)
action = "recover_waiting_has_result"
elif task.provider_task_id or task.seedance_task_id:
await log_task_event(
task,
event_type="GENERATION_RECOVERY_ENQUEUE",
message="启动时发现远程等待/轮询阶段任务未完成，恢复投递轮询队列",
detail={"pipeline_stage": task.pipeline_stage},
)
task.pipeline_stage = "waiting_remote"
await db.commit()
poll_generation_task.apply_async(
args=[task.id],
queue="gen_provider_poll",
countdown=0,
)
action = "recover_poll"
else:
await log_task_event(
task,
event_type="GENERATION_RECOVERY_ENQUEUE",
message="启动时发现任务缺少供应商任务ID，恢复投递创建队列",
detail={"pipeline_stage": task.pipeline_stage},
)
task.pipeline_stage = "queued"
await db.commit()
chatapi_create_generation_task.apply_async(
args=[task.id],
queue="gen_chatapi_create",
countdown=0,
)
action = "recover_create_missing_provider_id"
await log_task_event(
task,
event_type="GENERATION_RECOVERY_ENQUEUE",
message=f"{source} 发现任务缺少供应商任务ID，恢复投递创建队列",
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
)
task.pipeline_stage = "queued"
await db.commit()
await _remove_poll_active(task.id)
chatapi_create_generation_task.apply_async(
args=[task.id],
queue="gen_chatapi_create",
countdown=0,
)
return "recover_create_missing_provider_id"
elif task.pipeline_stage == "result_ready":
if task.remote_result_url:
await enqueue_download_task(
db,
task,
recover=True,
reason="startup_generation_result_ready",
)
action = "recover_result_ready"
else:
action = "skip_result_ready_no_url"
if task.pipeline_stage == "result_ready":
await _remove_poll_active(task.id)
if task.remote_result_url:
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_generation_result_ready",
)
return "recover_result_ready"
return "skip_result_ready_no_url"
return f"skip_stage_{task.pipeline_stage}"
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""启动时生成链路容灾扫描。
不新增 Celery beat，不新增 worker 命令；worker 启动时由 Redis 锁保证只投递一次。
恢复顺序：
1. Redis poll active_index 到期任务；
2. DB fallback 扫描 queued/creating/waiting_remote/polling/result_ready；
3. 下载阶段仍由 recover_download_tasks_once 兜底。
"""
checked_ids: set[str] = set()
results: dict[str, int] = {}
due_poll_ids = await redis_get_due_registry_ids(
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
limit=int(settings.POLL_RECOVERY_BATCH_SIZE or settings.GENERATION_RECOVERY_BATCH_SIZE or 100),
log_context="poll_active",
)
poll_payloads = await redis_get_registry_payloads(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
item_ids=due_poll_ids,
log_context="poll_active",
)
for task_id in due_poll_ids:
result = await db.execute(
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == task_id,
ChatGenerationTask.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
task = result.scalar_one_or_none()
if task is None:
await _remove_poll_active(task_id)
action = "clean_missing_poll_task"
else:
action = f"skip_stage_{task.pipeline_stage}"
checked_ids.add(task.id)
action = await recover_one_generation_task(
db,
task,
payload=poll_payloads.get(task_id),
source="startup_poll_redis",
)
results[action] = results.get(action, 0) + 1
batch_size = int(settings.GENERATION_RECOVERY_BATCH_SIZE or settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100)
max_rounds = max(1, int(settings.GENERATION_RECOVERY_MAX_ROUNDS or 1))
total_db_checked = 0
for _round in range(max_rounds):
query_result = await db.execute(
select(ChatGenerationTask)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate", "shot_replicate"]),
ChatGenerationTask.status == "generating",
ChatGenerationTask.pipeline_stage.in_(
[
"queued",
"preparing",
"creating_provider_task",
"waiting_remote",
"polling",
"result_ready",
]
),
)
.order_by(ChatGenerationTask.updated_at.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
tasks = query_result.scalars().all()
if not tasks:
break
progressed_this_round = 0
for task in tasks:
if task.id in checked_ids:
continue
action = await recover_one_generation_task(
db,
task,
payload=None,
source="startup_db",
)
results[action] = results.get(action, 0) + 1
checked_ids.add(task.id)
total_db_checked += 1
progressed_this_round += 1
if len(tasks) < batch_size or progressed_this_round <= 0:
break
# 下载阶段单独跑 DB fallback。
download_result = await recover_download_tasks_once(db)
return {
"checked": len(tasks),
"checked": len(checked_ids),
"db_checked": total_db_checked,
"results": results,
"download_recovery": download_result,
}
video-gen-api/app/services/generation_refund_service.py
+1 -1
View File
@@ -180,7 +180,7 @@ async def mark_chat_generation_task_failed_and_refund_once(
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == task_id,
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate"]),
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate", "shot_replicate"]),
ChatGenerationTask.deleted_at.is_(None),
)
.with_for_update()
video-gen-api/app/services/hot_opening_replicate_service.py
+55 -1
View File
@@ -44,7 +44,7 @@ from app.services.generation_billing_service import charge_module_prompt_usage
from app.services.generation_refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.generation_task_factory_service import create_chat_generation_task_for_module
from app.services.hot_opening_video_prompt_service import build_final_video_prompt, optimize_hot_opening_video_prompt, patch_video_prompt_schema_from_client
from app.services.module_generation_log_service import log_module_event_file, log_module_prompt_event
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
from app.services.llm import optimize_prompt
from app.services.resource_accounting_service import soft_delete_chat_task_resources
from app.services.resource_signed_url_service import build_resource_signed_url
@@ -206,6 +206,28 @@ async def log_module_event(
)
def _log_project_error(
*,
project: ModuleGenerationProject | None,
event_type: str,
message: str,
exc: BaseException | None = None,
step: ModuleGenerationStep | None = None,
detail: dict[str, Any] | None = None,
) -> None:
log_module_error(
module=(project.module if project else MODULE),
event_type=event_type,
project_id=(project.id if project else None),
step_id=(step.id if step else None),
user_id=(project.user_id if project else None),
message=message,
detail=detail,
exc=exc,
)
async def _get_project_for_user(
db: AsyncSession,
*,
@@ -1039,6 +1061,17 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step.completed_at = _now()
project.status = ModuleProjectStatusEnum.FAILED.value
project.error_message = f"图片 AI 提词生成失败: {exc}"
log_module_prompt_event(
event_type="module_prompt_error",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
module=project.module,
prompt_type=ModulePromptTypeEnum.IMAGE_PROMPT.value,
request=locals().get("request_log", {}),
error=str(exc),
)
_log_project_error(project=project, step=step, event_type="IMAGE_PROMPT_FAILED", message=project.error_message, exc=exc)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.IMAGE_PROMPT_FAILED.value, message=project.error_message)
return step
@@ -1317,6 +1350,17 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step.completed_at = _now()
project.status = ModuleProjectStatusEnum.FAILED.value
project.error_message = f"视频 AI 提词生成失败: {exc}"
log_module_prompt_event(
event_type="module_prompt_error",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
module=project.module,
prompt_type=ModulePromptTypeEnum.VIDEO_PROMPT.value,
request=locals().get("request_log", {}),
error=str(exc),
)
_log_project_error(project=project, step=step, event_type="VIDEO_PROMPT_FAILED", message=project.error_message, exc=exc)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.VIDEO_PROMPT_FAILED.value, message=project.error_message)
return step
@@ -1517,6 +1561,16 @@ async def mark_hot_opening_step_dispatch_failed(
step.completed_at = _now()
project.status = ModuleProjectStatusEnum.FAILED.value
project.error_message = error_message
log_module_error(
module=project.module,
event_type="CELERY_DISPATCH_FAILED",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
message=error_message,
detail={"reason": "celery_dispatch_failed", "chat_task_id": step.chat_task_id},
error=error_message,
)
await log_module_event(
db,
project=project,
video-gen-api/app/services/hot_opening_video_prompt_service.py
+1 -1
View File
@@ -628,7 +628,7 @@ async def optimize_hot_opening_video_prompt(
) -> tuple[dict[str, Any], str, dict[str, Any]]:
duration = int(video_config["duration"])
references = [
{"type": "video", "url": material_video_url},
{"type": "video", "url": _build_file_url_or_data_uri(material_video_url)},
{"type": "image", "url": _build_file_url_or_data_uri(generated_image_url)},
]
client_schema = build_dynamic_schema(video_config)
video-gen-api/app/services/module_generation_log_service.py
+30 -7
View File
@@ -3,12 +3,14 @@ from __future__ import annotations
import json
import os
import re
import traceback
from datetime import datetime
from typing import Any
from app.services.log_config import LOG_DATE_FORMAT, LOG_DIR, is_enabled
MAX_LOG_FIELD_LENGTH = 20000
MAX_TRACEBACK_LENGTH = 12000
MODULE_LOG_ROOT = os.path.join(os.path.dirname(LOG_DIR), "ModuleGeneration")
@@ -33,6 +35,23 @@ def _safe_dump_value(value: Any) -> Any:
return value
def build_exception_detail(exc: BaseException | None, extra: dict[str, Any] | None = None) -> dict[str, Any]:
"""构造统一异常日志 detail。日志方法必须吞异常，业务不能被日志影响。"""
detail: dict[str, Any] = dict(extra or {})
if exc is not None:
tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
if len(tb) > MAX_TRACEBACK_LENGTH:
tb = tb[:MAX_TRACEBACK_LENGTH] + f"...<traceback_truncated:{len(tb) - MAX_TRACEBACK_LENGTH}>"
detail.update(
{
"exception_type": type(exc).__name__,
"exception_message": str(exc),
"traceback": tb,
}
)
return detail
def _append_module_log(module: str, entry: dict[str, Any]) -> None:
if not is_enabled():
return
@@ -92,10 +111,7 @@ def log_module_prompt_event(
token_usage: dict[str, Any] | None = None,
error: str | None = None,
) -> None:
"""记录模块 AI 提词请求/响应到 JSONL 文件。
与模块事件共用同一个服务，但按 module 分目录，方便按模块排查。
"""
"""记录模块 AI 提词/分析请求和响应到 JSONL 文件。"""
entry = {
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"log_type": "module_prompt",
@@ -123,8 +139,15 @@ def log_module_error(
message: str | None = None,
detail: dict[str, Any] | None = None,
error: str | None = None,
exc: BaseException | None = None,
) -> None:
"""记录模块异常日志。"""
"""记录模块异常日志。
- 兼容原有 detail/error 参数。
- 新增 exc 后自动记录 exception_type、message、traceback。
- 日志写入失败会被底层吞掉，不影响主流程。
"""
merged_detail = build_exception_detail(exc, detail)
entry = {
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"log_type": "module_error",
@@ -134,7 +157,7 @@ def log_module_error(
"step_id": step_id,
"user_id": user_id,
"message": message,
"detail": _safe_dump_value(detail or {}),
"error": error,
"detail": _safe_dump_value(merged_detail),
"error": error if error is not None else (str(exc) if exc is not None else None),
}
_append_module_log(module, entry)
video-gen-api/app/services/redis_registry_service.py
+354
View File
@@ -0,0 +1,354 @@
# app/services/redis_registry_service.py
from __future__ import annotations
import asyncio
import inspect
import json
import logging
import os
import threading
import uuid
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Optional, Union
from app.config import settings
try:
from redis.exceptions import RedisError
except ImportError: # pragma: no cover - redis 未安装时降级
RedisError = RuntimeError # type: ignore[assignment]
logger = logging.getLogger("video_gen")
_redis_clients: Dict[tuple[int, int, int], Any] = {}
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def ensure_aware_utc(value: Optional[datetime]) -> Optional[datetime]:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def datetime_to_epoch(value: Optional[datetime]) -> int:
checked_value = ensure_aware_utc(value) or utc_now()
return int(checked_value.timestamp())
def normalize_registry_score(value: Optional[Union[datetime, int, float]]) -> int:
if isinstance(value, datetime):
return datetime_to_epoch(value)
if value is None:
return datetime_to_epoch(utc_now())
return int(float(value))
def registry_redis_url() -> str:
"""Celery 容灾注册表统一使用 Celery broker Redis。
不能改成只读 settings.REDIS_URL，否则线上 CELERY_BROKER_URL 使用独立
Redis DB 时，旧下载 active 注册表会被写到另一个库，导致恢复扫描失效。
"""
return settings.CELERY_BROKER_URL or settings.REDIS_URL or ""
def _is_supported_redis_url(redis_url: str) -> bool:
if not redis_url:
return False
lowered = redis_url.lower()
return lowered.startswith(("redis://", "rediss://", "unix://"))
async def get_registry_redis() -> Optional[Any]:
"""获取 Celery 容灾 Redis 连接。
重点：redis.asyncio 的连接/连接池绑定 event loop，不能跨 loop 复用。
Celery -P threads 或 worker_ready + task 线程混用时，如果使用单个全局
Redis 客户端，会触发 got Future attached to a different loop。
因此这里按 pid + thread_id + event_loop_id 缓存客户端，确保同一个客户端
只在创建它的事件循环里使用。Redis 不可用时返回 None，调用方降级为
DB fallback，不能影响生成主链路。
"""
redis_url = registry_redis_url()
if not _is_supported_redis_url(redis_url):
if redis_url:
logger.warning(
"Celery 容灾 Redis 注册表仅支持 redis/rediss/unix URL，当前 broker 不是 Redis，降级为 DB 容灾。url=%s",
redis_url,
)
return None
try:
from redis.asyncio import Redis
except ImportError as exc:
logger.warning(
"Celery 容灾 Redis 注册表不可用，redis 依赖未安装。error=%s",
exc,
)
return None
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return None
client_key = (os.getpid(), threading.get_ident(), id(loop))
cached = _redis_clients.get(client_key)
if cached is not None:
return cached
try:
redis_client = Redis.from_url(redis_url, decode_responses=True)
await redis_client.ping()
_redis_clients[client_key] = redis_client
return redis_client
except (RedisError, OSError, RuntimeError) as exc:
logger.warning(
"Celery 容灾 Redis 注册表不可用，降级为仅 DB 容灾。error=%s",
exc,
)
_redis_clients.pop(client_key, None)
return None
async def close_registry_redis() -> None:
"""关闭当前进程内已缓存的 Redis 注册表连接。
关闭动作尽量只关闭当前 event loop 对应的客户端；如果调用方处于进程
退出阶段，则逐个尝试关闭，失败忽略，避免影响 worker 退出。
"""
if not _redis_clients:
return
try:
loop = asyncio.get_running_loop()
current_key = (os.getpid(), threading.get_ident(), id(loop))
items = [(current_key, _redis_clients.pop(current_key, None))]
except RuntimeError:
items = list(_redis_clients.items())
_redis_clients.clear()
for _, client in items:
if client is None:
continue
try:
close_method = getattr(client, "close", None) or getattr(client, "aclose", None)
if close_method is None:
continue
close_result = close_method()
if inspect.isawaitable(close_result):
await close_result
except (RedisError, OSError, RuntimeError) as exc:
logger.debug("关闭 Celery 容灾 Redis 注册表连接失败。error=%s", exc)
async def redis_upsert_registry_item(
*,
hash_key: str,
zset_key: str,
item_id: str,
payload: Dict[str, Any],
check_at: Optional[Union[datetime, int, float]],
log_context: str = "registry",
) -> None:
redis = await get_registry_redis()
if redis is None:
return
score = normalize_registry_score(check_at)
updated_payload = dict(payload)
updated_payload["check_at"] = score
updated_payload["updated_at"] = updated_payload.get("updated_at") or datetime_to_epoch(utc_now())
try:
pipe: Any = redis.pipeline(transaction=True)
pipe.hset(hash_key, item_id, json.dumps(updated_payload, ensure_ascii=False, default=str))
pipe.zadd(zset_key, {item_id: score})
await pipe.execute()
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"写入 Redis 注册表失败。context=%s, item_id=%s, error=%s",
log_context,
item_id,
exc,
)
async def redis_remove_registry_item(
*,
hash_key: str,
zset_key: str,
item_id: str,
log_context: str = "registry",
) -> None:
redis = await get_registry_redis()
if redis is None:
return
try:
pipe: Any = redis.pipeline(transaction=True)
pipe.hdel(hash_key, item_id)
pipe.zrem(zset_key, item_id)
await pipe.execute()
except (RedisError, OSError, RuntimeError) as exc:
logger.warning(
"删除 Redis 注册表失败。context=%s, item_id=%s, error=%s",
log_context,
item_id,
exc,
)
async def redis_get_due_registry_ids(
*,
zset_key: str,
limit: Optional[int] = None,
now: Optional[datetime] = None,
log_context: str = "registry",
) -> List[str]:
redis = await get_registry_redis()
if redis is None:
return []
batch_limit = int(limit or 100)
score = datetime_to_epoch(now or utc_now())
try:
result = await redis.zrangebyscore(
zset_key,
min="-inf",
max=score,
start=0,
num=batch_limit,
)
return [str(item) for item in result]
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning("扫描 Redis ZSet 失败。context=%s, error=%s", log_context, exc)
return []
async def redis_get_registry_payloads(
*,
hash_key: str,
item_ids: Iterable[str],
log_context: str = "registry",
) -> Dict[str, Dict[str, Any]]:
cleaned_item_ids = [str(item) for item in item_ids if item]
if not cleaned_item_ids:
return {}
redis = await get_registry_redis()
if redis is None:
return {}
try:
raw_values = await redis.hmget(hash_key, cleaned_item_ids)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning("读取 Redis Hash 失败。context=%s, error=%s", log_context, exc)
return {}
result: Dict[str, Dict[str, Any]] = {}
for item_id, raw in zip(cleaned_item_ids, raw_values):
if not raw:
continue
try:
value = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
continue
if isinstance(value, dict):
result[item_id] = value
return result
async def redis_postpone_registry_item(
*,
hash_key: str,
zset_key: str,
item_id: str,
payload: Optional[Dict[str, Any]] = None,
check_at: Optional[Union[datetime, int, float]] = None,
log_context: str = "registry",
) -> None:
redis = await get_registry_redis()
if redis is None:
return
score = normalize_registry_score(check_at)
try:
pipe: Any = redis.pipeline(transaction=True)
pipe.zadd(zset_key, {item_id: score})
if payload is not None:
updated_payload = dict(payload)
updated_payload["check_at"] = score
updated_payload["updated_at"] = datetime_to_epoch(utc_now())
pipe.hset(hash_key, item_id, json.dumps(updated_payload, ensure_ascii=False, default=str))
await pipe.execute()
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"刷新 Redis 注册表检查时间失败。context=%s, item_id=%s, error=%s",
log_context,
item_id,
exc,
)
async def redis_acquire_lock(
*,
lock_key: str,
ttl_seconds: int,
token: Optional[str] = None,
log_context: str = "lock",
) -> Optional[str]:
"""尝试获取 Redis 分布式锁。
返回 token 表示抢锁成功；返回 None 表示 Redis 不可用或锁已被其他 worker 持有。
"""
redis = await get_registry_redis()
if redis is None:
return None
lock_token = token or uuid.uuid4().hex
ttl = max(1, int(ttl_seconds or 60))
try:
acquired = await redis.set(lock_key, lock_token, nx=True, ex=ttl)
return lock_token if acquired else None
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning("获取 Redis 锁失败。context=%s, lock_key=%s, error=%s", log_context, lock_key, exc)
return None
async def redis_release_lock(
*,
lock_key: str,
token: str,
log_context: str = "lock",
) -> bool:
"""只释放 token 匹配的锁，避免误删其他 worker 新抢到的锁。"""
redis = await get_registry_redis()
if redis is None:
return False
script = """
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('del', KEYS[1])
else
return 0
end
"""
try:
released = await redis.eval(script, 1, lock_key, token)
return bool(released)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning("释放 Redis 锁失败。context=%s, lock_key=%s, error=%s", log_context, lock_key, exc)
return False
video-gen-api/app/services/shot_replicate_flow_service.py
+1747
View File
File diff suppressed because it is too large Load Diff
video-gen-api/app/services/shot_replicate_recovery_service.py
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.shot_replicate import ShotSplitStatusEnum
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.tasks.celery_app import celery_app
def _now() -> datetime:
return datetime.now(timezone.utc)
def _ensure_aware(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _expired(value: datetime | None, now: datetime | None = None) -> bool:
checked = _ensure_aware(value)
if checked is None:
return True
return checked <= (now or _now())
def _queue_timeout(segment: ShotReplicateSegment, now: datetime | None = None) -> bool:
enqueued_at = _ensure_aware(segment.split_enqueued_at)
if enqueued_at is None:
return True
return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now())
async def recover_one_split_segment(db: AsyncSession, segment: ShotReplicateSegment, *, source: str = "startup_db") -> str:
from app.tasks.shot_replicate_tasks import split_one_segment
if not segment:
return "skip_missing_segment"
if segment.deleted_at is not None:
return "skip_deleted"
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value:
return "skip_completed"
if segment.split_status == ShotSplitStatusEnum.FAILED.value:
return "skip_failed"
current_time = _now()
should_recover = False
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
should_recover = _queue_timeout(segment, current_time)
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
should_recover = _expired(segment.split_lease_until, current_time)
elif segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
should_recover = _expired(segment.split_next_retry_at, current_time)
if not should_recover:
return f"skip_{segment.split_status}_not_due"
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_last_error = segment.split_last_error or f"{source} 恢复时超过最大重试次数"
segment.split_lease_until = None
segment.split_next_retry_at = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
return "mark_failed_max_retry"
segment.split_status = ShotSplitStatusEnum.PENDING.value
segment.split_enqueued_at = current_time
segment.split_lease_until = None
segment.split_next_retry_at = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
if celery_app:
split_one_segment.apply_async(
args=[segment.id],
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
countdown=0,
)
return f"recover_{source}"
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""拆镜 ffmpeg 任务容灾恢复。独立扫描 shot_replicate_segments，不复用 Chat 下载 active registry。"""
batch_size = int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50)
result = await db.execute(
select(ShotReplicateSegment)
.where(
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.split_status.in_(
[
ShotSplitStatusEnum.PENDING.value,
ShotSplitStatusEnum.PROCESSING.value,
ShotSplitStatusEnum.RETRY_WAITING.value,
]
),
)
.order_by(ShotReplicateSegment.updated_at.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
segments = list(result.scalars().all())
checked = 0
results: dict[str, int] = {}
touched_task_set_ids: set[str] = set()
for segment in segments:
action = await recover_one_split_segment(db, segment, source="startup_db")
checked += 1
touched_task_set_ids.add(segment.task_set_id)
results[action] = results.get(action, 0) + 1
for task_set_id in touched_task_set_ids:
await refresh_task_set_split_summary(db, task_set_id)
await db.commit()
return {"checked": checked, "results": results}
video-gen-api/app/services/shot_replicate_taskset_service.py
+529
View File
@@ -0,0 +1,529 @@
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from typing import Any
from fastapi import HTTPException
from app.config import settings
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.shot_replicate import (
ModuleCodeEnum,
ShotAnalysisStatusEnum,
ShotSegmentAnalysisStatusEnum,
ShotSegmentReplicateStatusEnum,
ShotSegmentSourceModeEnum,
ShotSplitStatusEnum,
ShotTaskSetStatusEnum,
)
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.models.user import User
from app.schemas.shot_replicate import (
ShotAISuggestionOut,
ShotSegmentDetailOut,
ShotSegmentListOut,
ShotSegmentOut,
ShotSplitByAIOut,
ShotSplitByAIRequest,
ShotSplitCustomOut,
ShotSplitCustomRequest,
ShotTaskSetCreate,
ShotTaskSetDetailOut,
ShotTaskSetListOut,
ShotTaskSetOut,
)
from app.services.module_generation_log_service import log_module_event_file
from app.services.upload_video_asset_service import (
build_time_node,
validate_split_range,
validate_upload_video_asset,
)
from app.tasks.celery_app import celery_app
from app.utils.id_gen import generate_id
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
def _now() -> datetime:
return datetime.now(timezone.utc)
def _normalize_suggestions(value: Any) -> list[dict[str, Any]]:
if not isinstance(value, list):
return []
normalized: list[dict[str, Any]] = []
for idx, item in enumerate(value, start=1):
if not isinstance(item, dict):
continue
start = item.get("拆镜开始秒")
end = item.get("拆镜结束秒")
try:
start_f = float(start)
end_f = float(end)
except Exception:
continue
if start_f < 0 or end_f <= start_f:
continue
normalized.append(
{
"index": idx,
"start_second": start_f,
"end_second": end_f,
"duration_seconds": round(end_f - start_f, 3),
"time_node": str(item.get("拆镜时间节点") or build_time_node(start_f, end_f)),
"content": str(item.get("对应时间节点内的内容") or "无"),
"category": str(item.get("分类") or "无"),
"audience": str(item.get("受众人群") or "无"),
"raw": item,
}
)
return normalized
def _task_set_to_out(task_set: ShotReplicateTaskSet) -> ShotTaskSetOut:
return ShotTaskSetOut.model_validate(task_set)
def _task_set_to_detail_out(task_set: ShotReplicateTaskSet) -> ShotTaskSetDetailOut:
suggestions = [ShotAISuggestionOut(**{k: v for k, v in item.items() if k != "raw"}) for item in _normalize_suggestions(task_set.ai_suggestion_json)]
base = ShotTaskSetDetailOut.model_validate(task_set)
base.ai_suggestions = suggestions
return base
def _segment_to_out(segment: ShotReplicateSegment) -> ShotSegmentOut:
data = ShotSegmentOut.model_validate(segment)
data.segment_name = f"片段{segment.segment_index}"
return data
def _segment_to_detail_out(segment: ShotReplicateSegment) -> ShotSegmentDetailOut:
data = ShotSegmentDetailOut.model_validate(segment)
data.segment_name = f"片段{segment.segment_index}"
return data
async def get_task_set_for_user(
db: AsyncSession,
*,
task_set_id: str,
user: User,
for_update: bool = False,
) -> ShotReplicateTaskSet:
query = select(ShotReplicateTaskSet).where(
ShotReplicateTaskSet.id == task_set_id,
ShotReplicateTaskSet.deleted_at.is_(None),
)
if not user.is_admin:
query = query.where(ShotReplicateTaskSet.user_id == user.id)
if for_update:
query = query.with_for_update()
result = await db.execute(query.limit(1))
task_set = result.scalar_one_or_none()
if not task_set:
raise HTTPException(status_code=404, detail="拆镜总任务集不存在")
return task_set
async def get_segment_for_user(
db: AsyncSession,
*,
segment_id: str,
user: User,
for_update: bool = False,
) -> ShotReplicateSegment:
query = select(ShotReplicateSegment).where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
)
if not user.is_admin:
query = query.where(ShotReplicateSegment.user_id == user.id)
if for_update:
query = query.with_for_update()
result = await db.execute(query.limit(1))
segment = result.scalar_one_or_none()
if not segment:
raise HTTPException(status_code=404, detail="拆镜片段不存在")
return segment
async def create_task_set(db: AsyncSession, *, current_user: User, req: ShotTaskSetCreate) -> ShotReplicateTaskSet:
if req.idempotency_key:
existing_result = await db.execute(
select(ShotReplicateTaskSet).where(
ShotReplicateTaskSet.user_id == current_user.id,
ShotReplicateTaskSet.idempotency_key == req.idempotency_key,
ShotReplicateTaskSet.deleted_at.is_(None),
).limit(1)
)
existing = existing_result.scalar_one_or_none()
if existing:
return existing
asset = validate_upload_video_asset(req.video_url, req.video_duration_seconds)
task_set = ShotReplicateTaskSet(
id=generate_id(),
user_id=current_user.id,
title=req.title or "拆镜复刻任务",
video_url=asset.url,
video_path=str(asset.path),
video_duration_seconds=asset.duration_seconds,
status=ShotTaskSetStatusEnum.PENDING_ANALYSIS.value,
analysis_status=ShotAnalysisStatusEnum.PENDING.value,
split_status=ShotSplitStatusEnum.NONE.value,
segment_count=0,
completed_segment_count=0,
failed_segment_count=0,
idempotency_key=req.idempotency_key,
)
db.add(task_set)
await db.flush()
log_module_event_file(
module=MODULE,
event_type="SHOT_TASK_SET_CREATED",
project_id=task_set.id,
user_id=task_set.user_id,
message="创建拆镜总任务集",
detail={
"task_set_id": task_set.id,
"title": task_set.title,
"video_url": task_set.video_url,
"video_path": task_set.video_path,
"video_duration_seconds": task_set.video_duration_seconds,
"idempotency_key": task_set.idempotency_key,
},
)
return task_set
async def list_task_sets(
db: AsyncSession,
*,
current_user: User,
status: str | None = None,
analysis_status: str | None = None,
split_status: str | None = None,
keyword: str | None = None,
page: int = 1,
page_size: int = 20,
) -> ShotTaskSetListOut:
query = select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.deleted_at.is_(None))
if not current_user.is_admin:
query = query.where(ShotReplicateTaskSet.user_id == current_user.id)
if status:
query = query.where(ShotReplicateTaskSet.status == status)
if analysis_status:
query = query.where(ShotReplicateTaskSet.analysis_status == analysis_status)
if split_status:
query = query.where(ShotReplicateTaskSet.split_status == split_status)
if keyword:
like = f"%{keyword.strip()}%"
query = query.where(
(ShotReplicateTaskSet.title.ilike(like))
| (ShotReplicateTaskSet.original_video_content.ilike(like))
| (ShotReplicateTaskSet.original_video_category.ilike(like))
)
total_result = await db.execute(select(func.count()).select_from(query.subquery()))
total = int(total_result.scalar() or 0)
rows = await db.execute(
query.order_by(ShotReplicateTaskSet.created_at.desc())
.offset((page - 1) * page_size)
.limit(page_size)
)
return ShotTaskSetListOut(total=total, page=page, page_size=page_size, items=[_task_set_to_out(item) for item in rows.scalars().all()])
async def task_set_detail(db: AsyncSession, *, current_user: User, task_set_id: str) -> ShotTaskSetDetailOut:
task_set = await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user)
return _task_set_to_detail_out(task_set)
async def _next_segment_index(db: AsyncSession, task_set_id: str) -> int:
result = await db.execute(
select(func.max(ShotReplicateSegment.segment_index)).where(
ShotReplicateSegment.task_set_id == task_set_id,
ShotReplicateSegment.deleted_at.is_(None),
)
)
return int(result.scalar() or 0) + 1
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
task_set_result = await db.execute(select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.id == task_set_id).with_for_update().limit(1))
task_set = task_set_result.scalar_one_or_none()
if not task_set:
return
result = await db.execute(
select(ShotReplicateSegment).where(
ShotReplicateSegment.task_set_id == task_set_id,
ShotReplicateSegment.deleted_at.is_(None),
)
)
segments = list(result.scalars().all())
total = len(segments)
completed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.COMPLETED.value])
failed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.FAILED.value])
task_set.segment_count = total
task_set.completed_segment_count = completed
task_set.failed_segment_count = failed
if total <= 0:
task_set.split_status = ShotSplitStatusEnum.NONE.value
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
return
old_status = task_set.status
old_split_status = task_set.split_status
if completed == total:
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
elif failed == total:
task_set.split_status = ShotSplitStatusEnum.FAILED.value
task_set.status = ShotTaskSetStatusEnum.FAILED.value
elif failed > 0:
task_set.split_status = ShotSplitStatusEnum.FAILED.value
task_set.status = ShotTaskSetStatusEnum.PARTIAL_FAILED.value
else:
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
if old_status != task_set.status or old_split_status != task_set.split_status:
log_module_event_file(
module=MODULE,
event_type="SHOT_SPLIT_STATUS_CHANGED",
project_id=task_set.id,
user_id=task_set.user_id,
message="拆镜总任务集拆分状态变更",
detail={
"task_set_id": task_set.id,
"from_status": old_status,
"to_status": task_set.status,
"from_split_status": old_split_status,
"to_split_status": task_set.split_status,
"segment_count": total,
"completed_segment_count": completed,
"failed_segment_count": failed,
},
)
async def create_segments_by_ai(
db: AsyncSession,
*,
current_user: User,
task_set_id: str,
req: ShotSplitByAIRequest,
) -> ShotSplitByAIOut:
task_set = await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user, for_update=True)
if task_set.analysis_status != ShotAnalysisStatusEnum.COMPLETED.value:
raise HTTPException(status_code=400, detail="原视频分析未完成，不能按 AI 建议拆镜")
suggestions = _normalize_suggestions(task_set.ai_suggestion_json)
if not suggestions:
raise HTTPException(status_code=400, detail="当前没有可用 AI 建议拆镜方案，请使用自定义拆镜")
if req.selected_indices:
selected_set = {int(x) for x in req.selected_indices}
suggestions = [item for item in suggestions if int(item["index"]) in selected_set]
if not suggestions:
raise HTTPException(status_code=400, detail="selected_indices 没有匹配到可用 AI 建议")
old_result = await db.execute(
select(ShotReplicateSegment).where(
ShotReplicateSegment.task_set_id == task_set.id,
ShotReplicateSegment.source_mode == ShotSegmentSourceModeEnum.AI_SUGGESTION.value,
ShotReplicateSegment.deleted_at.is_(None),
)
)
old_segments = list(old_result.scalars().all())
if old_segments and not req.replace_existing:
raise HTTPException(status_code=409, detail="已存在 AI 建议拆镜片段，如需重拆请传 replace_existing=true")
if old_segments and req.replace_existing:
now = _now()
for segment in old_segments:
segment.deleted_at = now
created: list[ShotReplicateSegment] = []
next_index = await _next_segment_index(db, task_set.id)
for item in suggestions:
start, end, duration = validate_split_range(
start_second=item["start_second"],
end_second=item["end_second"],
video_duration_seconds=task_set.video_duration_seconds,
)
segment = ShotReplicateSegment(
id=generate_id(),
task_set_id=task_set.id,
user_id=task_set.user_id,
segment_index=next_index,
source_mode=ShotSegmentSourceModeEnum.AI_SUGGESTION.value,
start_second=start,
end_second=end,
duration_seconds=duration,
time_node=build_time_node(start, end),
split_status=ShotSplitStatusEnum.PENDING.value,
analysis_status=ShotSegmentAnalysisStatusEnum.NOT_REQUIRED.value,
replicate_status=ShotSegmentReplicateStatusEnum.NOT_STARTED.value,
original_video_content=task_set.original_video_content,
original_video_category=task_set.original_video_category,
original_video_audience=task_set.original_video_audience,
segment_content=item.get("content"),
segment_category=item.get("category"),
segment_audience=item.get("audience"),
ai_suggestion_json=item.get("raw") or item,
split_enqueued_at=_now(),
split_celery_task_id=f"shot-split:{uuid.uuid4().hex}",
)
db.add(segment)
created.append(segment)
next_index += 1
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.flush()
await refresh_task_set_split_summary(db, task_set.id)
await db.flush()
log_module_event_file(
module=MODULE,
event_type="SHOT_SPLIT_BY_AI_SUBMITTED",
project_id=task_set.id,
user_id=task_set.user_id,
message="按 AI 建议创建拆镜片段",
detail={
"task_set_id": task_set.id,
"selected_indices": req.selected_indices,
"replace_existing": req.replace_existing,
"created_segment_count": len(created),
"segment_ids": [segment.id for segment in created],
},
)
return ShotSplitByAIOut(
task_set_id=task_set.id,
status=task_set.status,
split_status=task_set.split_status,
created_segment_count=len(created),
segments=[_segment_to_out(segment) for segment in created],
)
async def create_custom_segment(
db: AsyncSession,
*,
current_user: User,
task_set_id: str,
req: ShotSplitCustomRequest,
) -> ShotSplitCustomOut:
task_set = await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user, for_update=True)
start, end, duration = validate_split_range(
start_second=req.start_second,
end_second=req.end_second,
video_duration_seconds=task_set.video_duration_seconds,
)
next_index = await _next_segment_index(db, task_set.id)
segment = ShotReplicateSegment(
id=generate_id(),
task_set_id=task_set.id,
user_id=task_set.user_id,
segment_index=next_index,
source_mode=ShotSegmentSourceModeEnum.CUSTOM.value,
start_second=start,
end_second=end,
duration_seconds=duration,
time_node=build_time_node(start, end),
split_status=ShotSplitStatusEnum.PENDING.value,
analysis_status=ShotSegmentAnalysisStatusEnum.PENDING.value,
replicate_status=ShotSegmentReplicateStatusEnum.NOT_STARTED.value,
split_enqueued_at=_now(),
split_celery_task_id=f"shot-split:{uuid.uuid4().hex}",
)
db.add(segment)
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.flush()
await refresh_task_set_split_summary(db, task_set.id)
await db.flush()
log_module_event_file(
module=MODULE,
event_type="SHOT_SPLIT_CUSTOM_SUBMITTED",
project_id=task_set.id,
step_id=segment.id,
user_id=task_set.user_id,
message="按用户自定义时间创建拆镜片段",
detail={
"task_set_id": task_set.id,
"segment_id": segment.id,
"start_second": start,
"end_second": end,
"duration_seconds": duration,
"time_node": segment.time_node,
},
)
return ShotSplitCustomOut(task_set_id=task_set.id, segment=_segment_to_out(segment))
async def enqueue_segment_split(segment_id: str, *, countdown: int | None = None, recover: bool = False) -> None:
if not celery_app:
return
from app.tasks.shot_replicate_tasks import split_one_segment
split_one_segment.apply_async(
args=[segment_id],
queue="gen_result_download",
countdown=countdown,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
)
async def list_segments(
db: AsyncSession,
*,
current_user: User,
task_set_id: str,
source_mode: str | None = None,
split_status: str | None = None,
analysis_status: str | None = None,
replicate_status: str | None = None,
page: int = 1,
page_size: int = 20,
) -> ShotSegmentListOut:
await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user)
query = select(ShotReplicateSegment).where(
ShotReplicateSegment.task_set_id == task_set_id,
ShotReplicateSegment.deleted_at.is_(None),
)
if not current_user.is_admin:
query = query.where(ShotReplicateSegment.user_id == current_user.id)
if source_mode:
query = query.where(ShotReplicateSegment.source_mode == source_mode)
if split_status:
query = query.where(ShotReplicateSegment.split_status == split_status)
if analysis_status:
query = query.where(ShotReplicateSegment.analysis_status == analysis_status)
if replicate_status:
query = query.where(ShotReplicateSegment.replicate_status == replicate_status)
total_result = await db.execute(select(func.count()).select_from(query.subquery()))
total = int(total_result.scalar() or 0)
rows = await db.execute(
query.order_by(ShotReplicateSegment.segment_index.asc())
.offset((page - 1) * page_size)
.limit(page_size)
)
return ShotSegmentListOut(total=total, page=page, page_size=page_size, items=[_segment_to_out(item) for item in rows.scalars().all()])
async def segment_detail(db: AsyncSession, *, current_user: User, segment_id: str) -> ShotSegmentDetailOut:
segment = await get_segment_for_user(db, segment_id=segment_id, user=current_user)
return _segment_to_detail_out(segment)
video-gen-api/app/services/shot_video_analysis_service.py
+521
View File
@@ -0,0 +1,521 @@
from __future__ import annotations
import base64
import json
import mimetypes
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal
import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.models.model_config import ModelConfig
from app.models.token_usage import TokenUsage
from app.services.upload_video_asset_service import resolve_upload_video_path
from app.services.resource_signed_url_service import build_resource_signed_url
from app.utils.id_gen import generate_id
AnalysisMode = Literal["full_breakdown", "summary_only"]
@dataclass(slots=True)
class ShotVideoAnalysisResult:
result: dict[str, Any]
raw_response: dict[str, Any]
usage: dict[str, Any]
def _timeout_seconds() -> int:
return int(getattr(settings, "SHOT_ANALYSIS_TIMEOUT_SECONDS", 180) or 180)
def _video_fps() -> float:
return float(getattr(settings, "SHOT_ANALYSIS_VIDEO_FPS", 1.0) or 1.0)
def _split_min_seconds() -> float:
return float(getattr(settings, "SHOT_SPLIT_MIN_SECONDS", 1) or 1)
def _split_max_seconds() -> float:
return float(getattr(settings, "SHOT_SPLIT_MAX_SECONDS", 120) or 120)
def _resolve_local_file_path(file_url: str) -> str:
if file_url.startswith("/uploads/") or file_url.startswith("uploads/"):
return str(resolve_upload_video_path(file_url))
return file_url
def build_file_url_or_data_uri(file_url: str, fallback_mime: str = "video/mp4") -> str:
if file_url.startswith(("http://", "https://", "data:")):
return file_url
file_url_sign = build_resource_signed_url(resource_url=file_url, expire_seconds=86400)
return f"{settings.BASE_URL}{file_url_sign}"
# file_path = _resolve_local_file_path(file_url)
# path = Path(file_path)
# if not path.exists():
# raise FileNotFoundError(f"视频文件不存在: {file_path}")
#
# max_mb = float(getattr(settings, "SHOT_ANALYSIS_MAX_LOCAL_VIDEO_MB", 45) or 45)
# size_mb = path.stat().st_size / 1024 / 1024
# if size_mb > max_mb:
# raise ValueError(f"本地视频文件过大: {size_mb:.2f} MB，当前限制 {max_mb:g} MB")
#
# mime = mimetypes.guess_type(str(path))[0] or fallback_mime
# with open(path, "rb") as f:
# b64 = base64.b64encode(f.read()).decode("utf-8")
# return f"data:{mime};base64,{b64}"
def build_user_message(user_text: str, video_url: str) -> tuple[dict[str, Any], dict[str, Any]]:
real_url = build_file_url_or_data_uri(video_url)
content_parts = [
{
"type": "video_url",
"video_url": {
"url": real_url,
"fps": _video_fps(),
},
},
{"type": "text", "text": user_text},
]
log_content_parts = [
{
"type": "video_url",
"video_url": {
"url": video_url,
"fps": _video_fps(),
},
},
{"type": "text", "text": user_text},
]
return {"role": "user", "content": content_parts}, {"role": "user", "content": log_content_parts}
def build_video_analysis_system_prompt(*, mode: AnalysisMode) -> str:
if mode == "summary_only":
return """
你是专业的短视频内容分析师、广告素材拆解师。
你的任务：
1. 根据用户提供的视频附件，分析这个视频片段的整体内容。
2. 判断视频分类。
3. 判断视频可能面向的受众人群。
4. 必须输出严格 JSON 对象。
5. 不输出 Markdown。
6. 不输出解释文字。
7. 不返回 null，未知内容填“无”。
顶级字段只能包含：
- 原视频内容
- 原视频分类
- 原视频受众人群
- 拆镜内容剖析
summary_only 模式下“拆镜内容剖析”必须返回空数组。
安全规则：
1. 不要识别视频中人物身份。
2. 不要猜测真实姓名、联系方式、账号身份。
3. 如果视频是游戏录屏，只分析画面内容、玩法内容、玩家情绪表达、受众，不要编造不存在的剧情。
4. 如果视频包含广告内容，可以分析广告品类、目标用户、转化意图，但不要编造品牌信息。
""".strip()
return f"""
你是专业的短视频内容分析师、广告素材拆解师、视频分镜分析师。
你的任务：
1. 根据用户提供的视频附件，分析原视频整体内容。
2. 判断原视频分类。
3. 判断原视频可能面向的受众人群。
4. 对视频进行拆镜内容剖析。
5. 必须输出严格 JSON 对象。
6. 不输出 Markdown。
7. 不输出解释文字。
8. 不返回 null，未知内容填“无”。
顶级字段只能包含：
- 原视频内容
- 原视频分类
- 原视频受众人群
- 拆镜内容剖析
拆镜内容剖析必须是数组。
每个拆镜片段必须包含：
- 拆镜开始秒
- 拆镜结束秒
- 拆镜时间节点
- 对应时间节点内的内容
- 分类
- 受众人群
拆镜时间规则：
1. 拆镜开始秒必须是数字，例如 0、15、26。
2. 拆镜结束秒必须是数字，例如 15、26、31。
3. 拆镜时间节点必须由拆镜开始秒和拆镜结束秒组成，例如“0-15秒”。
4. 禁止输出“-15秒”这种缺少开始秒的时间节点。
5. 禁止输出“15-秒”这种缺少结束秒的时间节点。
6. 每个拆镜片段时长不能低于 {_split_min_seconds():g} 秒。
7. 每个拆镜片段时长不能高于 {_split_max_seconds():g} 秒。
8. 如果某段内容不足 {_split_min_seconds():g} 秒，不要单独拆出来。
9. 如果单个连续内容超过 {_split_max_seconds():g} 秒，需要按语义变化继续拆分。
10. 如果没有明显镜头变化、场景变化、人物动作变化、剧情变化、字幕重点变化或语义变化，不要强行剖析。
11. 如果无法可靠拆镜，则“拆镜内容剖析”返回空数组。
12. 拆镜时间必须从 0 秒或视频中实际可识别的开始时间开始，不允许出现负数。
13. 拆镜结束秒必须大于拆镜开始秒。
14. 拆镜片段必须按时间顺序排列。
安全规则：
1. 不要识别视频中人物身份。
2. 不要猜测真实姓名、联系方式、账号身份。
3. 如果视频是游戏录屏，只分析画面内容、玩法内容、玩家情绪表达、受众，不要编造不存在的剧情。
4. 如果视频包含广告内容，可以分析广告品类、目标用户、转化意图，但不要编造品牌信息。
""".strip()
def build_video_analysis_user_text(*, mode: AnalysisMode) -> str:
if mode == "summary_only":
payload = {
"任务": "请根据上传的视频片段附件，返回这个视频片段的内容分析 JSON。",
"输出JSON格式": {
"原视频内容": "概括这个视频片段整体内容，描述主要画面、主体、场景、动作、剧情或信息点",
"原视频分类": "判断视频类型，例如：游戏视频、产品广告视频、剧情视频、口播讲解视频、教程视频、生活记录视频等",
"原视频受众人群": "判断该片段更适合的人群",
"拆镜内容剖析": [],
},
"返回要求": ["只返回 JSON 对象", "不要返回 Markdown", "不要返回解释文字", "不要返回代码块", "不要返回 null，未知填无"],
}
return json.dumps(payload, ensure_ascii=False)
payload = {
"任务": "请根据上传的视频附件，返回原视频内容分析和拆镜内容剖析 JSON。",
"输出JSON格式": {
"原视频内容": "概括原视频整体内容，描述主要画面、人物/主体、场景、动作、剧情或信息点",
"原视频分类": "判断视频类型，例如：游戏视频、产品广告视频、剧情视频、口播讲解视频、教程视频、生活记录视频、直播切片视频、图文快闪视频等",
"原视频受众人群": "判断该视频更适合的人群，例如：游戏玩家、年轻娱乐用户、潜在购买用户、同城社交用户等",
"拆镜内容剖析": [
{
"拆镜开始秒": 0,
"拆镜结束秒": 15,
"拆镜时间节点": "0-15秒",
"对应时间节点内的内容": "描述这个时间片段内发生了什么",
"分类": "判断这个片段的内容分类，例如：开场吸引、冲突铺垫、玩法展示、卖点展示、情绪爆发、行动引导、结果展示等",
"受众人群": "判断这个片段主要吸引的人群",
}
],
},
"拆镜规则": [
f"每个拆镜片段时长必须大于等于 {_split_min_seconds():g} 秒",
f"每个拆镜片段时长必须小于等于 {_split_max_seconds():g} 秒",
"拆镜开始秒必须是数字",
"拆镜结束秒必须是数字",
"拆镜开始秒不能是负数",
"拆镜结束秒必须大于拆镜开始秒",
"拆镜时间节点必须等于：拆镜开始秒-拆镜结束秒秒",
"禁止输出“-15秒”",
"禁止输出“15-秒”",
"如果无法判断拆镜节点，拆镜内容剖析返回空数组",
],
"返回要求": ["只返回 JSON 对象", "不要返回 Markdown", "不要返回解释文字", "不要返回代码块", "不要返回 null，未知填无"],
}
return json.dumps(payload, ensure_ascii=False)
def strip_json_code_fence(text: str) -> str:
text = text.strip()
if text.startswith("```"):
text = re.sub(r"^```(?:json)?", "", text, flags=re.IGNORECASE).strip()
text = re.sub(r"```$", "", text).strip()
return text
def parse_model_json(content: str) -> dict[str, Any]:
cleaned = strip_json_code_fence(content)
data = json.loads(cleaned)
if not isinstance(data, dict):
raise ValueError(f"模型返回 JSON 不是对象类型: {type(data).__name__}")
return data
def get_message_content_or_raise(data: dict[str, Any]) -> str:
choices = data.get("choices") or []
if not choices:
raise RuntimeError(f"模型响应没有 choices: {json.dumps(data, ensure_ascii=False)}")
choice = choices[0]
finish_reason = choice.get("finish_reason")
if finish_reason == "length":
usage = data.get("usage", {})
raise RuntimeError(f"模型输出被长度限制截断，finish_reason={finish_reason}, usage={json.dumps(usage, ensure_ascii=False)}")
message = choice.get("message") or {}
content = message.get("content", "")
if not content:
raise RuntimeError(f"模型响应 content 为空: {json.dumps(data, ensure_ascii=False)}")
return content.strip()
def fill_none_with_wu(value: Any) -> Any:
if value is None:
return "无"
if isinstance(value, str):
return value if value.strip() else "无"
if isinstance(value, list):
return [fill_none_with_wu(item) for item in value]
if isinstance(value, dict):
return {k: fill_none_with_wu(v) for k, v in value.items()}
return value
def first_present(data: dict[str, Any], keys: list[str]) -> Any:
for key in keys:
if key in data:
return data.get(key)
return None
def parse_number(value: Any) -> float | None:
if isinstance(value, (int, float)):
return float(value)
text = str(value or "").strip()
if not text:
return None
m = re.search(r"-?\d+(?:\.\d+)?", text)
return float(m.group(0)) if m else None
def parse_time_value_to_seconds(value: str) -> float | None:
value = str(value or "").strip()
if not value:
return None
m = re.match(r"^(\d+(?:\.\d+)?)\s*(?:秒|s)?$", value, flags=re.IGNORECASE)
if m:
return float(m.group(1))
parts = value.split(":")
if len(parts) in (2, 3) and all(re.match(r"^\d+(?:\.\d+)?$", p.strip()) for p in parts):
nums = [float(p.strip()) for p in parts]
if len(nums) == 2:
minute, second = nums
return minute * 60 + second
hour, minute, second = nums
return hour * 3600 + minute * 60 + second
return None
def parse_time_node_to_range(time_node: str) -> tuple[float, float] | None:
text = str(time_node or "").strip()
if not text:
return None
text = text.replace("—", "-").replace("–", "-").replace("－", "-")
text = text.replace("到", "-").replace("至", "-").replace("~", "-").replace("～", "-")
text = text.replace("第", "").replace("时间段", "").replace("：", ":")
m = re.search(r"(\d{1,2}:\d{1,2}(?::\d{1,2})?)\s*-\s*(\d{1,2}:\d{1,2}(?::\d{1,2})?)", text)
if m:
start = parse_time_value_to_seconds(m.group(1))
end = parse_time_value_to_seconds(m.group(2))
if start is not None and end is not None and end > start:
return start, end
m = re.search(r"(\d+(?:\.\d+)?)\s*(?:秒|s)?\s*-\s*(\d+(?:\.\d+)?)\s*(?:秒|s)?", text, flags=re.IGNORECASE)
if m:
start = float(m.group(1))
end = float(m.group(2))
if end > start:
return start, end
m = re.search(r"^\s*-\s*(\d+(?:\.\d+)?)\s*(?:秒|s)?\s*$", text, flags=re.IGNORECASE)
if m:
end = float(m.group(1))
if end > 0:
return 0.0, end
return None
def format_second(value: float) -> int | float:
checked = float(value)
return int(checked) if checked.is_integer() else round(checked, 2)
def normalize_time_node_by_range(start: float, end: float) -> str:
return f"{format_second(start)}-{format_second(end)}秒"
def ensure_result_schema(result: dict[str, Any]) -> dict[str, Any]:
final_result = {
"原视频内容": result.get("原视频内容", "无"),
"原视频分类": result.get("原视频分类", "无"),
"原视频受众人群": result.get("原视频受众人群", "无"),
"拆镜内容剖析": result.get("拆镜内容剖析", []),
}
for key in ("原视频内容", "原视频分类", "原视频受众人群"):
if not isinstance(final_result[key], str):
final_result[key] = json.dumps(final_result[key], ensure_ascii=False)
if not isinstance(final_result["拆镜内容剖析"], list):
final_result["拆镜内容剖析"] = []
return final_result
def filter_and_normalize_breakdown(result: dict[str, Any], *, mode: AnalysisMode = "full_breakdown") -> dict[str, Any]:
if mode == "summary_only":
result["拆镜内容剖析"] = []
return result
breakdown = result.get("拆镜内容剖析")
if not isinstance(breakdown, list):
result["拆镜内容剖析"] = []
return result
normalized_items: list[dict[str, Any]] = []
for item in breakdown:
if not isinstance(item, dict):
continue
raw_start = first_present(item, ["拆镜开始秒", "开始秒", "起始秒", "开始时间", "起始时间", "start", "start_second", "start_seconds"])
raw_end = first_present(item, ["拆镜结束秒", "结束秒", "结束时间", "end", "end_second", "end_seconds"])
start = parse_number(raw_start)
end = parse_number(raw_end)
time_node = str(first_present(item, ["拆镜时间节点", "时间节点", "时间段", "镜头时间", "time_node", "time_range"]) or "").strip()
if start is None or end is None:
parsed_range = parse_time_node_to_range(time_node)
if parsed_range is None:
continue
start, end = parsed_range
if start is None or end is None or start < 0 or end < 0 or end <= start:
continue
duration = end - start
if duration < _split_min_seconds() or duration > _split_max_seconds():
continue
content = first_present(item, ["对应时间节点内的内容", "内容", "画面内容", "片段内容", "镜头内容", "content"]) or "无"
category = first_present(item, ["分类", "片段分类", "内容分类", "镜头分类", "category"]) or "无"
audience = first_present(item, ["受众人群", "目标受众", "片段受众", "镜头受众", "audience"]) or "无"
normalized_items.append({
"拆镜开始秒": format_second(start),
"拆镜结束秒": format_second(end),
"拆镜时间节点": normalize_time_node_by_range(start, end),
"对应时间节点内的内容": str(content or "无"),
"分类": str(category or "无"),
"受众人群": str(audience or "无"),
})
normalized_items.sort(key=lambda x: float(x.get("拆镜开始秒", 0)))
result["拆镜内容剖析"] = normalized_items
return result
async def _select_model_config(db: AsyncSession) -> ModelConfig | None:
result = await db.execute(
select(ModelConfig)
.where(ModelConfig.is_active == True)
.order_by(ModelConfig.priority.desc())
.limit(1)
)
return result.scalar_one_or_none()
def _int_usage(value: Any) -> int:
try:
return int(value or 0)
except Exception:
return 0
async def analyze_video_for_shot_split(
db: AsyncSession,
video_url: str,
*,
user_id: str | None = None,
mode: AnalysisMode = "full_breakdown",
) -> ShotVideoAnalysisResult:
"""调用模型完成拆镜/片段分析。
模型配置统一从 model_configs 表选择当前启用且 priority 最高的配置；
不再读取 SHOT_ANALYSIS_API_BASE / SHOT_ANALYSIS_API_KEY / SHOT_ANALYSIS_MODEL_NAME，
也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。
"""
config = await _select_model_config(db)
if not config:
raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型")
if not str(config.api_key or "").strip():
raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}")
if not str(config.api_base or "").strip():
raise RuntimeError(f"拆镜分析模型 API Base 为空: model_config_id={config.id}")
if not str(config.model_name or "").strip():
raise RuntimeError(f"拆镜分析模型名称为空: model_config_id={config.id}")
system_prompt = build_video_analysis_system_prompt(mode=mode)
user_text = build_video_analysis_user_text(mode=mode)
user_message, log_user_message = build_user_message(user_text, video_url)
request_data: dict[str, Any] = {
"model": config.model_name,
"messages": [
{"role": "system", "content": system_prompt},
user_message,
],
"max_tokens": int(getattr(settings, "SHOT_ANALYSIS_MAX_TOKENS", 5000) or getattr(config, "max_tokens", 5000) or 5000),
"temperature": float(getattr(settings, "SHOT_ANALYSIS_TEMPERATURE", 0.1) or getattr(config, "temperature", 0.1) or 0.1),
"response_format": {"type": "json_object"},
}
log_request_data: dict[str, Any] = {
**request_data,
"messages": [
{"role": "system", "content": system_prompt},
log_user_message,
],
"model_config_id": config.id,
"model_config_name": config.name,
"provider": config.provider,
"analysis_mode": mode,
}
url = f"{str(config.api_base).rstrip('/')}/chat/completions"
async with httpx.AsyncClient(timeout=_timeout_seconds()) as client:
response = await client.post(
url,
headers={"Authorization": f"Bearer {config.api_key}", "Content-Type": "application/json"},
json=request_data,
)
if response.status_code >= 400:
raise RuntimeError(f"视频拆镜分析 API 请求失败: HTTP {response.status_code}: {response.text}")
raw = response.json()
content = get_message_content_or_raise(raw)
result = parse_model_json(content)
result = fill_none_with_wu(result)
result = ensure_result_schema(result)
result = filter_and_normalize_breakdown(result, mode=mode)
usage = raw.get("usage") or {}
token_usage = {
"input_tokens": _int_usage(usage.get("prompt_tokens") or usage.get("input_tokens")),
"output_tokens": _int_usage(usage.get("completion_tokens") or usage.get("output_tokens")),
"total_tokens": _int_usage(usage.get("total_tokens")),
"finish_reason": ((raw.get("choices") or [{}])[0] or {}).get("finish_reason"),
"model_config_id": config.id,
"model_config_name": config.name,
"model": config.model_name,
"provider": config.provider,
"video_fps": _video_fps(),
"split_min_seconds": _split_min_seconds(),
"split_max_seconds": _split_max_seconds(),
"analysis_mode": mode,
"log_request": log_request_data,
}
if not token_usage["total_tokens"]:
token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"]
db.add(
TokenUsage(
id=generate_id(),
model_config_id=config.id,
user_id=user_id,
input_tokens=token_usage["input_tokens"],
output_tokens=token_usage["output_tokens"],
total_tokens=token_usage["total_tokens"],
)
)
await db.flush()
return ShotVideoAnalysisResult(result=result, raw_response=raw, usage=token_usage)
video-gen-api/app/services/shot_video_split_service.py
+173
View File
@@ -0,0 +1,173 @@
from __future__ import annotations
import asyncio
import os
import subprocess
from dataclasses import dataclass
from pathlib import Path
from app.config import settings
from app.services.upload_video_asset_service import (
build_upload_url_from_path,
ensure_shot_segment_dir,
get_ffmpeg_bin,
)
@dataclass(slots=True)
class ShotSplitResult:
url: str
path: str
file_size_bytes: int
def _date_dir_from_segment_id(segment_id: str) -> str:
# 由调用方更适合按 created_at 传入；这里兜底按当前日期。
from datetime import datetime
return datetime.now().strftime("%Y/%m/%d")
def _safe_unlink(path: Path) -> None:
try:
if path.exists():
path.unlink()
except Exception:
pass
def split_video_segment(
*,
source_path: str | Path,
segment_id: str,
start_second: float,
end_second: float,
date_dir: str | None = None,
) -> ShotSplitResult:
"""使用 ffmpeg 拆出单个视频片段，输出到 storage/uploads/shot_segments。"""
source_path = Path(source_path)
if not source_path.exists():
raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}")
start = max(float(start_second), 0.0)
end = max(float(end_second), 0.0)
duration = end - start
if duration <= 0:
raise RuntimeError(
f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}"
)
date_dir = date_dir or _date_dir_from_segment_id(segment_id)
output_dir = ensure_shot_segment_dir(date_dir)
output_path = output_dir / f"{segment_id}.mp4"
# 注意：
# 不能用 xxx.mp4.part，因为 ffmpeg 会按最后一个扩展名 .part 判断输出格式，导致：
# Unable to choose an output format
# 这里改为 xxx.part.mp4，让 ffmpeg 能识别 mp4 容器。
part_path = output_dir / f"{segment_id}.part.mp4"
_safe_unlink(part_path)
timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120)
cmd = [
get_ffmpeg_bin(),
"-y",
# 先 seek 到起始秒，再按 duration 切割，避免 -to 在不同 ffmpeg 参数位置下语义不一致。
"-ss",
f"{start:.3f}",
"-i",
str(source_path),
"-t",
f"{duration:.3f}",
# 只取主视频流，音频可选，避免 map 0 把字幕/数据流带进去导致 mp4 封装失败。
"-map",
"0:v:0",
"-map",
"0:a:0?",
# 当前是拆镜片段，重编码更稳，避免关键帧不准导致片段首尾异常。
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"23",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
"-b:a",
"128k",
"-movflags",
"+faststart",
# 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。
"-f",
"mp4",
str(part_path),
]
try:
completed = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired as exc:
_safe_unlink(part_path)
raise RuntimeError(
f"ffmpeg 拆镜超时：timeout={timeout}s, start={start:.3f}, end={end:.3f}"
) from exc
if completed.returncode != 0:
_safe_unlink(part_path)
raise RuntimeError(
f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}"
)
if not part_path.exists() or part_path.stat().st_size <= 0:
_safe_unlink(part_path)
raise RuntimeError("ffmpeg 拆镜失败：输出文件为空")
os.replace(part_path, output_path)
return ShotSplitResult(
url=build_upload_url_from_path(output_path),
path=str(output_path),
file_size_bytes=output_path.stat().st_size,
)
async def split_video_segment_async(
*,
source_path: str | Path,
segment_id: str,
start_second: float,
end_second: float,
date_dir: str | None = None,
) -> ShotSplitResult:
"""异步拆镜入口。
ffmpeg 本身是同步阻塞命令，不能直接在 Celery 进程内唯一 event loop 中执行。
这里通过 asyncio.to_thread 跑同步拆镜函数，避免阻塞 asyncpg / Redis / HTTP 等异步任务。
"""
return await asyncio.to_thread(
split_video_segment,
source_path=source_path,
segment_id=segment_id,
start_second=start_second,
end_second=end_second,
date_dir=date_dir,
)
video-gen-api/app/services/upload_video_asset_service.py
+211
View File
@@ -0,0 +1,211 @@
from __future__ import annotations
import json
import os
import subprocess
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlparse
from fastapi import HTTPException
from app.config import settings
VIDEO_EXTENSIONS = {".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv"}
@dataclass(slots=True)
class UploadVideoAsset:
url: str
path: Path
duration_seconds: float
def _project_root() -> Path:
return Path.cwd()
def _abs_path(value: str | Path) -> Path:
path = Path(value)
if not path.is_absolute():
path = _project_root() / path
return path.resolve()
def upload_root() -> Path:
return _abs_path(settings.UPLOAD_LOCAL_PATH)
def shot_segment_root() -> Path:
return _abs_path(getattr(settings, "SHOT_SEGMENT_LOCAL_PATH", "./storage/uploads/shot_segments"))
def _strip_base_url(url: str) -> str:
text = str(url or "").strip()
if not text:
return text
base_url = str(getattr(settings, "BASE_URL", "") or "").strip().rstrip("/")
if base_url and text.startswith(base_url + "/"):
return text[len(base_url):]
parsed = urlparse(text)
if parsed.scheme in ("http", "https"):
# 只接受本系统 BASE_URL 下的上传资源；外部 URL 不允许 ffmpeg 本地切片。
raise HTTPException(status_code=400, detail="拆镜源视频必须来自本系统上传接口，不能传外部 http/https URL")
return text
def _safe_relative_from_upload_url(url: str) -> str:
value = _strip_base_url(url)
value = value.split("?", 1)[0].split("#", 1)[0]
if value.startswith("/uploads/"):
rel = value.replace("/uploads/", "", 1)
elif value.startswith("uploads/"):
rel = value.replace("uploads/", "", 1)
else:
raise HTTPException(status_code=400, detail="拆镜源视频链接必须是 /uploads/ 下的上传资源")
rel = rel.lstrip("/")
if not rel or ".." in Path(rel).parts:
raise HTTPException(status_code=400, detail="上传视频路径非法")
return rel
def resolve_upload_video_path(video_url: str) -> Path:
rel = _safe_relative_from_upload_url(video_url)
root = upload_root()
path = (root / rel).resolve()
try:
path.relative_to(root)
except ValueError as exc:
raise HTTPException(status_code=400, detail="上传视频路径越界") from exc
if path.suffix.lower() not in VIDEO_EXTENSIONS:
raise HTTPException(status_code=400, detail="上传资源不是支持的视频格式")
if not path.exists() or not path.is_file():
raise HTTPException(status_code=404, detail=f"上传视频文件不存在: {video_url}")
return path
def build_upload_url_from_path(path: str | Path) -> str:
root = upload_root()
checked_path = _abs_path(path)
try:
rel = checked_path.relative_to(root).as_posix()
except ValueError as exc:
raise HTTPException(status_code=500, detail="生成上传资源 URL 失败：路径不在 uploads 目录下") from exc
return f"/uploads/{rel}"
def get_ffmpeg_bin() -> str:
return str(getattr(settings, "FFMPEG_BIN", "") or "ffmpeg")
def get_ffprobe_bin() -> str:
configured = str(getattr(settings, "FFPROBE_BIN", "") or "").strip()
if configured:
return configured
ffmpeg_bin = get_ffmpeg_bin()
if ffmpeg_bin.endswith("ffmpeg.exe"):
return ffmpeg_bin[:-10] + "ffprobe.exe"
if ffmpeg_bin.endswith("ffmpeg"):
return ffmpeg_bin[:-6] + "ffprobe"
return "ffprobe"
def probe_video_duration_seconds(video_path: str | Path) -> float:
path = _abs_path(video_path)
timeout = int(getattr(settings, "SHOT_FFPROBE_TIMEOUT_SECONDS", 20) or 20)
cmd = [
get_ffprobe_bin(),
"-v", "error",
"-show_entries", "format=duration",
"-of", "json",
str(path),
]
try:
completed = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
check=False,
)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"ffprobe 获取视频时长失败: {exc}") from exc
if completed.returncode != 0:
raise HTTPException(status_code=400, detail=f"ffprobe 获取视频时长失败: {completed.stderr.strip()}")
try:
data = json.loads(completed.stdout or "{}")
duration = float((data.get("format") or {}).get("duration") or 0)
except Exception as exc:
raise HTTPException(status_code=400, detail="ffprobe 返回的视频时长无法解析") from exc
if duration <= 0:
raise HTTPException(status_code=400, detail="视频时长无效")
return round(duration, 3)
def validate_upload_video_asset(video_url: str, frontend_duration_seconds: float | None = None) -> UploadVideoAsset:
path = resolve_upload_video_path(video_url)
real_duration = probe_video_duration_seconds(path)
if frontend_duration_seconds is not None and frontend_duration_seconds > 0:
tolerance = float(getattr(settings, "SHOT_DURATION_TOLERANCE_SECONDS", 1.0) or 1.0)
# 超出误差时以后端 ffprobe 为准，不拒绝，避免前端浮点或浏览器 metadata 偏差导致创建失败。
if abs(float(frontend_duration_seconds) - real_duration) <= tolerance:
real_duration = round(float(frontend_duration_seconds), 3)
return UploadVideoAsset(url=_strip_base_url(video_url), path=path, duration_seconds=real_duration)
def validate_split_range(*, start_second: float, end_second: float, video_duration_seconds: float) -> tuple[float, float, float]:
start = round(float(start_second), 3)
end = round(float(end_second), 3)
if start < 0:
raise HTTPException(status_code=400, detail="开始秒不能小于0")
if end <= start:
raise HTTPException(status_code=400, detail="结束秒必须大于开始秒")
tolerance = float(getattr(settings, "SHOT_SPLIT_END_TOLERANCE_SECONDS", 0.5) or 0.5)
if end > float(video_duration_seconds) + tolerance:
raise HTTPException(status_code=400, detail="结束秒不能超过视频总时长")
duration = round(end - start, 3)
min_seconds = float(getattr(settings, "SHOT_SPLIT_MIN_SECONDS", 1) or 1)
max_seconds = float(getattr(settings, "SHOT_SPLIT_MAX_SECONDS", 120) or 120)
if duration < min_seconds:
raise HTTPException(status_code=400, detail=f"拆镜片段不能低于 {min_seconds:g} 秒")
if duration > max_seconds:
raise HTTPException(status_code=400, detail=f"拆镜片段不能超过 {max_seconds:g} 秒")
return start, end, duration
def format_second(value: float) -> int | float:
checked = float(value)
if checked.is_integer():
return int(checked)
return round(checked, 2)
def build_time_node(start_second: float, end_second: float) -> str:
return f"{format_second(start_second)}-{format_second(end_second)}秒"
def ensure_shot_segment_dir(date_dir: str) -> Path:
root = shot_segment_root()
output_dir = (root / date_dir).resolve()
try:
output_dir.relative_to(root)
except ValueError as exc:
raise RuntimeError("拆镜输出目录越界") from exc
os.makedirs(output_dir, exist_ok=True)
return output_dir
video-gen-api/app/tasks/__init__.py
+3 -1
View File
@@ -10,7 +10,9 @@ try:
generation_poll_tasks,
generation_download_tasks,
generation_recovery_tasks,
hot_opening_replicate_tasks
hot_opening_replicate_tasks,
shot_replicate_tasks,
shot_replicate_flow_tasks
)
except Exception:
pass
video-gen-api/app/tasks/async_runner.py
+111 -13
View File
@@ -1,23 +1,35 @@
from __future__ import annotations
import asyncio
import os
import threading
from concurrent.futures import Future
from typing import Awaitable, TypeVar
from app.config import settings
T = TypeVar("T")
_thread_local = threading.local()
_single_loop_lock = threading.RLock()
_single_loop: asyncio.AbstractEventLoop | None = None
_single_loop_thread: threading.Thread | None = None
_single_loop_pid: int | None = None
_single_loop_ready: threading.Event | None = None
def _get_or_create_loop() -> asyncio.AbstractEventLoop:
"""
给当前进程/线程维护一个长期 event loop。
def _runner_mode() -> str:
mode = str(getattr(settings, "CELERY_ASYNC_RUNNER_MODE", "single_loop") or "single_loop").strip().lower()
if mode not in {"single_loop", "direct"}:
return "single_loop"
return mode
Linux prefork：
每个 Celery 子进程通常单线程跑任务，这里相当于每个子进程一个长期 loop。
Windows -P threads：
每个线程一个 loop，但注意 asyncpg pool 仍不适合跨线程共享；
Windows threads 模式建议继续用 NullPool 或只做本地调试。
def _get_or_create_thread_local_loop() -> asyncio.AbstractEventLoop:
"""兼容旧方案：当前线程持有一个长期 event loop。
仅作为降级模式使用。长期推荐 single_loop，避免 Windows threads 下
多线程 event loop 复用 asyncpg / redis.asyncio 连接对象。
"""
pid = os.getpid()
loop = getattr(_thread_local, "loop", None)
@@ -31,16 +43,102 @@ def _get_or_create_loop() -> asyncio.AbstractEventLoop:
return loop
def _single_loop_worker(loop: asyncio.AbstractEventLoop, ready: threading.Event) -> None:
asyncio.set_event_loop(loop)
ready.set()
loop.run_forever()
pending = [task for task in asyncio.all_tasks(loop) if not task.done()]
if pending:
for task in pending:
task.cancel()
loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def _get_or_create_single_loop() -> asyncio.AbstractEventLoop:
"""获取当前 Celery 进程内唯一 async event loop。
Linux prefork：每个 Celery 子进程各自一个 loop。
Windows threads：同一 worker 进程内所有任务线程共享同一个 loop。
"""
global _single_loop, _single_loop_thread, _single_loop_pid, _single_loop_ready
pid = os.getpid()
with _single_loop_lock:
if (
_single_loop is not None
and not _single_loop.is_closed()
and _single_loop_thread is not None
and _single_loop_thread.is_alive()
and _single_loop_pid == pid
):
return _single_loop
# fork 后 pid 变化，必须丢弃父进程状态，重新创建子进程自己的 loop。
_single_loop = asyncio.new_event_loop()
_single_loop_pid = pid
_single_loop_ready = threading.Event()
_single_loop_thread = threading.Thread(
target=_single_loop_worker,
args=(_single_loop, _single_loop_ready),
name=f"celery-async-runner-{pid}",
daemon=True,
)
_single_loop_thread.start()
_single_loop_ready.wait(timeout=5)
return _single_loop
def run_async(coro: Awaitable[T]) -> T:
"""Celery 同步 task 调用异步协程的统一入口。
默认 single_loop 模式：
- 一个 Celery 子进程只有一个专用 event loop；
- 所有 asyncpg / redis.asyncio 操作都在这个 loop 内创建和使用；
- 避免 got Future attached to a different loop。
降级 direct 模式：
- 兼容旧的线程本地 loop 方案；
- 如果使用 direct，建议同时开启 CELERY_DB_USE_NULLPOOL=true。
"""
Celery 同步 task 调用异步协程的统一入口。
不使用 asyncio.run()，避免每个 task 结束时关闭 event loop。
"""
loop = _get_or_create_loop()
return loop.run_until_complete(coro)
if _runner_mode() == "direct":
loop = _get_or_create_thread_local_loop()
return loop.run_until_complete(coro)
loop = _get_or_create_single_loop()
try:
running_loop = asyncio.get_running_loop()
except RuntimeError:
running_loop = None
if running_loop is loop:
raise RuntimeError("run_async() 不能在 Celery async_runner 的事件循环内部被同步调用")
future: Future[T] = asyncio.run_coroutine_threadsafe(coro, loop)
return future.result()
def close_loop() -> None:
"""关闭当前进程内 async runner loop。"""
global _single_loop, _single_loop_thread, _single_loop_pid, _single_loop_ready
# 关闭 single_loop。
with _single_loop_lock:
loop = _single_loop
thread = _single_loop_thread
if loop is not None and not loop.is_closed() and thread is not None and thread.is_alive():
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=5)
_single_loop = None
_single_loop_thread = None
_single_loop_pid = None
_single_loop_ready = None
# 关闭 direct 降级模式的线程本地 loop。
loop = getattr(_thread_local, "loop", None)
if loop is not None and not loop.is_closed():
loop.close()
video-gen-api/app/tasks/celery_app.py
+39 -5
View File
@@ -51,6 +51,12 @@ if broker_url:
"generation.download_generation_result_task": {"queue": "gen_result_download"},
"hot_opening.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
"hot_opening.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
"shot_replicate.analyze_original_video": {"queue": "gen_chatapi_create"},
"shot_replicate.analyze_custom_segment_video": {"queue": "gen_chatapi_create"},
"shot_replicate.split_one_segment": {"queue": "gen_result_download"},
"shot_replicate.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
"shot_replicate.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
"shot_replicate.recover_split_tasks_once": {"queue": "gen_result_download"},
"generation.recover_download_tasks_once": {"queue": "gen_result_download"},
"generation.recover_generation_tasks_once": {"queue": "gen_result_download"},
"app.tasks.cleanup.*": {"queue": "default"},
@@ -61,6 +67,18 @@ else:
celery_app = None
async def _try_acquire_startup_recovery_lock() -> bool:
"""任意 worker 启动时都可尝试抢恢复锁，避免依赖 hostname 命名。"""
from app.services.redis_registry_service import redis_acquire_lock
token = await redis_acquire_lock(
lock_key=settings.CELERY_STARTUP_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_STARTUP_RECOVERY_LOCK_TTL_SECONDS or 120),
log_context="celery_startup_recovery",
)
return bool(token)
@worker_ready.connect
def on_worker_ready(sender=None, **kwargs):
"""Celery worker 启动时做一次容灾恢复。
@@ -68,13 +86,21 @@ def on_worker_ready(sender=None, **kwargs):
注意：
- 不启用 Celery beat。
- 不要求新增第四条启动命令。
- 只让 gen_result_download worker 投递恢复任务，避免三个 worker 同时重复扫描。
- 不再依赖 worker hostname 是否包含 gen_result_download。
- 所有 worker 都尝试抢 Redis 锁，只有抢到锁的 worker 投递恢复任务。
"""
if celery_app is None:
return
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
return
hostname = str(getattr(sender, "hostname", "") or "")
if "gen_result_download" not in hostname:
try:
if not run_async(_try_acquire_startup_recovery_lock()):
return
except Exception:
# Redis 不可用时不阻塞 worker 启动，避免影响稳定生成链路。
logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递")
return
try:
@@ -82,6 +108,7 @@ def on_worker_ready(sender=None, **kwargs):
recover_download_tasks_once,
recover_generation_tasks_once,
)
from app.tasks.shot_replicate_tasks import recover_split_tasks_once
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
@@ -95,6 +122,13 @@ def on_worker_ready(sender=None, **kwargs):
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
recover_split_tasks_once.apply_async(
countdown=countdown + 10,
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
logger.info("启动容灾恢复任务已投递。countdown=%s", countdown)
except Exception:
logger.exception("启动容灾恢复任务投递失败")
@@ -110,14 +144,14 @@ def on_worker_process_init(**kwargs):
@worker_process_shutdown.connect
def on_worker_process_shutdown(**kwargs):
"""子进程退出前关闭连接池和 event loop。"""
"""子进程退出前关闭连接池、Redis 注册表连接和 event loop。"""
try:
run_async(engine.dispose())
except Exception:
pass
try:
from app.services.celery_download_recovery_service import close_registry_redis
from app.services.redis_registry_service import close_registry_redis
run_async(close_registry_redis())
except Exception:
video-gen-api/app/tasks/generation_create_tasks.py
+10 -4
View File
@@ -11,9 +11,10 @@ from app.services.error_codes import extract_error_message
from app.services.generation_log_service import log_task_event
from app.services.generation_refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.generation_provider_service import create_provider_task
from app.services.redis_registry_service import ensure_aware_utc
from app.tasks.celery_app import celery_app
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate"}
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate", "shot_replicate"}
def _get_first_value(obj: Any, *field_names: str) -> Optional[Any]:
@@ -69,7 +70,7 @@ def _build_optimized_prompt_by_params(task: ChatGenerationTask) -> str:
# 爆款开头复刻第5步的视频生成，original_prompt 已经是视频提词 JSON schema。
# 不能再追加“时长/比例/分辨率”中文参数，否则会污染 schema。
if generation_mode == "hot_opening_replicate" and gen_type == "video":
if generation_mode in {"hot_opening_replicate", "shot_replicate"} and gen_type == "video":
stripped = base_prompt.strip()
if stripped.startswith("{") or stripped.startswith("["):
return base_prompt
@@ -129,7 +130,8 @@ async def _run(task_id: str):
if task.status != "generating":
return
if task.deadline_at and datetime.now(timezone.utc) > task.deadline_at:
deadline_at = ensure_aware_utc(task.deadline_at)
if deadline_at and datetime.now(timezone.utc) > deadline_at:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
@@ -242,7 +244,11 @@ async def _run(task_id: str):
else:
from app.tasks.generation_poll_tasks import poll_generation_task
poll_generation_task.delay(task.id)
poll_generation_task.apply_async(
args=[task.id],
queue="gen_provider_poll",
countdown=0,
)
except Exception as exc:
try:
video-gen-api/app/tasks/generation_download_tasks.py
+1 -1
View File
@@ -21,7 +21,7 @@ from app.services.generation_refund_service import mark_chat_generation_task_fai
from app.services.resource_accounting_service import record_chat_task_generated_resource
from app.tasks.celery_app import celery_app
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate"}
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate", "shot_replicate"}
DOWNLOAD_QUEUE = "gen_result_download"
DOWNLOAD_STAGE_QUEUED = "download_queued"
video-gen-api/app/tasks/generation_poll_tasks.py
+169 -56
View File
@@ -1,6 +1,7 @@
from app.tasks.async_runner import run_async
import json
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
@@ -11,9 +12,21 @@ from app.services.error_codes import extract_error_message
from app.services.generation_log_service import log_task_event, log_provider_call
from app.services.generation_refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.generation_provider_service import poll_provider_task
from app.services.redis_registry_service import (
datetime_to_epoch,
ensure_aware_utc,
redis_remove_registry_item,
redis_upsert_registry_item,
utc_now,
)
from app.tasks.celery_app import celery_app
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate"}
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate", "shot_replicate"}
POLL_QUEUE = "gen_provider_poll"
def _now() -> datetime:
return datetime.now(timezone.utc)
def _is_success(status: str) -> bool:
@@ -31,6 +44,86 @@ def _engine_snapshot(task: ChatGenerationTask) -> dict:
return {}
def _deadline_expired(task: ChatGenerationTask, now: datetime | None = None) -> bool:
deadline_at = ensure_aware_utc(task.deadline_at)
return bool(deadline_at and deadline_at <= (now or _now()))
def _poll_check_at(*, delay_seconds: int | float | None = None, now: datetime | None = None) -> datetime:
current_time = now or _now()
delay = int(delay_seconds or settings.CHATAPI_ASYNC_POLL_INTERVAL_SECONDS or 30)
grace = int(settings.POLL_TASK_QUEUE_TIMEOUT_SECONDS or 120)
return current_time + timedelta(seconds=max(1, delay) + max(0, grace))
def _poll_lease_until(now: datetime | None = None) -> datetime:
current_time = now or _now()
return current_time + timedelta(seconds=int(settings.POLL_TASK_LEASE_SECONDS or 300))
def _build_poll_active_payload(
task: ChatGenerationTask,
*,
stage: str,
reason: str,
next_poll_at: datetime | None = None,
check_at: datetime | None = None,
) -> dict[str, Any]:
current_time = utc_now()
checked_next_poll_at = ensure_aware_utc(next_poll_at)
checked_check_at = ensure_aware_utc(check_at)
return {
"task_id": task.id,
"provider_task_id": task.provider_task_id,
"seedance_task_id": task.seedance_task_id,
"generation_mode": task.generation_mode,
"gen_type": task.gen_type,
"stage": stage,
"queue": POLL_QUEUE,
"poll_count": int(task.poll_count or 0),
"retry_count": int(task.retry_count or 0),
"last_poll_at": datetime_to_epoch(task.last_poll_at) if task.last_poll_at else None,
"next_poll_at": datetime_to_epoch(checked_next_poll_at) if checked_next_poll_at else None,
"deadline_at": datetime_to_epoch(task.deadline_at) if task.deadline_at else None,
"check_at": datetime_to_epoch(checked_check_at) if checked_check_at else None,
"updated_at": datetime_to_epoch(current_time),
"reason": reason,
}
async def register_poll_active(
task: ChatGenerationTask,
*,
check_at: datetime,
reason: str,
next_poll_at: datetime | None = None,
) -> None:
payload = _build_poll_active_payload(
task,
stage=task.pipeline_stage or "",
reason=reason,
next_poll_at=next_poll_at,
check_at=check_at,
)
await redis_upsert_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=task.id,
payload=payload,
check_at=check_at,
log_context="poll_active",
)
async def remove_poll_active(task_id: str) -> None:
await redis_remove_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=task_id,
log_context="poll_active",
)
async def _notify_finished(db, task: ChatGenerationTask) -> None:
from app.services.generation_module_hook_service import notify_chat_generation_task_finished
@@ -55,6 +148,32 @@ async def _reload_task(db, task_id: str) -> ChatGenerationTask | None:
return result.scalar_one_or_none()
async def _mark_timeout(db, task: ChatGenerationTask, *, message: str = "任务轮询超时") -> None:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message=message,
pipeline_stage="timeout",
)
await _notify_finished(db, task)
await db.commit()
await remove_poll_active(task.id)
await log_task_event(task, event_type="TASK_TIMEOUT", to_status="failed", to_stage="timeout")
async def _mark_failed(db, task: ChatGenerationTask, *, message: str, detail: Any = None) -> None:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message=message,
pipeline_stage="failed",
)
await _notify_finished(db, task)
await db.commit()
await remove_poll_active(task.id)
await log_task_event(task, event_type="POLL_FAILED", message=task.error_message, detail=detail)
async def _run(task_id: str):
async with async_session() as db:
result = await db.execute(select(ChatGenerationTask).where(
@@ -62,43 +181,37 @@ async def _run(task_id: str):
ChatGenerationTask.deleted_at.is_(None),
).with_for_update().limit(1))
task = result.scalar_one_or_none()
if not task or task.generation_mode not in ALLOWED_GENERATION_MODES:
if not task:
await remove_poll_active(task_id)
return
if task.generation_mode not in ALLOWED_GENERATION_MODES:
await remove_poll_active(task.id)
return
# 只处理正在生成，且处于远程等待/轮询中的任务。
if task.status != "generating" or task.pipeline_stage not in ("waiting_remote", "polling"):
await remove_poll_active(task.id)
return
if task.deadline_at and datetime.now(timezone.utc) > task.deadline_at:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message="任务轮询超时",
pipeline_stage="timeout",
)
await _notify_finished(db, task)
await db.commit()
await log_task_event(task, event_type="TASK_TIMEOUT", to_status="failed", to_stage="timeout")
if _deadline_expired(task):
await _mark_timeout(db, task, message="任务轮询超时")
return
if not (task.seedance_task_id or task.provider_task_id):
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message="缺少外部任务ID",
pipeline_stage="failed",
)
await _notify_finished(db, task)
await db.commit()
await log_task_event(task, event_type="POLL_FAILED", message=task.error_message)
await _mark_failed(db, task, message="缺少外部任务ID")
return
# 标记本次正在轮询。
# 注意：pending 后会再改回 waiting_remote，避免任务长期卡在 polling。
# 标记本次正在轮询，并登记 poll lease。
# 如果 worker 在供应商接口调用过程中退出，启动恢复会在 lease 过期后重新投递。
task.pipeline_stage = "polling"
task.poll_count = (task.poll_count or 0) + 1
task.last_poll_at = datetime.now(timezone.utc)
task.last_poll_at = _now()
await db.commit()
await register_poll_active(
task,
check_at=_poll_lease_until(task.last_poll_at),
reason="polling_lease",
)
try:
poll_result = await poll_provider_task(db, task)
@@ -134,20 +247,13 @@ async def _run(task_id: str):
task.provider_response_json = response_data
if not task.remote_result_url:
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message="供应商任务成功但未返回结果URL",
pipeline_stage="failed",
)
await _notify_finished(db, task)
await db.commit()
await log_task_event(task, event_type="POLL_FAILED", message=task.error_message)
await _mark_failed(db, task, message="供应商任务成功但未返回结果URL", detail=poll_result)
return
task.pipeline_stage = "result_ready"
task.retry_count = 0
await db.commit()
await remove_poll_active(task.id)
await log_task_event(task, event_type="POLL_SUCCESS", to_stage="result_ready")
@@ -158,34 +264,38 @@ async def _run(task_id: str):
if _is_failed(status):
task.provider_response_json = response_data
await mark_chat_generation_task_failed_and_refund_once(
await _mark_failed(
db,
task=task,
error_message=poll_result.get("error") or f"供应商任务失败: {status}",
pipeline_stage="failed",
task,
message=poll_result.get("error") or f"供应商任务失败: {status}",
detail=poll_result,
)
await _notify_finished(db, task)
await db.commit()
await log_task_event(task, event_type="POLL_FAILED", message=task.error_message, detail=poll_result)
return
# 关键修改 1：
# 供应商仍在 pending / running 时，把阶段从 polling 改回 waiting_remote。
# 这样数据库状态表示“等待下一次轮询”，不会长期停在 polling。
# 同时可以降低重复 Celery 消息形成多条轮询链的概率。
# 同时登记下一次 poll active，Celery countdown 丢失时可由恢复任务拉起。
task.pipeline_stage = "waiting_remote"
task.retry_count = 0
await db.commit()
await log_task_event(task, event_type="POLL_PENDING", message=f"status={status}")
delay_seconds = int(settings.CHATAPI_ASYNC_POLL_INTERVAL_SECONDS or 30)
next_poll_at = _now() + timedelta(seconds=max(1, delay_seconds))
await register_poll_active(
task,
check_at=_poll_check_at(delay_seconds=delay_seconds),
next_poll_at=next_poll_at,
reason="poll_pending_next",
)
poll_generation_task.apply_async(
args=[task.id],
countdown=settings.CHATAPI_ASYNC_POLL_INTERVAL_SECONDS,
queue=POLL_QUEUE,
countdown=delay_seconds,
)
except Exception as exc:
# 关键修改 2：
# 异常后先 rollback，再重新查询 task，不继续使用 rollback 前的旧 ORM 对象。
try:
await db.rollback()
@@ -194,30 +304,33 @@ async def _run(task_id: str):
task = await _reload_task(db, task_id)
if not task:
await remove_poll_active(task_id)
return
task.retry_count = (task.retry_count or 0) + 1
if task.retry_count > settings.CHATAPI_ASYNC_MAX_RETRIES:
error_message = extract_error_message(exc, "轮询") if callable(extract_error_message) else str(exc)
await mark_chat_generation_task_failed_and_refund_once(
db,
task=task,
error_message=error_message,
pipeline_stage="failed",
)
await _notify_finished(db, task)
await db.commit()
await log_task_event(task, event_type="POLL_FAILED", message=task.error_message)
await _mark_failed(db, task, message=error_message)
else:
# 临时轮询异常时，不让任务停在 polling。
# 回到 waiting_remote，等待下一次重试轮询。
task.pipeline_stage = "waiting_remote"
await db.commit()
delay_seconds = int(settings.CHATAPI_ASYNC_RETRY_BACKOFF_SECONDS or 30) * int(task.retry_count or 1)
next_poll_at = _now() + timedelta(seconds=max(1, delay_seconds))
await register_poll_active(
task,
check_at=_poll_check_at(delay_seconds=delay_seconds),
next_poll_at=next_poll_at,
reason="poll_exception_retry",
)
poll_generation_task.apply_async(
args=[task.id],
countdown=settings.CHATAPI_ASYNC_RETRY_BACKOFF_SECONDS * task.retry_count,
queue=POLL_QUEUE,
countdown=delay_seconds,
)
video-gen-api/app/tasks/shot_replicate_flow_tasks.py
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
from app.models.base import async_session
from app.services.shot_replicate_flow_service import run_image_prompt_optimize, run_video_prompt_optimize
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
async def _run_image_prompt(project_id: str, step_id: str | None = None):
async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
await db.commit()
async def _run_video_prompt(project_id: str, step_id: str | None = None):
async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
await db.commit()
if celery_app:
@celery_app.task(name="shot_replicate.start_image_prompt_optimize", bind=True, max_retries=3, default_retry_delay=30)
def start_image_prompt_optimize(self, project_id: str, step_id: str | None = None):
return run_async(_run_image_prompt(project_id, step_id))
@celery_app.task(name="shot_replicate.start_video_prompt_optimize", bind=True, max_retries=3, default_retry_delay=30)
def start_video_prompt_optimize(self, project_id: str, step_id: str | None = None):
return run_async(_run_video_prompt(project_id, step_id))
else:
class _DisabledTask:
def delay(self, *args, **kwargs):
raise RuntimeError("Celery is disabled")
def apply_async(self, *args, **kwargs):
raise RuntimeError("Celery is disabled")
start_image_prompt_optimize = _DisabledTask()
start_video_prompt_optimize = _DisabledTask()
video-gen-api/app/tasks/shot_replicate_tasks.py
+505
View File
@@ -0,0 +1,505 @@
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from app.config import settings
from app.enums.shot_replicate import (
ModuleCodeEnum,
ShotAnalysisStatusEnum,
ShotSegmentAnalysisStatusEnum,
ShotSegmentSourceModeEnum,
ShotSplitStatusEnum,
ShotTaskSetStatusEnum,
)
from app.models.base import async_session
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
from app.services.redis_registry_service import redis_acquire_lock, redis_release_lock
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
from app.services.shot_video_split_service import split_video_segment_async
from app.services.upload_video_asset_service import validate_split_range
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
SPLIT_QUEUE = "gen_result_download"
ANALYSIS_QUEUE = "gen_chatapi_create"
def _now() -> datetime:
return datetime.now(timezone.utc)
def _lease_until(now: datetime | None = None) -> datetime:
return (now or _now()) + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))
def _retry_at(attempt: int, now: datetime | None = None) -> datetime:
base = int(settings.SHOT_SPLIT_RETRY_BACKOFF_SECONDS or settings.DOWNLOAD_TASK_RETRY_BACKOFF_SECONDS or 30)
return (now or _now()) + timedelta(seconds=max(1, base * max(1, attempt)))
async def _acquire_split_semaphore(segment_id: str) -> str | None:
"""简单 Redis 并发闸门：用固定槽位锁限制 ffmpeg 同时运行数量。"""
max_concurrent = max(1, int(settings.SHOT_SPLIT_MAX_CONCURRENT or 1))
ttl = int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)
for slot in range(max_concurrent):
key = f"{settings.SHOT_SPLIT_SEMAPHORE_KEY_PREFIX}:{slot}"
token = await redis_acquire_lock(lock_key=key, ttl_seconds=ttl, token=segment_id, log_context="shot_split_semaphore")
if token:
return key
return None
async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> None:
if lock_key:
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
async def _run_analyze_original_video(task_set_id: str) -> None:
task_set_user_id: str | None = None
video_url: str | None = None
try:
async with async_session() as db:
result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
return
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
return
task_set_user_id = task_set.user_id
video_url = task_set.video_url
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
task_set.analysis_error_message = None
await db.commit()
log_module_event_file(
module=MODULE,
event_type="SHOT_ANALYSIS_STARTED",
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析开始",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=task_set_user_id, mode="full_breakdown")
result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await db.rollback()
return
result_json = analyzed.result
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
task_set.original_video_category = str(result_json.get("原视频分类") or "无")
task_set.original_video_audience = str(result_json.get("原视频受众人群") or "无")
task_set.ai_suggestion_json = result_json.get("拆镜内容剖析") or []
task_set.analysis_raw_json = analyzed.raw_response
task_set.analysis_result_json = result_json
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
task_set.analysis_error_message = None
await db.commit()
log_module_prompt_event(
event_type="SHOT_ANALYSIS_SUCCESS",
project_id=task_set_id,
step_id=task_set_id,
user_id=task_set_user_id or "",
module=MODULE,
prompt_type="shot_video_analysis",
request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {},
response=analyzed.result,
token_usage=analyzed.usage,
)
log_module_event_file(
module=MODULE,
event_type="SHOT_ANALYSIS_SUCCESS",
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析成功",
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "token_usage": analyzed.usage},
)
except Exception as exc:
async with async_session() as db:
result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = result.scalar_one_or_none()
if task_set:
task_set_user_id = task_set_user_id or task_set.user_id
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
task_set.analysis_error_message = str(exc)
await db.commit()
log_module_error(
module=MODULE,
event_type="SHOT_ANALYSIS_FAILED",
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析失败",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
exc=exc,
)
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
user_id: str | None = None
task_set_id: str | None = None
video_url: str | None = None
try:
async with async_session() as db:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or not segment.segment_video_url:
return
if segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
return
user_id = segment.user_id
task_set_id = segment.task_set_id
video_url = segment.segment_video_url
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
segment.analysis_error_message = None
await db.commit()
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_ANALYSIS_STARTED",
project_id=task_set_id,
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析开始",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=user_id, mode="summary_only")
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await db.rollback()
return
result_json = analyzed.result
segment.original_video_content = str(result_json.get("原视频内容") or "无")
segment.original_video_category = str(result_json.get("原视频分类") or "无")
segment.original_video_audience = str(result_json.get("原视频受众人群") or "无")
segment.segment_content = segment.original_video_content
segment.segment_category = segment.original_video_category
segment.segment_audience = segment.original_video_audience
segment.analysis_json = result_json
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
segment.analysis_error_message = None
await db.commit()
log_module_prompt_event(
event_type="SHOT_SEGMENT_ANALYSIS_SUCCESS",
project_id=task_set_id or segment_id,
step_id=segment_id,
user_id=user_id or "",
module=MODULE,
prompt_type="shot_segment_analysis",
request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {},
response=analyzed.result,
token_usage=analyzed.usage,
)
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_ANALYSIS_SUCCESS",
project_id=task_set_id,
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析成功",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "token_usage": analyzed.usage},
)
except Exception as exc:
async with async_session() as db:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if segment:
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
segment.analysis_error_message = str(exc)
await db.commit()
log_module_error(
module=MODULE,
event_type="SHOT_SEGMENT_ANALYSIS_FAILED",
project_id=task_set_id,
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析失败",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
exc=exc,
)
async def _run_split_one_segment(segment_id: str) -> None:
segment_lock_key = f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}"
segment_lock_token = await redis_acquire_lock(
lock_key=segment_lock_key,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
log_context="shot_split_segment_lock",
)
if not segment_lock_token:
return
semaphore_key: str | None = None
user_id: str | None = None
task_set_id: str | None = None
source_path: str | None = None
try:
semaphore_key = await _acquire_split_semaphore(segment_id)
if not semaphore_key:
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
step_id=segment_id,
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
detail={"segment_id": segment_id, "reason": "semaphore_full"},
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=10, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL)
return
async with async_session() as db:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
return
user_id = segment.user_id
task_set_id = segment.task_set_id
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = task_set_result.scalar_one_or_none()
if not task_set:
return
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
return
validate_split_range(
start_second=segment.start_second,
end_second=segment.end_second,
video_duration_seconds=task_set.video_duration_seconds,
)
now = _now()
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
segment.split_started_at = now
segment.split_lease_until = _lease_until(now)
segment.split_retry_count = int(segment.split_retry_count or 0) + 1
segment.split_next_retry_at = None
segment.split_last_error = None
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.commit()
source_path = task_set.video_path
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
start_second = segment.start_second
end_second = segment.end_second
attempt = segment.split_retry_count
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_STARTED",
project_id=task_set_id,
step_id=segment_id,
user_id=user_id,
message="拆镜片段 ffmpeg 切割开始",
detail={
"segment_id": segment_id,
"task_set_id": task_set_id,
"source_path": source_path,
"start_second": start_second,
"end_second": end_second,
"attempt": attempt,
},
)
split_result = await split_video_segment_async(
source_path=source_path,
segment_id=segment_id,
start_second=start_second,
end_second=end_second,
date_dir=date_dir,
)
async with async_session() as db:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
return
segment.segment_video_url = split_result.url
segment.segment_video_path = split_result.path
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
segment.split_completed_at = _now()
segment.split_lease_until = None
segment.split_next_retry_at = None
segment.split_last_error = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=segment.task_set_id,
step_id=segment.id,
user_id=segment.user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment.id,
"task_set_id": segment.task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": segment.source_mode,
},
)
if segment.source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
analyze_custom_segment_video.apply_async(args=[segment.id], queue=ANALYSIS_QUEUE, countdown=0)
except Exception as exc:
next_retry_delay: int | None = None
final_failed = False
async with async_session() as db:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
return
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
attempt = int(segment.split_retry_count or 0)
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
if segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value and celery_app:
next_retry_delay = max(1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()))
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
log_module_error(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
project_id=task_set_id,
step_id=segment_id,
user_id=user_id,
message="拆镜片段 ffmpeg 切割失败" if final_failed else "拆镜片段 ffmpeg 切割失败，等待重试",
detail={
"segment_id": segment_id,
"task_set_id": task_set_id,
"source_path": source_path,
"next_retry_delay_seconds": next_retry_delay,
"final_failed": final_failed,
},
exc=exc,
)
finally:
await _release_split_semaphore(semaphore_key, segment_id)
await redis_release_lock(lock_key=segment_lock_key, token=segment_lock_token, log_context="shot_split_segment_lock")
async def _run_recover_split_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
async with async_session() as db:
return await recover_shot_split_tasks_once(db)
if celery_app:
@celery_app.task(name="shot_replicate.analyze_original_video")
def analyze_original_video(task_set_id: str) -> None:
return run_async(_run_analyze_original_video(task_set_id))
@celery_app.task(name="shot_replicate.split_one_segment", bind=True, max_retries=0)
def split_one_segment(self, segment_id: str) -> None:
return run_async(_run_split_one_segment(segment_id))
@celery_app.task(name="shot_replicate.analyze_custom_segment_video")
def analyze_custom_segment_video(segment_id: str) -> None:
return run_async(_run_analyze_custom_segment_video(segment_id))
@celery_app.task(name="shot_replicate.recover_split_tasks_once")
def recover_split_tasks_once() -> dict[str, Any]:
return run_async(_run_recover_split_tasks_once())
else:
class _DisabledTask:
def delay(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
def apply_async(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
analyze_original_video = _DisabledTask()
split_one_segment = _DisabledTask()
analyze_custom_segment_video = _DisabledTask()
recover_split_tasks_once = _DisabledTask()