from __future__ import annotations import logging import uuid from datetime import datetime, timedelta, timezone from typing import Any from sqlalchemy import select, update from app.config import settings from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType from app.enums.celery_queue import CeleryQueue, CeleryTaskName from app.enums.celery_runtime import CeleryRuntimeDomain from app.enums.shot_replicate import ( ModuleCodeEnum, ShotAnalysisStatusEnum, ShotReplicateLogEventEnum, ShotSegmentAnalysisStatusEnum, ShotSegmentSourceModeEnum, ShotSplitStatusEnum, ShotTaskSetStatusEnum, ) from app.models.base import async_session from app.models.shot_replicate_segment import ShotReplicateSegment from app.models.shot_replicate_task_set import ShotReplicateTaskSet from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event from app.services.redis_registry_service import ( RedisExecutionLockError, RedisExecutionLockLease, redis_acquire_lock, redis_release_lock, ) from app.services.celery_runtime.recovery_service import guard_periodic_recovery from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary from app.services.shot_video_analysis_service import analyze_video_for_shot_split from app.services.generation.billing_service import charge_shot_video_analysis_usage from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async from app.services.upload_video_asset_service import validate_split_range from app.services.upload_resource import record_shot_segment_upload_resource from app.tasks.async_runner import run_async from app.tasks.celery_app import celery_app logger = logging.getLogger("video_gen") MODULE = ModuleCodeEnum.SHOT_REPLICATE.value SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value def _now() -> datetime: return datetime.now(timezone.utc) def _lease_until(now: datetime | None = None) -> datetime: return (now or _now()) + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)) def _retry_at(attempt: int, now: datetime | None = None) -> datetime: base = int(settings.SHOT_SPLIT_RETRY_BACKOFF_SECONDS or settings.DOWNLOAD_TASK_RETRY_BACKOFF_SECONDS or 30) return (now or _now()) + timedelta(seconds=max(1, base * max(1, attempt))) async def _acquire_split_semaphore(segment_id: str) -> str | None: """简单 Redis 并发闸门：用固定槽位锁限制 ffmpeg 同时运行数量。""" max_concurrent = max(1, int(settings.SHOT_SPLIT_MAX_CONCURRENT or 1)) ttl = int(settings.SHOT_SPLIT_LEASE_SECONDS or 600) for slot in range(max_concurrent): key = f"{settings.SHOT_SPLIT_SEMAPHORE_KEY_PREFIX}:{slot}" token = await redis_acquire_lock(lock_key=key, ttl_seconds=ttl, token=segment_id, log_context="shot_split_semaphore") if token: return key return None async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> None: if lock_key: await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore") async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool: async with async_session() as db: result = await db.execute( update(ShotReplicateTaskSet) .where( ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None), ShotReplicateTaskSet.analysis_attempt_no == attempt_no, ShotReplicateTaskSet.analysis_claim_token == token, ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value, ) .values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))) ) await db.commit() return bool(result.rowcount == 1) async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool: async with async_session() as db: result = await db.execute( update(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ShotReplicateSegment.analysis_attempt_no == attempt_no, ShotReplicateSegment.analysis_claim_token == token, ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value, ) .values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))) ) await db.commit() return bool(result.rowcount == 1) def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str: return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}" async def _run_analyze_original_video(task_set_id: str) -> None: token = uuid.uuid4().hex attempt_no = 1 async with async_session() as db: row = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .limit(1) ) initial = row.scalar_one_or_none() if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value: return attempt_no = int(initial.analysis_attempt_no or 1) await db.rollback() lease = await CeleryRuntimeLease.acquire( identity=RuntimeIdentity( domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value, owner_type="shot_task_set", owner_id=task_set_id, attempt_no=attempt_no, task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value, queue=ANALYSIS_QUEUE, ), lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no), hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY, zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY, token=token, ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180), heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30), pipeline_stage="analysis_processing", db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token), ) if lease is None: return task_set_user_id: str | None = None video_url: str | None = None try: async with async_session() as db: result = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value: await db.rollback() return current_lease = task_set.analysis_lease_until if ( task_set.analysis_claim_token and task_set.analysis_claim_token != token and current_lease and current_lease > _now() ): await db.rollback() return if int(task_set.analysis_attempt_no or 1) != attempt_no: await db.rollback() return task_set_user_id = str(task_set.user_id) video_url = str(task_set.video_url) task_set.status = ShotTaskSetStatusEnum.ANALYZING.value task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value task_set.analysis_claim_token = token task_set.analysis_started_at = _now() task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)) task_set.analysis_error_message = None await db.commit() log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_STARTED.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频拆镜分析开始", detail={ "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown", "analysis_attempt_no": attempt_no, "queue": ANALYSIS_QUEUE, }, ) async with async_session() as call_db: analyzed = await analyze_video_for_shot_split( call_db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}", ) await lease.ensure_owned() result = await call_db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() if ( not task_set or int(task_set.analysis_attempt_no or 1) != attempt_no or task_set.analysis_claim_token != token or str(task_set.video_url) != str(video_url) or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value ): await call_db.rollback() return result_json = analyzed.result task_set.original_video_content = str(result_json.get("原视频内容") or "无") task_set.original_video_category = str(result_json.get("原视频分类") or "无") task_set.original_video_audience = str(result_json.get("原视频受众人群") or "无") task_set.ai_suggestion_json = result_json.get("拆镜内容剖析") or [] task_set.analysis_raw_json = analyzed.raw_response task_set.analysis_result_json = result_json task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value task_set.analysis_claim_token = None task_set.analysis_lease_until = None task_set.analysis_error_message = None await charge_shot_video_analysis_usage( call_db, user_id=task_set.user_id, owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value, owner_id=task_set.id, usage=analyzed.usage, description="拆镜复刻-原视频分析", billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value, source_project_id=task_set.id, attempt_no=attempt_no, ) await call_db.commit() log_module_prompt_event( event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=task_set_id, user_id=task_set_user_id or "", module=MODULE, prompt_type="shot_video_analysis", request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {}, response=analyzed.result, token_usage=analyzed.usage, ) log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频拆镜分析成功", detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no}, ) except RedisExecutionLockError: raise except Exception as exc: async with async_session() as db: result = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() if ( task_set and int(task_set.analysis_attempt_no or 1) == attempt_no and task_set.analysis_claim_token == token and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value ): task_set_user_id = task_set_user_id or str(task_set.user_id) task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value task_set.analysis_claim_token = None task_set.analysis_lease_until = None task_set.analysis_error_message = str(exc) await db.commit() else: await db.rollback() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频拆镜分析失败", detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no}, exc=exc, ) finally: await lease.close() async def _run_analyze_custom_segment_video(segment_id: str) -> None: token = uuid.uuid4().hex attempt_no = 1 async with async_session() as db: row = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .limit(1) ) initial = row.scalar_one_or_none() if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value: return attempt_no = int(initial.analysis_attempt_no or 1) await db.rollback() lease = await CeleryRuntimeLease.acquire( identity=RuntimeIdentity( domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value, owner_type="shot_segment", owner_id=segment_id, attempt_no=attempt_no, task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value, queue=ANALYSIS_QUEUE, ), lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no), hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY, zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY, token=token, ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180), heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30), pipeline_stage="analysis_processing", db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token), ) if lease is None: return user_id: str | None = None task_set_id: str | None = None video_url: str | None = None try: async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value: await db.rollback() return current_lease = segment.analysis_lease_until if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now(): await db.rollback() return if int(segment.analysis_attempt_no or 1) != attempt_no: await db.rollback() return user_id = str(segment.user_id) task_set_id = str(segment.task_set_id) video_url = str(segment.segment_video_url) segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value segment.analysis_claim_token = token segment.analysis_started_at = _now() segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)) segment.analysis_error_message = None await db.commit() log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_STARTED.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义拆镜片段分析开始", detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no}, ) async with async_session() as call_db: analyzed = await analyze_video_for_shot_split( call_db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}", ) await lease.ensure_owned() result = await call_db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( not segment or int(segment.analysis_attempt_no or 1) != attempt_no or segment.analysis_claim_token != token or str(segment.segment_video_url) != str(video_url) or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value ): await call_db.rollback() return result_json = analyzed.result segment.original_video_content = str(result_json.get("原视频内容") or "无") segment.original_video_category = str(result_json.get("原视频分类") or "无") segment.original_video_audience = str(result_json.get("原视频受众人群") or "无") segment.segment_content = segment.original_video_content segment.segment_category = segment.original_video_category segment.segment_audience = segment.original_video_audience segment.analysis_json = result_json segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value segment.analysis_claim_token = None segment.analysis_lease_until = None segment.analysis_error_message = None await charge_shot_video_analysis_usage( call_db, user_id=segment.user_id, owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value, owner_id=segment.id, usage=analyzed.usage, description="拆镜复刻-片段视频分析", billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value, source_project_id=segment.task_set_id, source_step_id=segment.id, attempt_no=attempt_no, ) await call_db.commit() log_module_prompt_event( event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=segment_id, user_id=user_id or "", module=MODULE, prompt_type="shot_segment_analysis", request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {}, response=analyzed.result, token_usage=analyzed.usage, ) log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义拆镜片段分析成功", detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no}, ) except RedisExecutionLockError: raise except Exception as exc: async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( segment and int(segment.analysis_attempt_no or 1) == attempt_no and segment.analysis_claim_token == token and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value ): user_id = user_id or str(segment.user_id) task_set_id = task_set_id or str(segment.task_set_id) segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value segment.analysis_claim_token = None segment.analysis_lease_until = None segment.analysis_error_message = str(exc) await db.commit() else: await db.rollback() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义拆镜片段分析失败", detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no}, exc=exc, ) finally: await lease.close() async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool: async with async_session() as db: result = await db.execute( update(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ShotReplicateSegment.split_retry_count == attempt_no, ShotReplicateSegment.split_claim_token == token, ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value, ) .values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))) ) await db.commit() return bool(result.rowcount == 1) async def _run_split_one_segment(segment_id: str) -> None: async with async_session() as db: row = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .limit(1) ) initial = row.scalar_one_or_none() if not initial: return if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url: return attempt = int(initial.split_retry_count or 0) + 1 await db.rollback() token = uuid.uuid4().hex lease = await CeleryRuntimeLease.acquire( identity=RuntimeIdentity( domain=CeleryRuntimeDomain.SHOT_SPLIT.value, owner_type="shot_segment", owner_id=segment_id, attempt_no=attempt, task_name=CeleryTaskName.SHOT_SPLIT_ONE.value, queue=SPLIT_QUEUE, ), lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}", hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY, zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY, token=token, ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600), heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30), pipeline_stage=ShotSplitStatusEnum.PROCESSING.value, db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token), ) if lease is None: return semaphore_key: str | None = None user_id: str | None = None task_set_id: str | None = None source_path: str | None = None split_result = None try: semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}") if not semaphore_key: delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10)) log_module_event_file( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING", step_id=segment_id, message="拆镜 ffmpeg 并发闸门已满，稍后重试", detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt}, ) if celery_app: split_one_segment.apply_async( args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL, ) return async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if not segment: await db.rollback() return user_id = str(segment.user_id) task_set_id = str(segment.task_set_id) task_set_result = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = task_set_result.scalar_one_or_none() if not task_set: await db.rollback() return if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url: await db.rollback() return if ( segment.split_claim_token and segment.split_claim_token != token and segment.split_lease_until and segment.split_lease_until > _now() ): await db.rollback() return validate_split_range( start_second=segment.start_second, end_second=segment.end_second, video_duration_seconds=task_set.video_duration_seconds, ) now = _now() segment.split_status = ShotSplitStatusEnum.PROCESSING.value segment.split_claim_token = token segment.split_started_at = now segment.split_lease_until = _lease_until(now) segment.split_retry_count = attempt segment.split_next_retry_at = None segment.split_last_error = None task_set.status = ShotTaskSetStatusEnum.SPLITTING.value task_set.split_status = ShotSplitStatusEnum.PROCESSING.value source_path = str(task_set.video_path) date_dir = (segment.created_at or now).strftime("%Y/%m/%d") start_second = float(segment.start_second) end_second = float(segment.end_second) await db.commit() log_module_event_file( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_STARTED", project_id=task_set_id, step_id=segment_id, user_id=user_id, message="拆镜片段 ffmpeg 切割开始", detail={ "segment_id": segment_id, "task_set_id": task_set_id, "source_path": source_path, "start_second": start_second, "end_second": end_second, "attempt": attempt, "queue": SPLIT_QUEUE, }, ) split_result = await split_video_segment_async( source_path=source_path or "", segment_id=segment_id, start_second=start_second, end_second=end_second, date_dir=date_dir, attempt_key=f"attempt-{attempt}-{token[-8:]}", finalize=False, ) await lease.ensure_owned() async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( not segment or int(segment.split_retry_count or 0) != attempt or segment.split_claim_token != token or segment.split_status != ShotSplitStatusEnum.PROCESSING.value ): await db.rollback() cleanup_split_result(split_result) return split_result = finalize_split_result(split_result) segment.segment_video_url = split_result.url segment.segment_video_path = split_result.path await record_shot_segment_upload_resource( db, segment=segment, storage_path=split_result.path, resource_url=split_result.url, file_size_bytes=split_result.file_size_bytes, ) segment.split_status = ShotSplitStatusEnum.COMPLETED.value segment.split_claim_token = None segment.split_completed_at = _now() segment.split_lease_until = None segment.split_next_retry_at = None segment.split_last_error = None source_mode = str(segment.source_mode) final_task_set_id = str(segment.task_set_id) final_user_id = str(segment.user_id) await refresh_task_set_split_summary(db, segment.task_set_id) await db.commit() log_module_event_file( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_SUCCESS", project_id=final_task_set_id, step_id=segment_id, user_id=final_user_id, message="拆镜片段 ffmpeg 切割成功", detail={ "segment_id": segment_id, "task_set_id": final_task_set_id, "segment_video_url": split_result.url, "segment_video_path": split_result.path, "source_mode": source_mode, "attempt": attempt, }, ) if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app: analyze_custom_segment_video.apply_async( args=[segment_id], queue=ANALYSIS_QUEUE, countdown=0, task_id=f"shot-analysis:segment:{segment_id}:attempt:1", ) except RedisExecutionLockError: cleanup_split_result(split_result) raise except Exception as exc: cleanup_split_result(split_result) next_retry_delay: int | None = None final_failed = False async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( segment and int(segment.split_retry_count or 0) == attempt and segment.split_claim_token == token and segment.split_status == ShotSplitStatusEnum.PROCESSING.value ): user_id = user_id or str(segment.user_id) task_set_id = task_set_id or str(segment.task_set_id) segment.split_claim_token = None segment.split_last_error = str(exc) segment.split_lease_until = None if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3): segment.split_status = ShotSplitStatusEnum.FAILED.value segment.split_next_retry_at = None final_failed = True else: segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value segment.split_next_retry_at = _retry_at(attempt) next_retry_delay = max( 1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()), ) await refresh_task_set_split_summary(db, segment.task_set_id) await db.commit() else: await db.rollback() return if next_retry_delay and celery_app: split_one_segment.apply_async( args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER, ) log_module_error( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING", project_id=task_set_id, step_id=segment_id, user_id=user_id, message="拆镜片段 ffmpeg 切割失败" if final_failed else "拆镜片段 ffmpeg 切割失败，等待重试", detail={ "segment_id": segment_id, "task_set_id": task_set_id, "source_path": source_path, "next_retry_delay_seconds": next_retry_delay, "final_failed": final_failed, "attempt": attempt, }, exc=exc, ) finally: await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}") await lease.close() async def _run_recover_split_tasks_once() -> dict[str, Any]: from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once barrier = await guard_periodic_recovery() if barrier is not None: return barrier lease = await RedisExecutionLockLease.acquire( lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY, ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600), renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)), log_context="shot_split_recovery", ) if lease is None: return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY} async with lease: async with async_session() as db: result = await recover_shot_split_tasks_once(db) result["execution_lock"] = "lock_acquired" return result async def _run_recover_analysis_tasks_once() -> dict[str, Any]: from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once barrier = await guard_periodic_recovery() if barrier is not None: return barrier lease = await RedisExecutionLockLease.acquire( lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY, ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600), renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)), log_context="shot_analysis_recovery", ) if lease is None: return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY} async with lease: async with async_session() as db: result = await recover_shot_analysis_tasks_once(db) result["execution_lock"] = "lock_acquired" return result if celery_app: @celery_app.task( name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value, bind=True, max_retries=3, default_retry_delay=60, soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS, ignore_result=True, ) def analyze_original_video(self, task_set_id: str) -> None: try: return run_async(_run_analyze_original_video(task_set_id)) except RedisExecutionLockError as exc: raise self.retry(exc=exc, countdown=60) @celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True) def split_one_segment(self, segment_id: str) -> None: try: return run_async(_run_split_one_segment(segment_id)) except RedisExecutionLockError as exc: raise self.retry(exc=exc, countdown=30) @celery_app.task( name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value, bind=True, max_retries=3, default_retry_delay=60, soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS, ignore_result=True, ) def analyze_custom_segment_video(self, segment_id: str) -> None: try: return run_async(_run_analyze_custom_segment_video(segment_id)) except RedisExecutionLockError as exc: raise self.retry(exc=exc, countdown=60) @celery_app.task( name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value, bind=True, soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS, ) def recover_analysis_tasks_once(self) -> dict[str, Any]: return run_async(_run_recover_analysis_tasks_once()) @celery_app.task( name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value, bind=True, soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS, ) def recover_split_tasks_once(self) -> dict[str, Any]: return run_async(_run_recover_split_tasks_once()) else: class _DisabledTask: def delay(self, *args: Any, **kwargs: Any) -> None: raise RuntimeError("Celery is disabled") def apply_async(self, *args: Any, **kwargs: Any) -> None: raise RuntimeError("Celery is disabled") analyze_original_video = _DisabledTask() split_one_segment = _DisabledTask() analyze_custom_segment_video = _DisabledTask() recover_split_tasks_once = _DisabledTask() recover_analysis_tasks_once = _DisabledTask()