from __future__ import annotations import logging import uuid from datetime import datetime, timedelta, timezone from typing import Any from sqlalchemy import select, update from app.config import settings from app.enums.credit_record import CreditRecordBillingScene, CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule, CreditRecordSourceStepCode from app.enums.llm_billing import LlmBillingConfigKey from app.enums.celery_queue import CeleryQueue, CeleryTaskName from app.enums.celery_runtime import CeleryRuntimeDomain from app.enums.shot_replicate import ( ModuleCodeEnum, ShotAnalysisStatusEnum, ShotReplicateLogEventEnum, ShotSegmentAnalysisStatusEnum, ShotSegmentSourceModeEnum, ShotSplitStatusEnum, ShotTaskSetStatusEnum, ) from app.models.base import async_session from app.models.shot_replicate_segment import ShotReplicateSegment from app.models.shot_replicate_task_set import ShotReplicateTaskSet from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event from app.services.redis_registry_service import ( RedisExecutionLockError, RedisExecutionLockLease, redis_acquire_lock, redis_release_lock, ) from app.services.celery_runtime.recovery_service import guard_periodic_recovery from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity from app.services.shot_replicate_taskset_service import ( build_segment_analysis_billing_context, refresh_task_set_split_summary, ) from app.services.shot_video_analysis_service import analyze_video_for_shot_split from app.services.llm_billing import ( LlmBillingContext, ensure_hold_exists, log_provider_failure, log_provider_start, log_provider_success, release_on_failure, settle_success, ) from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async from app.services.upload_video_asset_service import validate_split_range from app.services.upload_resource import record_shot_segment_upload_resource from app.tasks.async_runner import run_async from app.tasks.celery_app import celery_app from app.utils.exceptions import InsufficientCreditsError logger = logging.getLogger("video_gen") MODULE = ModuleCodeEnum.SHOT_REPLICATE.value SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value def _now() -> datetime: return datetime.now(timezone.utc) def _lease_until(now: datetime | None = None) -> datetime: return (now or _now()) + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)) def _retry_at(attempt: int, now: datetime | None = None) -> datetime: base = int(settings.SHOT_SPLIT_RETRY_BACKOFF_SECONDS or settings.DOWNLOAD_TASK_RETRY_BACKOFF_SECONDS or 30) return (now or _now()) + timedelta(seconds=max(1, base * max(1, attempt))) async def _acquire_split_semaphore(segment_id: str) -> str | None: """简单 Redis 并发闸门：用固定槽位锁限制 ffmpeg 同时运行数量。""" max_concurrent = max(1, int(settings.SHOT_SPLIT_MAX_CONCURRENT or 1)) ttl = int(settings.SHOT_SPLIT_LEASE_SECONDS or 600) for slot in range(max_concurrent): key = f"{settings.SHOT_SPLIT_SEMAPHORE_KEY_PREFIX}:{slot}" token = await redis_acquire_lock(lock_key=key, ttl_seconds=ttl, token=segment_id, log_context="shot_split_semaphore") if token: return key return None async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> None: if lock_key: await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore") async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool: async with async_session() as db: result = await db.execute( update(ShotReplicateTaskSet) .where( ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None), ShotReplicateTaskSet.analysis_attempt_no == attempt_no, ShotReplicateTaskSet.analysis_claim_token == token, ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value, ) .values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))) ) await db.commit() return bool(result.rowcount == 1) async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool: async with async_session() as db: result = await db.execute( update(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ShotReplicateSegment.analysis_attempt_no == attempt_no, ShotReplicateSegment.analysis_claim_token == token, ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value, ) .values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))) ) await db.commit() return bool(result.rowcount == 1) def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str: return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}" def _log_stale_analysis_attempt( *, owner_type: str, owner_id: str, expected_attempt_no: int | None, actual_attempt_no: int | None, analysis_status: str | None, task_set_id: str | None = None, user_id: str | None = None, reason: str, ) -> None: project_id = task_set_id or (owner_id if owner_type == "shot_task_set" else None) log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_STALE_ATTEMPT_SKIPPED.value, project_id=project_id, step_id=owner_id if owner_type == "shot_segment" else None, user_id=user_id, message="拆镜分析任务 attempt 已失效，跳过执行", detail={ "owner_type": owner_type, "owner_id": owner_id, "expected_attempt_no": expected_attempt_no, "actual_attempt_no": actual_attempt_no, "analysis_status": analysis_status, "reason": reason, }, event_status="skipped", ) async def _persist_original_analysis_result( *, task_set_id: str, attempt_no: int, token: str, video_url: str | None, analyzed: Any, billing_context: LlmBillingContext, allow_business_write: bool, ) -> str: """持久化原视频分析结果并完成账务；供应商成功后禁止再次调用模型。""" async with async_session() as db: result = await db.execute( select(ShotReplicateTaskSet) .where( ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None), ) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() is_current = bool( allow_business_write and task_set and int(task_set.analysis_attempt_no or 1) == attempt_no and task_set.analysis_claim_token == token and str(task_set.video_url) == str(video_url) and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value ) if is_current and task_set is not None: result_json = analyzed.result task_set.original_video_content = str(result_json.get("原视频内容") or "无") task_set.original_video_category = str(result_json.get("原视频分类") or "无") task_set.original_video_audience = str(result_json.get("原视频受众人群") or "无") task_set.ai_suggestion_json = result_json.get("拆镜内容剖析") or [] task_set.analysis_raw_json = analyzed.raw_response task_set.analysis_result_json = result_json task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value task_set.analysis_claim_token = None task_set.analysis_lease_until = None task_set.analysis_error_message = None description = "拆镜复刻-原视频分析" outcome = "completed" else: description = "拆镜复刻-原视频分析（失效结果结算）" outcome = "stale_settled" await settle_success( db, billing_context, usage=analyzed.usage, description=description, ) await db.commit() return outcome async def _persist_segment_analysis_result( *, segment_id: str, attempt_no: int, token: str, video_url: str | None, analyzed: Any, billing_context: LlmBillingContext, allow_business_write: bool, ) -> str: """持久化自定义切片分析结果并完成账务；供应商成功后禁止再次调用模型。""" async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() is_current = bool( allow_business_write and segment and int(segment.analysis_attempt_no or 1) == attempt_no and segment.analysis_claim_token == token and str(segment.segment_video_url) == str(video_url) and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value ) if is_current and segment is not None: result_json = analyzed.result segment.original_video_content = str(result_json.get("原视频内容") or "无") segment.original_video_category = str(result_json.get("原视频分类") or "无") segment.original_video_audience = str(result_json.get("原视频受众人群") or "无") segment.segment_content = segment.original_video_content segment.segment_category = segment.original_video_category segment.segment_audience = segment.original_video_audience segment.analysis_json = result_json segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value segment.analysis_claim_token = None segment.analysis_lease_until = None segment.analysis_error_message = None description = "拆镜复刻-片段视频分析" outcome = "completed" else: description = "拆镜复刻-片段视频分析（失效结果结算）" outcome = "stale_settled" await settle_success( db, billing_context, usage=analyzed.usage, description=description, ) await db.commit() return outcome async def _mark_original_provider_success_pending_manual( *, task_set_id: str, attempt_no: int, token: str, error_message: str, ) -> bool: """供应商已成功但本地结算失败：终止自动恢复，保留 HOLD 等待人工对账。""" async with async_session() as db: result = await db.execute( select(ShotReplicateTaskSet) .where( ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None), ) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() if not ( task_set and int(task_set.analysis_attempt_no or 1) == attempt_no and task_set.analysis_claim_token == token and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value ): await db.rollback() return False task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value task_set.analysis_claim_token = None task_set.analysis_lease_until = None task_set.analysis_error_message = error_message await db.commit() return True async def _mark_segment_provider_success_pending_manual( *, segment_id: str, attempt_no: int, token: str, error_message: str, ) -> bool: """供应商已成功但本地结算失败：终止自动恢复，保留 HOLD 等待人工对账。""" async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if not ( segment and int(segment.analysis_attempt_no or 1) == attempt_no and segment.analysis_claim_token == token and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value ): await db.rollback() return False segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value segment.analysis_claim_token = None segment.analysis_lease_until = None segment.analysis_error_message = error_message await db.commit() return True async def _run_analyze_original_video(task_set_id: str, expected_attempt_no: int | None) -> None: token = uuid.uuid4().hex if expected_attempt_no is None: _log_stale_analysis_attempt( owner_type="shot_task_set", owner_id=task_set_id, expected_attempt_no=None, actual_attempt_no=None, analysis_status=None, reason="missing_expected_attempt_no", ) return attempt_no = max(1, int(expected_attempt_no)) async with async_session() as db: row = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .limit(1) ) initial = row.scalar_one_or_none() if not initial: return actual_attempt_no = int(initial.analysis_attempt_no or 1) if ( actual_attempt_no != attempt_no or initial.analysis_status not in (ShotAnalysisStatusEnum.PENDING.value, ShotAnalysisStatusEnum.PROCESSING.value) ): _log_stale_analysis_attempt( owner_type="shot_task_set", owner_id=task_set_id, expected_attempt_no=attempt_no, actual_attempt_no=actual_attempt_no, analysis_status=initial.analysis_status, user_id=str(initial.user_id), reason="attempt_or_status_mismatch_before_lock", ) await db.rollback() return await db.rollback() lease = await CeleryRuntimeLease.acquire( identity=RuntimeIdentity( domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value, owner_type="shot_task_set", owner_id=task_set_id, attempt_no=attempt_no, task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value, queue=ANALYSIS_QUEUE, ), lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no), hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY, zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY, token=token, ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180), heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30), pipeline_stage="analysis_processing", db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token), ) if lease is None: return task_set_user_id: str | None = None video_url: str | None = None try: async with async_session() as db: result = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() if not task_set: await db.rollback() return actual_attempt_no = int(task_set.analysis_attempt_no or 1) if ( actual_attempt_no != attempt_no or task_set.analysis_status not in (ShotAnalysisStatusEnum.PENDING.value, ShotAnalysisStatusEnum.PROCESSING.value) ): _log_stale_analysis_attempt( owner_type="shot_task_set", owner_id=task_set_id, expected_attempt_no=attempt_no, actual_attempt_no=actual_attempt_no, analysis_status=task_set.analysis_status, user_id=str(task_set.user_id), reason="attempt_or_status_mismatch_after_lock", ) await db.rollback() return current_lease = task_set.analysis_lease_until if ( task_set.analysis_claim_token and task_set.analysis_claim_token != token and current_lease and current_lease > _now() ): await db.rollback() return task_set_user_id = str(task_set.user_id) video_url = str(task_set.video_url) task_set.status = ShotTaskSetStatusEnum.ANALYZING.value task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value task_set.analysis_claim_token = token task_set.analysis_started_at = _now() task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)) task_set.analysis_error_message = None llm_billing_context = LlmBillingContext( user_id=task_set_user_id, owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value, owner_id=task_set_id, attempt_no=attempt_no, charge_kind=CreditRecordChargeKind.VIDEO_ANALYSIS.value, billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value, source_module=CreditRecordSourceModule.SHOT_REPLICATE.value, source_project_id=task_set_id, source_step_id=task_set_id, source_step_code=CreditRecordSourceStepCode.VIDEO_ANALYSIS.value, related_id=task_set_id, hold_config_key=LlmBillingConfigKey.HOLD_SHOT_VIDEO_ANALYSIS.value, description_prefix="拆镜复刻原视频分析", trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}", ) hold_validation = await ensure_hold_exists(db, llm_billing_context) if not hold_validation.can_execute: error_message = f"LLM账务状态异常（{hold_validation.state.value}），已终止原视频分析任务" task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value task_set.analysis_claim_token = None task_set.analysis_lease_until = None task_set.analysis_error_message = error_message await db.commit() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value, project_id=task_set_id, user_id=task_set_user_id, message=error_message, detail={"task_set_id": task_set_id, "analysis_attempt_no": attempt_no}, ) return await db.commit() log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_STARTED.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频拆镜分析开始", detail={ "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown", "analysis_attempt_no": attempt_no, "queue": ANALYSIS_QUEUE, }, ) provider_succeeded = False analyzed = None log_provider_start( llm_billing_context, detail={"analysis_mode": "full_breakdown", "video_url": video_url}, ) async with async_session() as call_db: analyzed = await analyze_video_for_shot_split( call_db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}", ) provider_succeeded = True log_provider_success(llm_billing_context, usage=analyzed.usage) try: await lease.ensure_owned() allow_business_write = True except RedisExecutionLockError: # Provider 已成功后不能让 Celery retry 再次调用模型；失去执行权时仅结算。 allow_business_write = False persist_outcome = await _persist_original_analysis_result( task_set_id=task_set_id, attempt_no=attempt_no, token=token, video_url=video_url, analyzed=analyzed, billing_context=llm_billing_context, allow_business_write=allow_business_write, ) if persist_outcome != "completed": return log_module_prompt_event( event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=task_set_id, user_id=task_set_user_id or "", module=MODULE, prompt_type="shot_video_analysis", request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {}, response=analyzed.result, token_usage=analyzed.usage, ) log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频拆镜分析成功", detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no}, ) except RedisExecutionLockError: raise except Exception as exc: if "llm_billing_context" in locals() and not locals().get("provider_succeeded", False): log_provider_failure(llm_billing_context, error=str(exc)) if "llm_billing_context" in locals() and locals().get("provider_succeeded", False) and locals().get("analyzed") is not None: try: try: await lease.ensure_owned() allow_business_write = True except RedisExecutionLockError: allow_business_write = False persist_outcome = await _persist_original_analysis_result( task_set_id=task_set_id, attempt_no=attempt_no, token=token, video_url=video_url, analyzed=analyzed, billing_context=llm_billing_context, allow_business_write=allow_business_write, ) if persist_outcome == "completed": log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频分析本地异常后幂等恢复成功", detail={"task_set_id": task_set_id, "analysis_attempt_no": attempt_no}, ) return except Exception as settlement_exc: manual_error = ( "供应商已成功，但本地结果保存或积分结算失败；已终止自动恢复并保留冻结积分，" f"需人工对账。error={settlement_exc}" ) await _mark_original_provider_success_pending_manual( task_set_id=task_set_id, attempt_no=attempt_no, token=token, error_message=manual_error, ) exc = settlement_exc elif "llm_billing_context" in locals(): async with async_session() as db: result = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = result.scalar_one_or_none() if ( task_set and int(task_set.analysis_attempt_no or 1) == attempt_no and task_set.analysis_claim_token == token and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value ): task_set_user_id = task_set_user_id or str(task_set.user_id) task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value task_set.analysis_claim_token = None task_set.analysis_lease_until = None task_set.analysis_error_message = str(exc) await release_on_failure(db, llm_billing_context, error=str(exc)) await db.commit() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value, project_id=task_set_id, user_id=task_set_user_id, message="原视频拆镜分析失败", detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no}, exc=exc, ) finally: await lease.close() async def _run_analyze_custom_segment_video(segment_id: str, expected_attempt_no: int | None) -> None: token = uuid.uuid4().hex if expected_attempt_no is None: _log_stale_analysis_attempt( owner_type="shot_segment", owner_id=segment_id, expected_attempt_no=None, actual_attempt_no=None, analysis_status=None, reason="missing_expected_attempt_no", ) return attempt_no = max(1, int(expected_attempt_no)) async with async_session() as db: row = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .limit(1) ) initial = row.scalar_one_or_none() if not initial or not initial.segment_video_url: return actual_attempt_no = int(initial.analysis_attempt_no or 1) if ( actual_attempt_no != attempt_no or initial.analysis_status not in (ShotSegmentAnalysisStatusEnum.PENDING.value, ShotSegmentAnalysisStatusEnum.PROCESSING.value) ): _log_stale_analysis_attempt( owner_type="shot_segment", owner_id=segment_id, expected_attempt_no=attempt_no, actual_attempt_no=actual_attempt_no, analysis_status=initial.analysis_status, task_set_id=str(initial.task_set_id), user_id=str(initial.user_id), reason="attempt_or_status_mismatch_before_lock", ) await db.rollback() return await db.rollback() lease = await CeleryRuntimeLease.acquire( identity=RuntimeIdentity( domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value, owner_type="shot_segment", owner_id=segment_id, attempt_no=attempt_no, task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value, queue=ANALYSIS_QUEUE, ), lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no), hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY, zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY, token=token, ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180), heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30), pipeline_stage="analysis_processing", db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token), ) if lease is None: return user_id: str | None = None task_set_id: str | None = None video_url: str | None = None try: async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if not segment or not segment.segment_video_url: await db.rollback() return actual_attempt_no = int(segment.analysis_attempt_no or 1) if ( actual_attempt_no != attempt_no or segment.analysis_status not in (ShotSegmentAnalysisStatusEnum.PENDING.value, ShotSegmentAnalysisStatusEnum.PROCESSING.value) ): _log_stale_analysis_attempt( owner_type="shot_segment", owner_id=segment_id, expected_attempt_no=attempt_no, actual_attempt_no=actual_attempt_no, analysis_status=segment.analysis_status, task_set_id=str(segment.task_set_id), user_id=str(segment.user_id), reason="attempt_or_status_mismatch_after_lock", ) await db.rollback() return current_lease = segment.analysis_lease_until if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now(): await db.rollback() return user_id = str(segment.user_id) task_set_id = str(segment.task_set_id) video_url = str(segment.segment_video_url) segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value segment.analysis_claim_token = token segment.analysis_started_at = _now() segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)) segment.analysis_error_message = None llm_billing_context = LlmBillingContext( user_id=user_id, owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value, owner_id=segment_id, attempt_no=attempt_no, charge_kind=CreditRecordChargeKind.VIDEO_ANALYSIS.value, billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value, source_module=CreditRecordSourceModule.SHOT_REPLICATE.value, source_project_id=task_set_id, source_step_id=segment_id, source_step_code=CreditRecordSourceStepCode.VIDEO_ANALYSIS.value, related_id=segment_id, hold_config_key=LlmBillingConfigKey.HOLD_SHOT_VIDEO_ANALYSIS.value, description_prefix="拆镜复刻片段视频分析", trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}", ) hold_validation = await ensure_hold_exists(db, llm_billing_context) if not hold_validation.can_execute: error_message = f"LLM账务状态异常（{hold_validation.state.value}），已终止片段视频分析任务" segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value segment.analysis_claim_token = None segment.analysis_lease_until = None segment.analysis_error_message = error_message await db.commit() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message=error_message, detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no}, ) return await db.commit() log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_STARTED.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义拆镜片段分析开始", detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no}, ) provider_succeeded = False analyzed = None log_provider_start( llm_billing_context, detail={"analysis_mode": "summary_only", "video_url": video_url}, ) async with async_session() as call_db: analyzed = await analyze_video_for_shot_split( call_db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}", ) provider_succeeded = True log_provider_success(llm_billing_context, usage=analyzed.usage) try: await lease.ensure_owned() allow_business_write = True except RedisExecutionLockError: allow_business_write = False persist_outcome = await _persist_segment_analysis_result( segment_id=segment_id, attempt_no=attempt_no, token=token, video_url=video_url, analyzed=analyzed, billing_context=llm_billing_context, allow_business_write=allow_business_write, ) if persist_outcome != "completed": return log_module_prompt_event( event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=segment_id, user_id=user_id or "", module=MODULE, prompt_type="shot_segment_analysis", request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {}, response=analyzed.result, token_usage=analyzed.usage, ) log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义拆镜片段分析成功", detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no}, ) except RedisExecutionLockError: raise except Exception as exc: if "llm_billing_context" in locals() and not locals().get("provider_succeeded", False): log_provider_failure(llm_billing_context, error=str(exc)) if "llm_billing_context" in locals() and locals().get("provider_succeeded", False) and locals().get("analyzed") is not None: try: try: await lease.ensure_owned() allow_business_write = True except RedisExecutionLockError: allow_business_write = False persist_outcome = await _persist_segment_analysis_result( segment_id=segment_id, attempt_no=attempt_no, token=token, video_url=video_url, analyzed=analyzed, billing_context=llm_billing_context, allow_business_write=allow_business_write, ) if persist_outcome == "completed": log_module_event_file( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="切片视频分析本地异常后幂等恢复成功", detail={"segment_id": segment_id, "analysis_attempt_no": attempt_no}, ) return except Exception as settlement_exc: manual_error = ( "供应商已成功，但本地结果保存或积分结算失败；已终止自动恢复并保留冻结积分，" f"需人工对账。error={settlement_exc}" ) await _mark_segment_provider_success_pending_manual( segment_id=segment_id, attempt_no=attempt_no, token=token, error_message=manual_error, ) exc = settlement_exc elif "llm_billing_context" in locals(): async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( segment and int(segment.analysis_attempt_no or 1) == attempt_no and segment.analysis_claim_token == token and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value ): user_id = user_id or str(segment.user_id) task_set_id = task_set_id or str(segment.task_set_id) segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value segment.analysis_claim_token = None segment.analysis_lease_until = None segment.analysis_error_message = str(exc) await release_on_failure(db, llm_billing_context, error=str(exc)) await db.commit() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义拆镜片段分析失败", detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no}, exc=exc, ) finally: await lease.close() async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool: async with async_session() as db: result = await db.execute( update(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ShotReplicateSegment.split_retry_count == attempt_no, ShotReplicateSegment.split_claim_token == token, ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value, ) .values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))) ) await db.commit() return bool(result.rowcount == 1) async def _mark_auto_segment_analysis_dispatch_failed( *, segment_id: str, expected_attempt_no: int, error_message: str, ) -> bool: """切片成功后的自动分析投递失败补偿；不回滚或清理已完成的切片文件。""" task_set_id: str | None = None user_id: str | None = None async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where( ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None), ) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( not segment or int(segment.analysis_attempt_no or 1) != int(expected_attempt_no) or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PENDING.value ): await db.rollback() return False task_set_id = str(segment.task_set_id) user_id = str(segment.user_id) segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value segment.analysis_claim_token = None segment.analysis_started_at = None segment.analysis_lease_until = None segment.analysis_error_message = error_message await release_on_failure( db, build_segment_analysis_billing_context(segment), error=error_message, ) await db.commit() log_module_error( module=MODULE, event_type=ShotReplicateLogEventEnum.CELERY_DISPATCH_FAILED.value, project_id=task_set_id, step_id=segment_id, user_id=user_id, message="自定义切片分析任务投递失败，已标记分析失败并释放冻结积分", detail={ "segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": expected_attempt_no, }, exc=RuntimeError(error_message), ) return True async def _run_split_one_segment(segment_id: str) -> None: async with async_session() as db: row = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .limit(1) ) initial = row.scalar_one_or_none() if not initial: return if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url: return attempt = int(initial.split_retry_count or 0) + 1 await db.rollback() token = uuid.uuid4().hex lease = await CeleryRuntimeLease.acquire( identity=RuntimeIdentity( domain=CeleryRuntimeDomain.SHOT_SPLIT.value, owner_type="shot_segment", owner_id=segment_id, attempt_no=attempt, task_name=CeleryTaskName.SHOT_SPLIT_ONE.value, queue=SPLIT_QUEUE, ), lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}", hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY, zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY, token=token, ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600), heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30), pipeline_stage=ShotSplitStatusEnum.PROCESSING.value, db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token), ) if lease is None: return semaphore_key: str | None = None user_id: str | None = None task_set_id: str | None = None source_path: str | None = None split_result = None try: semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}") if not semaphore_key: delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10)) log_module_event_file( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING", step_id=segment_id, message="拆镜 ffmpeg 并发闸门已满，稍后重试", detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt}, ) if celery_app: split_one_segment.apply_async( args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL, ) return async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if not segment: await db.rollback() return user_id = str(segment.user_id) task_set_id = str(segment.task_set_id) task_set_result = await db.execute( select(ShotReplicateTaskSet) .where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .with_for_update() .limit(1) ) task_set = task_set_result.scalar_one_or_none() if not task_set: await db.rollback() return if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url: await db.rollback() return if ( segment.split_claim_token and segment.split_claim_token != token and segment.split_lease_until and segment.split_lease_until > _now() ): await db.rollback() return validate_split_range( start_second=segment.start_second, end_second=segment.end_second, video_duration_seconds=task_set.video_duration_seconds, ) now = _now() segment.split_status = ShotSplitStatusEnum.PROCESSING.value segment.split_claim_token = token segment.split_started_at = now segment.split_lease_until = _lease_until(now) segment.split_retry_count = attempt segment.split_next_retry_at = None segment.split_last_error = None task_set.status = ShotTaskSetStatusEnum.SPLITTING.value task_set.split_status = ShotSplitStatusEnum.PROCESSING.value source_path = str(task_set.video_path) date_dir = (segment.created_at or now).strftime("%Y/%m/%d") start_second = float(segment.start_second) end_second = float(segment.end_second) await db.commit() log_module_event_file( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_STARTED", project_id=task_set_id, step_id=segment_id, user_id=user_id, message="拆镜片段 ffmpeg 切割开始", detail={ "segment_id": segment_id, "task_set_id": task_set_id, "source_path": source_path, "start_second": start_second, "end_second": end_second, "attempt": attempt, "queue": SPLIT_QUEUE, }, ) split_result = await split_video_segment_async( source_path=source_path or "", segment_id=segment_id, start_second=start_second, end_second=end_second, date_dir=date_dir, attempt_key=f"attempt-{attempt}-{token[-8:]}", finalize=False, ) await lease.ensure_owned() async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( not segment or int(segment.split_retry_count or 0) != attempt or segment.split_claim_token != token or segment.split_status != ShotSplitStatusEnum.PROCESSING.value ): await db.rollback() cleanup_split_result(split_result) return split_result = finalize_split_result(split_result) segment.segment_video_url = split_result.url segment.segment_video_path = split_result.path await record_shot_segment_upload_resource( db, segment=segment, storage_path=split_result.path, resource_url=split_result.url, file_size_bytes=split_result.file_size_bytes, ) segment.split_status = ShotSplitStatusEnum.COMPLETED.value segment.split_claim_token = None segment.split_completed_at = _now() segment.split_lease_until = None segment.split_next_retry_at = None segment.split_last_error = None source_mode = str(segment.source_mode) final_task_set_id = str(segment.task_set_id) final_user_id = str(segment.user_id) analysis_attempt_no = max(1, int(segment.analysis_attempt_no or 1)) await refresh_task_set_split_summary(db, segment.task_set_id) await db.commit() log_module_event_file( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_SUCCESS", project_id=final_task_set_id, step_id=segment_id, user_id=final_user_id, message="拆镜片段 ffmpeg 切割成功", detail={ "segment_id": segment_id, "task_set_id": final_task_set_id, "segment_video_url": split_result.url, "segment_video_path": split_result.path, "source_mode": source_mode, "attempt": attempt, }, ) if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app: try: analyze_custom_segment_video.apply_async( args=[segment_id, analysis_attempt_no], queue=ANALYSIS_QUEUE, countdown=0, task_id=f"shot-analysis:segment:{segment_id}:attempt:{analysis_attempt_no}", ) except Exception as dispatch_exc: await _mark_auto_segment_analysis_dispatch_failed( segment_id=segment_id, expected_attempt_no=analysis_attempt_no, error_message=f"自定义切片分析任务投递失败: {dispatch_exc}", ) except RedisExecutionLockError: cleanup_split_result(split_result) raise except Exception as exc: cleanup_split_result(split_result) next_retry_delay: int | None = None final_failed = False async with async_session() as db: result = await db.execute( select(ShotReplicateSegment) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .with_for_update() .limit(1) ) segment = result.scalar_one_or_none() if ( segment and int(segment.split_retry_count or 0) == attempt and segment.split_claim_token == token and segment.split_status == ShotSplitStatusEnum.PROCESSING.value ): user_id = user_id or str(segment.user_id) task_set_id = task_set_id or str(segment.task_set_id) segment.split_claim_token = None segment.split_last_error = str(exc) segment.split_lease_until = None if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3): segment.split_status = ShotSplitStatusEnum.FAILED.value segment.split_next_retry_at = None final_failed = True # 切片最终失败不释放片段分析 HOLD；手动切片重试继续沿用 # 原 attempt。用户最终删除片段/任务集时再做取消补偿。 else: segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value segment.split_next_retry_at = _retry_at(attempt) next_retry_delay = max( 1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()), ) await refresh_task_set_split_summary(db, segment.task_set_id) await db.commit() else: await db.rollback() return if next_retry_delay and celery_app: split_one_segment.apply_async( args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER, ) log_module_error( module=MODULE, event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING", project_id=task_set_id, step_id=segment_id, user_id=user_id, message="拆镜片段 ffmpeg 切割失败" if final_failed else "拆镜片段 ffmpeg 切割失败，等待重试", detail={ "segment_id": segment_id, "task_set_id": task_set_id, "source_path": source_path, "next_retry_delay_seconds": next_retry_delay, "final_failed": final_failed, "attempt": attempt, }, exc=exc, ) finally: await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}") await lease.close() async def _run_recover_split_tasks_once() -> dict[str, Any]: from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once barrier = await guard_periodic_recovery() if barrier is not None: return barrier lease = await RedisExecutionLockLease.acquire( lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY, ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600), renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)), log_context="shot_split_recovery", ) if lease is None: return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY} async with lease: async with async_session() as db: result = await recover_shot_split_tasks_once(db) result["execution_lock"] = "lock_acquired" return result async def _run_recover_analysis_tasks_once() -> dict[str, Any]: from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once barrier = await guard_periodic_recovery() if barrier is not None: return barrier lease = await RedisExecutionLockLease.acquire( lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY, ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600), renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)), log_context="shot_analysis_recovery", ) if lease is None: return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY} async with lease: async with async_session() as db: result = await recover_shot_analysis_tasks_once(db) result["execution_lock"] = "lock_acquired" return result if celery_app: @celery_app.task( name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value, bind=True, max_retries=3, default_retry_delay=60, soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS, ignore_result=True, ) def analyze_original_video(self, task_set_id: str, expected_attempt_no: int | None = None) -> None: try: return run_async(_run_analyze_original_video(task_set_id, expected_attempt_no)) except RedisExecutionLockError as exc: raise self.retry(exc=exc, countdown=60) @celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True) def split_one_segment(self, segment_id: str) -> None: try: return run_async(_run_split_one_segment(segment_id)) except RedisExecutionLockError as exc: raise self.retry(exc=exc, countdown=30) @celery_app.task( name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value, bind=True, max_retries=3, default_retry_delay=60, soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS, ignore_result=True, ) def analyze_custom_segment_video(self, segment_id: str, expected_attempt_no: int | None = None) -> None: try: return run_async(_run_analyze_custom_segment_video(segment_id, expected_attempt_no)) except RedisExecutionLockError as exc: raise self.retry(exc=exc, countdown=60) @celery_app.task( name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value, bind=True, soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS, ) def recover_analysis_tasks_once(self) -> dict[str, Any]: return run_async(_run_recover_analysis_tasks_once()) @celery_app.task( name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value, bind=True, soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS, time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS, ) def recover_split_tasks_once(self) -> dict[str, Any]: return run_async(_run_recover_split_tasks_once()) else: class _DisabledTask: def delay(self, *args: Any, **kwargs: Any) -> None: raise RuntimeError("Celery is disabled") def apply_async(self, *args: Any, **kwargs: Any) -> None: raise RuntimeError("Celery is disabled") analyze_original_video = _DisabledTask() split_one_segment = _DisabledTask() analyze_custom_segment_video = _DisabledTask() recover_split_tasks_once = _DisabledTask() recover_analysis_tasks_once = _DisabledTask()