923 lines
39 KiB
Python
923 lines
39 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select, update
|
|
|
|
from app.config import settings
|
|
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
|
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
|
from app.enums.shot_replicate import (
|
|
ModuleCodeEnum,
|
|
ShotAnalysisStatusEnum,
|
|
ShotReplicateLogEventEnum,
|
|
ShotSegmentAnalysisStatusEnum,
|
|
ShotSegmentSourceModeEnum,
|
|
ShotSplitStatusEnum,
|
|
ShotTaskSetStatusEnum,
|
|
)
|
|
from app.models.base import async_session
|
|
from app.models.shot_replicate_segment import ShotReplicateSegment
|
|
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
|
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
|
|
from app.services.redis_registry_service import (
|
|
RedisExecutionLockError,
|
|
RedisExecutionLockLease,
|
|
redis_acquire_lock,
|
|
redis_release_lock,
|
|
)
|
|
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
|
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
|
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
|
|
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
|
|
from app.services.generation.billing_service import charge_shot_video_analysis_usage
|
|
from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async
|
|
from app.services.upload_video_asset_service import validate_split_range
|
|
from app.services.upload_resource import record_shot_segment_upload_resource
|
|
from app.tasks.async_runner import run_async
|
|
from app.tasks.celery_app import celery_app
|
|
|
|
logger = logging.getLogger("video_gen")
|
|
|
|
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
|
|
SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value
|
|
ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _lease_until(now: datetime | None = None) -> datetime:
|
|
return (now or _now()) + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))
|
|
|
|
|
|
def _retry_at(attempt: int, now: datetime | None = None) -> datetime:
|
|
base = int(settings.SHOT_SPLIT_RETRY_BACKOFF_SECONDS or settings.DOWNLOAD_TASK_RETRY_BACKOFF_SECONDS or 30)
|
|
return (now or _now()) + timedelta(seconds=max(1, base * max(1, attempt)))
|
|
|
|
|
|
async def _acquire_split_semaphore(segment_id: str) -> str | None:
|
|
"""简单 Redis 并发闸门：用固定槽位锁限制 ffmpeg 同时运行数量。"""
|
|
max_concurrent = max(1, int(settings.SHOT_SPLIT_MAX_CONCURRENT or 1))
|
|
ttl = int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)
|
|
for slot in range(max_concurrent):
|
|
key = f"{settings.SHOT_SPLIT_SEMAPHORE_KEY_PREFIX}:{slot}"
|
|
token = await redis_acquire_lock(lock_key=key, ttl_seconds=ttl, token=segment_id, log_context="shot_split_semaphore")
|
|
if token:
|
|
return key
|
|
return None
|
|
|
|
|
|
async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> None:
|
|
if lock_key:
|
|
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
|
|
|
|
|
|
async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
update(ShotReplicateTaskSet)
|
|
.where(
|
|
ShotReplicateTaskSet.id == task_set_id,
|
|
ShotReplicateTaskSet.deleted_at.is_(None),
|
|
ShotReplicateTaskSet.analysis_attempt_no == attempt_no,
|
|
ShotReplicateTaskSet.analysis_claim_token == token,
|
|
ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value,
|
|
)
|
|
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
|
|
)
|
|
await db.commit()
|
|
return bool(result.rowcount == 1)
|
|
|
|
|
|
async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
update(ShotReplicateSegment)
|
|
.where(
|
|
ShotReplicateSegment.id == segment_id,
|
|
ShotReplicateSegment.deleted_at.is_(None),
|
|
ShotReplicateSegment.analysis_attempt_no == attempt_no,
|
|
ShotReplicateSegment.analysis_claim_token == token,
|
|
ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value,
|
|
)
|
|
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
|
|
)
|
|
await db.commit()
|
|
return bool(result.rowcount == 1)
|
|
|
|
|
|
def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str:
|
|
return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}"
|
|
|
|
|
|
async def _run_analyze_original_video(task_set_id: str) -> None:
|
|
token = uuid.uuid4().hex
|
|
attempt_no = 1
|
|
|
|
async with async_session() as db:
|
|
row = await db.execute(
|
|
select(ShotReplicateTaskSet)
|
|
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
|
.limit(1)
|
|
)
|
|
initial = row.scalar_one_or_none()
|
|
if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
|
return
|
|
attempt_no = int(initial.analysis_attempt_no or 1)
|
|
await db.rollback()
|
|
|
|
lease = await CeleryRuntimeLease.acquire(
|
|
identity=RuntimeIdentity(
|
|
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
|
|
owner_type="shot_task_set",
|
|
owner_id=task_set_id,
|
|
attempt_no=attempt_no,
|
|
task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
|
|
queue=ANALYSIS_QUEUE,
|
|
),
|
|
lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no),
|
|
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
|
|
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
|
|
token=token,
|
|
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
|
|
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
|
|
pipeline_stage="analysis_processing",
|
|
db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token),
|
|
)
|
|
if lease is None:
|
|
return
|
|
|
|
task_set_user_id: str | None = None
|
|
video_url: str | None = None
|
|
try:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateTaskSet)
|
|
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
task_set = result.scalar_one_or_none()
|
|
if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
|
await db.rollback()
|
|
return
|
|
current_lease = task_set.analysis_lease_until
|
|
if (
|
|
task_set.analysis_claim_token
|
|
and task_set.analysis_claim_token != token
|
|
and current_lease
|
|
and current_lease > _now()
|
|
):
|
|
await db.rollback()
|
|
return
|
|
if int(task_set.analysis_attempt_no or 1) != attempt_no:
|
|
await db.rollback()
|
|
return
|
|
task_set_user_id = str(task_set.user_id)
|
|
video_url = str(task_set.video_url)
|
|
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
|
|
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
|
|
task_set.analysis_claim_token = token
|
|
task_set.analysis_started_at = _now()
|
|
task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
|
|
task_set.analysis_error_message = None
|
|
await db.commit()
|
|
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type=ShotReplicateLogEventEnum.ANALYSIS_STARTED.value,
|
|
project_id=task_set_id,
|
|
user_id=task_set_user_id,
|
|
message="原视频拆镜分析开始",
|
|
detail={
|
|
"task_set_id": task_set_id,
|
|
"video_url": video_url,
|
|
"analysis_mode": "full_breakdown",
|
|
"analysis_attempt_no": attempt_no,
|
|
"queue": ANALYSIS_QUEUE,
|
|
},
|
|
)
|
|
|
|
async with async_session() as call_db:
|
|
analyzed = await analyze_video_for_shot_split(
|
|
call_db,
|
|
video_url or "",
|
|
user_id=task_set_user_id,
|
|
mode="full_breakdown",
|
|
task_set_id=task_set_id,
|
|
trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}",
|
|
)
|
|
await lease.ensure_owned()
|
|
result = await call_db.execute(
|
|
select(ShotReplicateTaskSet)
|
|
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
task_set = result.scalar_one_or_none()
|
|
if (
|
|
not task_set
|
|
or int(task_set.analysis_attempt_no or 1) != attempt_no
|
|
or task_set.analysis_claim_token != token
|
|
or str(task_set.video_url) != str(video_url)
|
|
or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value
|
|
):
|
|
await call_db.rollback()
|
|
return
|
|
result_json = analyzed.result
|
|
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
|
|
task_set.original_video_category = str(result_json.get("原视频分类") or "无")
|
|
task_set.original_video_audience = str(result_json.get("原视频受众人群") or "无")
|
|
task_set.ai_suggestion_json = result_json.get("拆镜内容剖析") or []
|
|
task_set.analysis_raw_json = analyzed.raw_response
|
|
task_set.analysis_result_json = result_json
|
|
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
|
|
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
|
|
task_set.analysis_claim_token = None
|
|
task_set.analysis_lease_until = None
|
|
task_set.analysis_error_message = None
|
|
await charge_shot_video_analysis_usage(
|
|
call_db,
|
|
user_id=task_set.user_id,
|
|
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
|
|
owner_id=task_set.id,
|
|
usage=analyzed.usage,
|
|
description="拆镜复刻-原视频分析",
|
|
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
|
|
source_project_id=task_set.id,
|
|
attempt_no=attempt_no,
|
|
)
|
|
await call_db.commit()
|
|
|
|
log_module_prompt_event(
|
|
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
|
|
project_id=task_set_id,
|
|
step_id=task_set_id,
|
|
user_id=task_set_user_id or "",
|
|
module=MODULE,
|
|
prompt_type="shot_video_analysis",
|
|
request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {},
|
|
response=analyzed.result,
|
|
token_usage=analyzed.usage,
|
|
)
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
|
|
project_id=task_set_id,
|
|
user_id=task_set_user_id,
|
|
message="原视频拆镜分析成功",
|
|
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no},
|
|
)
|
|
except RedisExecutionLockError:
|
|
raise
|
|
except Exception as exc:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateTaskSet)
|
|
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
task_set = result.scalar_one_or_none()
|
|
if (
|
|
task_set
|
|
and int(task_set.analysis_attempt_no or 1) == attempt_no
|
|
and task_set.analysis_claim_token == token
|
|
and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value
|
|
):
|
|
task_set_user_id = task_set_user_id or str(task_set.user_id)
|
|
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
|
|
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
|
|
task_set.analysis_claim_token = None
|
|
task_set.analysis_lease_until = None
|
|
task_set.analysis_error_message = str(exc)
|
|
await db.commit()
|
|
else:
|
|
await db.rollback()
|
|
log_module_error(
|
|
module=MODULE,
|
|
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
|
|
project_id=task_set_id,
|
|
user_id=task_set_user_id,
|
|
message="原视频拆镜分析失败",
|
|
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
|
|
exc=exc,
|
|
)
|
|
finally:
|
|
await lease.close()
|
|
|
|
|
|
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|
token = uuid.uuid4().hex
|
|
attempt_no = 1
|
|
async with async_session() as db:
|
|
row = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.limit(1)
|
|
)
|
|
initial = row.scalar_one_or_none()
|
|
if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
|
|
return
|
|
attempt_no = int(initial.analysis_attempt_no or 1)
|
|
await db.rollback()
|
|
|
|
lease = await CeleryRuntimeLease.acquire(
|
|
identity=RuntimeIdentity(
|
|
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
|
|
owner_type="shot_segment",
|
|
owner_id=segment_id,
|
|
attempt_no=attempt_no,
|
|
task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
|
|
queue=ANALYSIS_QUEUE,
|
|
),
|
|
lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no),
|
|
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
|
|
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
|
|
token=token,
|
|
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
|
|
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
|
|
pipeline_stage="analysis_processing",
|
|
db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token),
|
|
)
|
|
if lease is None:
|
|
return
|
|
|
|
user_id: str | None = None
|
|
task_set_id: str | None = None
|
|
video_url: str | None = None
|
|
try:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
segment = result.scalar_one_or_none()
|
|
if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
|
|
await db.rollback()
|
|
return
|
|
current_lease = segment.analysis_lease_until
|
|
if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now():
|
|
await db.rollback()
|
|
return
|
|
if int(segment.analysis_attempt_no or 1) != attempt_no:
|
|
await db.rollback()
|
|
return
|
|
user_id = str(segment.user_id)
|
|
task_set_id = str(segment.task_set_id)
|
|
video_url = str(segment.segment_video_url)
|
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
|
segment.analysis_claim_token = token
|
|
segment.analysis_started_at = _now()
|
|
segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
|
|
segment.analysis_error_message = None
|
|
await db.commit()
|
|
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_STARTED.value,
|
|
project_id=task_set_id,
|
|
step_id=segment_id,
|
|
user_id=user_id,
|
|
message="自定义拆镜片段分析开始",
|
|
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
|
|
)
|
|
|
|
async with async_session() as call_db:
|
|
analyzed = await analyze_video_for_shot_split(
|
|
call_db,
|
|
video_url or "",
|
|
user_id=user_id,
|
|
mode="summary_only",
|
|
task_set_id=task_set_id,
|
|
segment_id=segment_id,
|
|
trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}",
|
|
)
|
|
await lease.ensure_owned()
|
|
result = await call_db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
segment = result.scalar_one_or_none()
|
|
if (
|
|
not segment
|
|
or int(segment.analysis_attempt_no or 1) != attempt_no
|
|
or segment.analysis_claim_token != token
|
|
or str(segment.segment_video_url) != str(video_url)
|
|
or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
|
):
|
|
await call_db.rollback()
|
|
return
|
|
result_json = analyzed.result
|
|
segment.original_video_content = str(result_json.get("原视频内容") or "无")
|
|
segment.original_video_category = str(result_json.get("原视频分类") or "无")
|
|
segment.original_video_audience = str(result_json.get("原视频受众人群") or "无")
|
|
segment.segment_content = segment.original_video_content
|
|
segment.segment_category = segment.original_video_category
|
|
segment.segment_audience = segment.original_video_audience
|
|
segment.analysis_json = result_json
|
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
|
|
segment.analysis_claim_token = None
|
|
segment.analysis_lease_until = None
|
|
segment.analysis_error_message = None
|
|
await charge_shot_video_analysis_usage(
|
|
call_db,
|
|
user_id=segment.user_id,
|
|
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
|
|
owner_id=segment.id,
|
|
usage=analyzed.usage,
|
|
description="拆镜复刻-片段视频分析",
|
|
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
|
|
source_project_id=segment.task_set_id,
|
|
source_step_id=segment.id,
|
|
attempt_no=attempt_no,
|
|
)
|
|
await call_db.commit()
|
|
|
|
log_module_prompt_event(
|
|
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
|
|
project_id=task_set_id,
|
|
step_id=segment_id,
|
|
user_id=user_id or "",
|
|
module=MODULE,
|
|
prompt_type="shot_segment_analysis",
|
|
request=analyzed.usage.get("log_request") if isinstance(analyzed.usage, dict) else {},
|
|
response=analyzed.result,
|
|
token_usage=analyzed.usage,
|
|
)
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
|
|
project_id=task_set_id,
|
|
step_id=segment_id,
|
|
user_id=user_id,
|
|
message="自定义拆镜片段分析成功",
|
|
detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no},
|
|
)
|
|
except RedisExecutionLockError:
|
|
raise
|
|
except Exception as exc:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
segment = result.scalar_one_or_none()
|
|
if (
|
|
segment
|
|
and int(segment.analysis_attempt_no or 1) == attempt_no
|
|
and segment.analysis_claim_token == token
|
|
and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
|
):
|
|
user_id = user_id or str(segment.user_id)
|
|
task_set_id = task_set_id or str(segment.task_set_id)
|
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
|
|
segment.analysis_claim_token = None
|
|
segment.analysis_lease_until = None
|
|
segment.analysis_error_message = str(exc)
|
|
await db.commit()
|
|
else:
|
|
await db.rollback()
|
|
log_module_error(
|
|
module=MODULE,
|
|
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
|
|
project_id=task_set_id,
|
|
step_id=segment_id,
|
|
user_id=user_id,
|
|
message="自定义拆镜片段分析失败",
|
|
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
|
|
exc=exc,
|
|
)
|
|
finally:
|
|
await lease.close()
|
|
|
|
|
|
async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool:
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
update(ShotReplicateSegment)
|
|
.where(
|
|
ShotReplicateSegment.id == segment_id,
|
|
ShotReplicateSegment.deleted_at.is_(None),
|
|
ShotReplicateSegment.split_retry_count == attempt_no,
|
|
ShotReplicateSegment.split_claim_token == token,
|
|
ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value,
|
|
)
|
|
.values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)))
|
|
)
|
|
await db.commit()
|
|
return bool(result.rowcount == 1)
|
|
|
|
|
|
async def _run_split_one_segment(segment_id: str) -> None:
|
|
async with async_session() as db:
|
|
row = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.limit(1)
|
|
)
|
|
initial = row.scalar_one_or_none()
|
|
if not initial:
|
|
return
|
|
if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url:
|
|
return
|
|
attempt = int(initial.split_retry_count or 0) + 1
|
|
await db.rollback()
|
|
|
|
token = uuid.uuid4().hex
|
|
lease = await CeleryRuntimeLease.acquire(
|
|
identity=RuntimeIdentity(
|
|
domain=CeleryRuntimeDomain.SHOT_SPLIT.value,
|
|
owner_type="shot_segment",
|
|
owner_id=segment_id,
|
|
attempt_no=attempt,
|
|
task_name=CeleryTaskName.SHOT_SPLIT_ONE.value,
|
|
queue=SPLIT_QUEUE,
|
|
),
|
|
lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}",
|
|
hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
|
|
zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
|
|
token=token,
|
|
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
|
|
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
|
|
pipeline_stage=ShotSplitStatusEnum.PROCESSING.value,
|
|
db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token),
|
|
)
|
|
if lease is None:
|
|
return
|
|
|
|
semaphore_key: str | None = None
|
|
user_id: str | None = None
|
|
task_set_id: str | None = None
|
|
source_path: str | None = None
|
|
split_result = None
|
|
try:
|
|
semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}")
|
|
if not semaphore_key:
|
|
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
|
|
step_id=segment_id,
|
|
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
|
|
detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt},
|
|
)
|
|
if celery_app:
|
|
split_one_segment.apply_async(
|
|
args=[segment_id],
|
|
queue=SPLIT_QUEUE,
|
|
countdown=delay,
|
|
priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
|
|
)
|
|
return
|
|
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
segment = result.scalar_one_or_none()
|
|
if not segment:
|
|
await db.rollback()
|
|
return
|
|
user_id = str(segment.user_id)
|
|
task_set_id = str(segment.task_set_id)
|
|
task_set_result = await db.execute(
|
|
select(ShotReplicateTaskSet)
|
|
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
task_set = task_set_result.scalar_one_or_none()
|
|
if not task_set:
|
|
await db.rollback()
|
|
return
|
|
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
|
|
await db.rollback()
|
|
return
|
|
if (
|
|
segment.split_claim_token
|
|
and segment.split_claim_token != token
|
|
and segment.split_lease_until
|
|
and segment.split_lease_until > _now()
|
|
):
|
|
await db.rollback()
|
|
return
|
|
|
|
validate_split_range(
|
|
start_second=segment.start_second,
|
|
end_second=segment.end_second,
|
|
video_duration_seconds=task_set.video_duration_seconds,
|
|
)
|
|
now = _now()
|
|
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
|
|
segment.split_claim_token = token
|
|
segment.split_started_at = now
|
|
segment.split_lease_until = _lease_until(now)
|
|
segment.split_retry_count = attempt
|
|
segment.split_next_retry_at = None
|
|
segment.split_last_error = None
|
|
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
|
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
|
source_path = str(task_set.video_path)
|
|
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
|
|
start_second = float(segment.start_second)
|
|
end_second = float(segment.end_second)
|
|
await db.commit()
|
|
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type="SHOT_SEGMENT_SPLIT_STARTED",
|
|
project_id=task_set_id,
|
|
step_id=segment_id,
|
|
user_id=user_id,
|
|
message="拆镜片段 ffmpeg 切割开始",
|
|
detail={
|
|
"segment_id": segment_id,
|
|
"task_set_id": task_set_id,
|
|
"source_path": source_path,
|
|
"start_second": start_second,
|
|
"end_second": end_second,
|
|
"attempt": attempt,
|
|
"queue": SPLIT_QUEUE,
|
|
},
|
|
)
|
|
|
|
split_result = await split_video_segment_async(
|
|
source_path=source_path or "",
|
|
segment_id=segment_id,
|
|
start_second=start_second,
|
|
end_second=end_second,
|
|
date_dir=date_dir,
|
|
attempt_key=f"attempt-{attempt}-{token[-8:]}",
|
|
finalize=False,
|
|
)
|
|
await lease.ensure_owned()
|
|
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
segment = result.scalar_one_or_none()
|
|
if (
|
|
not segment
|
|
or int(segment.split_retry_count or 0) != attempt
|
|
or segment.split_claim_token != token
|
|
or segment.split_status != ShotSplitStatusEnum.PROCESSING.value
|
|
):
|
|
await db.rollback()
|
|
cleanup_split_result(split_result)
|
|
return
|
|
split_result = finalize_split_result(split_result)
|
|
segment.segment_video_url = split_result.url
|
|
segment.segment_video_path = split_result.path
|
|
await record_shot_segment_upload_resource(
|
|
db,
|
|
segment=segment,
|
|
storage_path=split_result.path,
|
|
resource_url=split_result.url,
|
|
file_size_bytes=split_result.file_size_bytes,
|
|
)
|
|
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
|
|
segment.split_claim_token = None
|
|
segment.split_completed_at = _now()
|
|
segment.split_lease_until = None
|
|
segment.split_next_retry_at = None
|
|
segment.split_last_error = None
|
|
source_mode = str(segment.source_mode)
|
|
final_task_set_id = str(segment.task_set_id)
|
|
final_user_id = str(segment.user_id)
|
|
await refresh_task_set_split_summary(db, segment.task_set_id)
|
|
await db.commit()
|
|
|
|
log_module_event_file(
|
|
module=MODULE,
|
|
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
|
|
project_id=final_task_set_id,
|
|
step_id=segment_id,
|
|
user_id=final_user_id,
|
|
message="拆镜片段 ffmpeg 切割成功",
|
|
detail={
|
|
"segment_id": segment_id,
|
|
"task_set_id": final_task_set_id,
|
|
"segment_video_url": split_result.url,
|
|
"segment_video_path": split_result.path,
|
|
"source_mode": source_mode,
|
|
"attempt": attempt,
|
|
},
|
|
)
|
|
|
|
if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
|
|
analyze_custom_segment_video.apply_async(
|
|
args=[segment_id],
|
|
queue=ANALYSIS_QUEUE,
|
|
countdown=0,
|
|
task_id=f"shot-analysis:segment:{segment_id}:attempt:1",
|
|
)
|
|
except RedisExecutionLockError:
|
|
cleanup_split_result(split_result)
|
|
raise
|
|
except Exception as exc:
|
|
cleanup_split_result(split_result)
|
|
next_retry_delay: int | None = None
|
|
final_failed = False
|
|
async with async_session() as db:
|
|
result = await db.execute(
|
|
select(ShotReplicateSegment)
|
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
.with_for_update()
|
|
.limit(1)
|
|
)
|
|
segment = result.scalar_one_or_none()
|
|
if (
|
|
segment
|
|
and int(segment.split_retry_count or 0) == attempt
|
|
and segment.split_claim_token == token
|
|
and segment.split_status == ShotSplitStatusEnum.PROCESSING.value
|
|
):
|
|
user_id = user_id or str(segment.user_id)
|
|
task_set_id = task_set_id or str(segment.task_set_id)
|
|
segment.split_claim_token = None
|
|
segment.split_last_error = str(exc)
|
|
segment.split_lease_until = None
|
|
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
|
|
segment.split_status = ShotSplitStatusEnum.FAILED.value
|
|
segment.split_next_retry_at = None
|
|
final_failed = True
|
|
else:
|
|
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
|
|
segment.split_next_retry_at = _retry_at(attempt)
|
|
next_retry_delay = max(
|
|
1,
|
|
int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()),
|
|
)
|
|
await refresh_task_set_split_summary(db, segment.task_set_id)
|
|
await db.commit()
|
|
else:
|
|
await db.rollback()
|
|
return
|
|
|
|
if next_retry_delay and celery_app:
|
|
split_one_segment.apply_async(
|
|
args=[segment_id],
|
|
queue=SPLIT_QUEUE,
|
|
countdown=next_retry_delay,
|
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
|
)
|
|
log_module_error(
|
|
module=MODULE,
|
|
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
|
|
project_id=task_set_id,
|
|
step_id=segment_id,
|
|
user_id=user_id,
|
|
message="拆镜片段 ffmpeg 切割失败" if final_failed else "拆镜片段 ffmpeg 切割失败，等待重试",
|
|
detail={
|
|
"segment_id": segment_id,
|
|
"task_set_id": task_set_id,
|
|
"source_path": source_path,
|
|
"next_retry_delay_seconds": next_retry_delay,
|
|
"final_failed": final_failed,
|
|
"attempt": attempt,
|
|
},
|
|
exc=exc,
|
|
)
|
|
finally:
|
|
await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}")
|
|
await lease.close()
|
|
|
|
|
|
async def _run_recover_split_tasks_once() -> dict[str, Any]:
|
|
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
|
|
|
|
barrier = await guard_periodic_recovery()
|
|
if barrier is not None:
|
|
return barrier
|
|
lease = await RedisExecutionLockLease.acquire(
|
|
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
|
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
|
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
|
|
log_context="shot_split_recovery",
|
|
)
|
|
if lease is None:
|
|
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
|
|
async with lease:
|
|
async with async_session() as db:
|
|
result = await recover_shot_split_tasks_once(db)
|
|
result["execution_lock"] = "lock_acquired"
|
|
return result
|
|
|
|
|
|
async def _run_recover_analysis_tasks_once() -> dict[str, Any]:
|
|
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
|
|
|
|
barrier = await guard_periodic_recovery()
|
|
if barrier is not None:
|
|
return barrier
|
|
lease = await RedisExecutionLockLease.acquire(
|
|
lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
|
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
|
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
|
|
log_context="shot_analysis_recovery",
|
|
)
|
|
if lease is None:
|
|
return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY}
|
|
async with lease:
|
|
async with async_session() as db:
|
|
result = await recover_shot_analysis_tasks_once(db)
|
|
result["execution_lock"] = "lock_acquired"
|
|
return result
|
|
|
|
|
|
if celery_app:
|
|
|
|
@celery_app.task(
|
|
name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
|
|
bind=True,
|
|
max_retries=3,
|
|
default_retry_delay=60,
|
|
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
|
|
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
|
|
ignore_result=True,
|
|
)
|
|
def analyze_original_video(self, task_set_id: str) -> None:
|
|
try:
|
|
return run_async(_run_analyze_original_video(task_set_id))
|
|
except RedisExecutionLockError as exc:
|
|
raise self.retry(exc=exc, countdown=60)
|
|
|
|
|
|
@celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True)
|
|
def split_one_segment(self, segment_id: str) -> None:
|
|
try:
|
|
return run_async(_run_split_one_segment(segment_id))
|
|
except RedisExecutionLockError as exc:
|
|
raise self.retry(exc=exc, countdown=30)
|
|
|
|
|
|
@celery_app.task(
|
|
name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
|
|
bind=True,
|
|
max_retries=3,
|
|
default_retry_delay=60,
|
|
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
|
|
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
|
|
ignore_result=True,
|
|
)
|
|
def analyze_custom_segment_video(self, segment_id: str) -> None:
|
|
try:
|
|
return run_async(_run_analyze_custom_segment_video(segment_id))
|
|
except RedisExecutionLockError as exc:
|
|
raise self.retry(exc=exc, countdown=60)
|
|
|
|
|
|
@celery_app.task(
|
|
name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
|
|
bind=True,
|
|
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
|
)
|
|
def recover_analysis_tasks_once(self) -> dict[str, Any]:
|
|
return run_async(_run_recover_analysis_tasks_once())
|
|
|
|
|
|
@celery_app.task(
|
|
name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
|
|
bind=True,
|
|
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
|
)
|
|
def recover_split_tasks_once(self) -> dict[str, Any]:
|
|
return run_async(_run_recover_split_tasks_once())
|
|
|
|
else:
|
|
|
|
class _DisabledTask:
|
|
def delay(self, *args: Any, **kwargs: Any) -> None:
|
|
raise RuntimeError("Celery is disabled")
|
|
|
|
def apply_async(self, *args: Any, **kwargs: Any) -> None:
|
|
raise RuntimeError("Celery is disabled")
|
|
|
|
analyze_original_video = _DisabledTask()
|
|
split_one_segment = _DisabledTask()
|
|
analyze_custom_segment_video = _DisabledTask()
|
|
recover_split_tasks_once = _DisabledTask()
|
|
recover_analysis_tasks_once = _DisabledTask()
|