mz/video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
root
2026-07-22 14:53:03 +08:00
parent 3f1c4063b0
commit 6b934b07dc
67 changed files with 6161 additions and 1958 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video-gen-api/app/tasks/shot_replicate_tasks.py
+482 -196
View File
@@ -1,13 +1,16 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from sqlalchemy import select, update
from app.config import settings
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.shot_replicate import (
ModuleCodeEnum,
ShotAnalysisStatusEnum,
@@ -21,25 +24,18 @@ from app.models.base import async_session
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
from app.services.module_async_recovery_service import (
OBJECT_SHOT_SEGMENT_ANALYSIS,
OBJECT_SHOT_SPLIT_SEGMENT,
OBJECT_SHOT_TASK_SET_ANALYSIS,
acquire_object_lock,
cleanup_active_if_terminal,
mark_active_started,
postpone_active_task,
register_shot_segment_analysis_task,
register_shot_split_task,
register_shot_task_set_analysis_task,
release_object_lock,
remove_active_task,
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
redis_acquire_lock,
redis_release_lock,
)
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
from app.services.generation.billing_service import charge_shot_video_analysis_usage
from app.services.shot_video_split_service import split_video_segment_async
from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async
from app.services.upload_video_asset_service import validate_split_range
from app.services.upload_resource import record_shot_segment_upload_resource
from app.tasks.async_runner import run_async
@@ -48,8 +44,8 @@ from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
SPLIT_QUEUE = "gen_result_download"
ANALYSIS_QUEUE = "gen_chatapi_create"
SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value
ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value
def _now() -> datetime:
@@ -82,12 +78,80 @@ async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> Non
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.id == task_set_id,
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_attempt_no == attempt_no,
ShotReplicateTaskSet.analysis_claim_token == token,
ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.analysis_attempt_no == attempt_no,
ShotReplicateSegment.analysis_claim_token == token,
ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str:
return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}"
async def _run_analyze_original_video(task_set_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if not lock_token:
token = uuid.uuid4().hex
attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_task_set",
owner_id=task_set_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token),
)
if lease is None:
return
await register_shot_task_set_analysis_task(task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
task_set_user_id: str | None = None
video_url: str | None = None
@@ -100,16 +164,28 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await db.rollback()
return
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
current_lease = task_set.analysis_lease_until
if (
task_set.analysis_claim_token
and task_set.analysis_claim_token != token
and current_lease
and current_lease > _now()
):
await db.rollback()
return
task_set_user_id = task_set.user_id
video_url = task_set.video_url
if int(task_set.analysis_attempt_no or 1) != attempt_no:
await db.rollback()
return
task_set_user_id = str(task_set.user_id)
video_url = str(task_set.video_url)
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
task_set.analysis_claim_token = token
task_set.analysis_started_at = _now()
task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
task_set.analysis_error_message = None
await db.commit()
@@ -119,21 +195,40 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析开始",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
detail={
"task_set_id": task_set_id,
"video_url": video_url,
"analysis_mode": "full_breakdown",
"analysis_attempt_no": attempt_no,
"queue": ANALYSIS_QUEUE,
},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}")
result = await db.execute(
async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(
call_db,
video_url or "",
user_id=task_set_user_id,
mode="full_breakdown",
task_set_id=task_set_id,
trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await db.rollback()
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if (
not task_set
or int(task_set.analysis_attempt_no or 1) != attempt_no
or task_set.analysis_claim_token != token
or str(task_set.video_url) != str(video_url)
or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return
result_json = analyzed.result
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -144,9 +239,11 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
task_set.analysis_result_json = result_json
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = None
await charge_shot_video_analysis_usage(
db,
call_db,
user_id=task_set.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
owner_id=task_set.id,
@@ -154,9 +251,9 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
description="拆镜复刻-原视频分析",
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
source_project_id=task_set.id,
attempt_no=attempt_no,
)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
await call_db.commit()
log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
@@ -175,8 +272,10 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析成功",
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "token_usage": analyzed.usage},
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no},
)
except RedisExecutionLockError:
raise
except Exception as exc:
async with async_session() as db:
result = await db.execute(
@@ -186,34 +285,69 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1)
)
task_set = result.scalar_one_or_none()
if task_set:
task_set_user_id = task_set_user_id or task_set.user_id
if (
task_set
and int(task_set.analysis_attempt_no or 1) == attempt_no
and task_set.analysis_claim_token == token
and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value
):
task_set_user_id = task_set_user_id or str(task_set.user_id)
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = str(exc)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
else:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
await db.rollback()
log_module_error(
module=MODULE,
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析失败",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc,
)
finally:
await release_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id, token=lock_token)
await lease.close()
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if not lock_token:
token = uuid.uuid4().hex
attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token),
)
if lease is None:
return
await register_shot_segment_analysis_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
user_id: str | None = None
task_set_id: str | None = None
@@ -227,17 +361,23 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or not segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await db.rollback()
return
if segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
current_lease = segment.analysis_lease_until
if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now():
await db.rollback()
return
user_id = segment.user_id
task_set_id = segment.task_set_id
video_url = segment.segment_video_url
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id)
if int(segment.analysis_attempt_no or 1) != attempt_no:
await db.rollback()
return
user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
video_url = str(segment.segment_video_url)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
segment.analysis_claim_token = token
segment.analysis_started_at = _now()
segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
segment.analysis_error_message = None
await db.commit()
@@ -248,21 +388,35 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析开始",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}")
result = await db.execute(
async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(
call_db,
video_url or "",
user_id=user_id,
mode="summary_only",
task_set_id=task_set_id,
segment_id=segment_id,
trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await db.rollback()
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if (
not segment
or int(segment.analysis_attempt_no or 1) != attempt_no
or segment.analysis_claim_token != token
or str(segment.segment_video_url) != str(video_url)
or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return
result_json = analyzed.result
segment.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -273,9 +427,11 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
segment.segment_audience = segment.original_video_audience
segment.analysis_json = result_json
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = None
await charge_shot_video_analysis_usage(
db,
call_db,
user_id=segment.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
owner_id=segment.id,
@@ -284,9 +440,9 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
source_project_id=segment.task_set_id,
source_step_id=segment.id,
attempt_no=attempt_no,
)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
await call_db.commit()
log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
@@ -306,8 +462,10 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析成功",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "token_usage": analyzed.usage},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no},
)
except RedisExecutionLockError:
raise
except Exception as exc:
async with async_session() as db:
result = await db.execute(
@@ -317,15 +475,21 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if segment:
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
if (
segment
and int(segment.analysis_attempt_no or 1) == attempt_no
and segment.analysis_claim_token == token
and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = str(exc)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
else:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
await db.rollback()
log_module_error(
module=MODULE,
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
@@ -333,49 +497,90 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析失败",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc,
)
finally:
await release_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id, token=lock_token)
await lease.close()
async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.split_retry_count == attempt_no,
ShotReplicateSegment.split_claim_token == token,
ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value,
)
.values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _run_split_one_segment(segment_id: str) -> None:
segment_lock_key = f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}"
segment_lock_token = await redis_acquire_lock(
lock_key=segment_lock_key,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
log_context="shot_split_segment_lock",
)
if not segment_lock_token:
return
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial:
return
if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url:
return
attempt = int(initial.split_retry_count or 0) + 1
await db.rollback()
await register_shot_split_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_SPLIT.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt,
task_name=CeleryTaskName.SHOT_SPLIT_ONE.value,
queue=SPLIT_QUEUE,
),
lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}",
hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ShotSplitStatusEnum.PROCESSING.value,
db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token),
)
if lease is None:
return
semaphore_key: str | None = None
user_id: str | None = None
task_set_id: str | None = None
source_path: str | None = None
split_result = None
try:
semaphore_key = await _acquire_split_semaphore(segment_id)
semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}")
if not semaphore_key:
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
step_id=segment_id,
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
detail={"segment_id": segment_id, "reason": "semaphore_full"},
)
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=delay,
reason="semaphore_full",
detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt},
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL)
split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
)
return
async with async_session() as db:
@@ -387,10 +592,10 @@ async def _run_split_one_segment(segment_id: str) -> None:
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
user_id = segment.user_id
task_set_id = segment.task_set_id
user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
@@ -399,10 +604,18 @@ async def _run_split_one_segment(segment_id: str) -> None:
)
task_set = task_set_result.scalar_one_or_none()
if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if (
segment.split_claim_token
and segment.split_claim_token != token
and segment.split_lease_until
and segment.split_lease_until > _now()
):
await db.rollback()
return
validate_split_range(
@@ -410,25 +623,21 @@ async def _run_split_one_segment(segment_id: str) -> None:
end_second=segment.end_second,
video_duration_seconds=task_set.video_duration_seconds,
)
now = _now()
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
segment.split_claim_token = token
segment.split_started_at = now
segment.split_lease_until = _lease_until(now)
segment.split_retry_count = int(segment.split_retry_count or 0) + 1
segment.split_retry_count = attempt
segment.split_next_retry_at = None
segment.split_last_error = None
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.commit()
await register_shot_split_task(segment_id, task_set_id=segment.task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
source_path = task_set.video_path
source_path = str(task_set.video_path)
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
start_second = segment.start_second
end_second = segment.end_second
attempt = segment.split_retry_count
start_second = float(segment.start_second)
end_second = float(segment.end_second)
await db.commit()
log_module_event_file(
module=MODULE,
@@ -444,16 +653,20 @@ async def _run_split_one_segment(segment_id: str) -> None:
"start_second": start_second,
"end_second": end_second,
"attempt": attempt,
"queue": SPLIT_QUEUE,
},
)
split_result = await split_video_segment_async(
source_path=source_path,
source_path=source_path or "",
segment_id=segment_id,
start_second=start_second,
end_second=end_second,
date_dir=date_dir,
attempt_key=f"attempt-{attempt}-{token[-8:]}",
finalize=False,
)
await lease.ensure_owned()
async with async_session() as db:
result = await db.execute(
@@ -463,9 +676,16 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
if (
not segment
or int(segment.split_retry_count or 0) != attempt
or segment.split_claim_token != token
or segment.split_status != ShotSplitStatusEnum.PROCESSING.value
):
await db.rollback()
cleanup_split_result(split_result)
return
split_result = finalize_split_result(split_result)
segment.segment_video_url = split_result.url
segment.segment_video_path = split_result.path
await record_shot_segment_upload_resource(
@@ -476,35 +696,46 @@ async def _run_split_one_segment(segment_id: str) -> None:
file_size_bytes=split_result.file_size_bytes,
)
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
segment.split_claim_token = None
segment.split_completed_at = _now()
segment.split_lease_until = None
segment.split_next_retry_at = None
segment.split_last_error = None
source_mode = str(segment.source_mode)
final_task_set_id = str(segment.task_set_id)
final_user_id = str(segment.user_id)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=segment.task_set_id,
step_id=segment.id,
user_id=segment.user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment.id,
"task_set_id": segment.task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": segment.source_mode,
},
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=final_task_set_id,
step_id=segment_id,
user_id=final_user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment_id,
"task_set_id": final_task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": source_mode,
"attempt": attempt,
},
)
if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
analyze_custom_segment_video.apply_async(
args=[segment_id],
queue=ANALYSIS_QUEUE,
countdown=0,
task_id=f"shot-analysis:segment:{segment_id}:attempt:1",
)
if segment.source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
analyze_custom_segment_video.apply_async(args=[segment.id], queue=ANALYSIS_QUEUE, countdown=0)
except RedisExecutionLockError:
cleanup_split_result(split_result)
raise
except Exception as exc:
cleanup_split_result(split_result)
next_retry_delay: int | None = None
final_failed = False
async with async_session() as db:
@@ -515,37 +746,41 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
return
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
attempt = int(segment.split_retry_count or 0)
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
if (
segment
and int(segment.split_retry_count or 0) == attempt
and segment.split_claim_token == token
and segment.split_status == ShotSplitStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.split_claim_token = None
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
next_retry_delay = max(
1,
int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()),
)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
if segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
next_retry_delay = max(1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=next_retry_delay,
reason="split_retry_waiting",
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
elif final_failed:
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if next_retry_delay and celery_app:
split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=next_retry_delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
log_module_error(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
@@ -559,61 +794,111 @@ async def _run_split_one_segment(segment_id: str) -> None:
"source_path": source_path,
"next_retry_delay_seconds": next_retry_delay,
"final_failed": final_failed,
"attempt": attempt,
},
exc=exc,
)
finally:
await _release_split_semaphore(semaphore_key, segment_id)
await redis_release_lock(lock_key=segment_lock_key, token=segment_lock_token, log_context="shot_split_segment_lock")
await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}")
await lease.close()
async def _run_recover_split_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
redis = await get_registry_redis()
token: str | None = None
if redis is not None:
token = await redis_acquire_lock(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="shot_split_recovery",
)
if not token:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
try:
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_split_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_split_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
return result
finally:
if token:
await redis_release_lock(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
token=token,
log_context="shot_split_recovery",
)
result["execution_lock"] = "lock_acquired"
return result
async def _run_recover_analysis_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_analysis_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_analysis_tasks_once(db)
result["execution_lock"] = "lock_acquired"
return result
if celery_app:
@celery_app.task(name="shot_replicate.analyze_original_video")
def analyze_original_video(task_set_id: str) -> None:
return run_async(_run_analyze_original_video(task_set_id))
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_original_video(self, task_set_id: str) -> None:
try:
return run_async(_run_analyze_original_video(task_set_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(name="shot_replicate.split_one_segment", bind=True, max_retries=0)
@celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True)
def split_one_segment(self, segment_id: str) -> None:
return run_async(_run_split_one_segment(segment_id))
@celery_app.task(name="shot_replicate.analyze_custom_segment_video")
def analyze_custom_segment_video(segment_id: str) -> None:
return run_async(_run_analyze_custom_segment_video(segment_id))
try:
return run_async(_run_split_one_segment(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=30)
@celery_app.task(
name="shot_replicate.recover_split_tasks_once",
name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_custom_segment_video(self, segment_id: str) -> None:
try:
return run_async(_run_analyze_custom_segment_video(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
)
def recover_analysis_tasks_once(self) -> dict[str, Any]:
return run_async(_run_recover_analysis_tasks_once())
@celery_app.task(
name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
@@ -634,3 +919,4 @@ else:
split_one_segment = _DisabledTask()
analyze_custom_segment_video = _DisabledTask()
recover_split_tasks_once = _DisabledTask()
recover_analysis_tasks_once = _DisabledTask()