mz/video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

celery 容灾升级

Browse Source
This commit is contained in:
GinHa
2026-07-22 14:48:29 +08:00
parent 3f1c4063b0
commit 69e7dec807
67 changed files with 6161 additions and 1958 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video-gen-api/app/tasks/celery_app.py
+245 -21
View File
@@ -1,7 +1,15 @@
import logging
from celery import Celery
from celery.signals import worker_process_init, worker_process_shutdown, worker_ready
from celery.signals import (
celeryd_init,
heartbeat_sent,
worker_init,
worker_process_init,
worker_process_shutdown,
worker_ready,
worker_shutdown,
)
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
@@ -24,9 +32,8 @@ CELERY_TASK_IMPORTS = (
"app.tasks.shot_replicate_flow_tasks",
"app.tasks.module_async_recovery_tasks",
"app.tasks.module_generation_v2_tasks",
"app.tasks.user_oauth_tasks",
"app.tasks.cleanup",
"app.tasks.private_portrait_asset_tasks",
"app.tasks.celery_runtime_tasks",
)
@@ -78,6 +85,31 @@ def _beat_schedule() -> dict:
"priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
},
}
schedule["generation-download-recovery"] = {
"task": CeleryTaskName.RECOVER_DOWNLOAD.value,
"schedule": max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["shot-split-recovery"] = {
"task": CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
"schedule": 60,
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["shot-analysis-recovery"] = {
"task": CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
"schedule": 60,
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["celery-runtime-reconcile"] = {
"task": CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
"schedule": max(60, int(settings.CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS or 300)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["celery-runtime-registry-gc"] = {
"task": CeleryTaskName.CELERY_RUNTIME_GC.value,
"schedule": max(60, int(settings.CELERY_RUNTIME_GC_INTERVAL_SECONDS or 600)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["private-portrait-sync-due-assets-every-minute"] = {
"task": CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value,
"schedule": 60,
@@ -121,8 +153,16 @@ if broker_url:
# "generation.download_generation_result_task": {"ignore_result": True},
"hot_opening.start_image_prompt_optimize": {"ignore_result": True},
"hot_opening.start_video_prompt_optimize": {"ignore_result": True},
"shot_replicate.analyze_original_video": {"ignore_result": True},
"shot_replicate.analyze_custom_segment_video": {"ignore_result": True},
"shot_replicate.analyze_original_video": {
"ignore_result": True,
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
},
"shot_replicate.analyze_custom_segment_video": {
"ignore_result": True,
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
},
"shot_replicate.split_one_segment": {"ignore_result": True},
"shot_replicate.start_image_prompt_optimize": {"ignore_result": True},
"shot_replicate.start_video_prompt_optimize": {"ignore_result": True},
@@ -139,13 +179,17 @@ if broker_url:
CeleryTaskName.VIDEO_UPSCALE_RECOVER.value: {"ignore_result": True},
CeleryTaskName.RECOVER_CREATE.value: {"ignore_result": True},
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"ignore_result": True},
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"ignore_result": True},
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"ignore_result": True},
},
worker_prefetch_multiplier=1,
worker_cancel_long_running_tasks_on_connection_loss=True,
broker_transport_options={
"visibility_timeout": max(
3600,
int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 600,
int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 600,
int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900) + 600,
),
"queue_order_strategy": "priority",
"priority_steps": list(range(10)),
@@ -174,24 +218,26 @@ if broker_url:
CeleryTaskName.DISPATCH_DUE_POLL.value: {"queue": RECOVERY_QUEUE},
"hot_opening.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"hot_opening.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.analyze_original_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.analyze_custom_segment_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.split_one_segment": {"queue": CeleryQueue.GEN_RESULT_DOWNLOAD.value},
CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
CeleryTaskName.SHOT_SPLIT_ONE.value: {"queue": CeleryQueue.GEN_SHOT_SPLIT.value},
"shot_replicate.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"module_generation_v2.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
# 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。
CeleryTaskName.STARTUP_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_DOWNLOAD.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_GENERATION.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_CREATE.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"queue": RECOVERY_QUEUE},
"user_oauth.update_oauth_accounts": {"queue": CeleryQueue.DEFAULT.value},
"app.tasks.cleanup.*": {"queue": CeleryQueue.DEFAULT.value},
CeleryTaskName.CELERY_RUNTIME_RECONCILE.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.CELERY_RUNTIME_GC.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
},
@@ -212,18 +258,138 @@ async def _try_acquire_startup_recovery_lock() -> bool:
return bool(token)
def _worker_name_from_sender(sender=None, **kwargs) -> str:
candidates = (
getattr(sender, "hostname", None),
getattr(sender, "name", None),
kwargs.get("hostname"),
kwargs.get("nodename"),
)
instance = kwargs.get("instance")
if instance is not None:
candidates += (
getattr(instance, "hostname", None),
getattr(instance, "name", None),
)
for value in candidates:
normalized = str(value or "").strip()
if normalized:
return normalized
return ""
def _worker_runtime_metadata(sender=None) -> dict:
queues: list[str] = []
try:
consumer = getattr(sender, "consumer", None)
task_consumer = getattr(consumer, "task_consumer", None)
for queue in list(getattr(task_consumer, "queues", None) or []):
name = str(getattr(queue, "name", queue) or "").strip()
if name and name not in queues:
queues.append(name)
except Exception:
pass
pool = getattr(sender, "pool", None)
pool_type = type(pool).__name__ if pool is not None else None
configured_concurrency = None
for value in (
getattr(pool, "limit", None),
getattr(sender, "concurrency", None),
):
try:
parsed = int(value)
except (TypeError, ValueError):
continue
if parsed > 0:
configured_concurrency = parsed
break
return {
"queues": queues,
"pool_type": pool_type,
"configured_concurrency": configured_concurrency,
}
@celeryd_init.connect
def on_celeryd_init(sender=None, instance=None, **kwargs):
"""尽早生成 Worker 主实例 token，确保 prefork 子进程继承。"""
try:
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
initialize_worker_main_identity(
_worker_name_from_sender(sender, instance=instance, **kwargs) or None,
before_pool=True,
)
except Exception:
logger.exception("Celery Worker 主实例身份初始化失败。signal=celeryd_init")
@worker_init.connect
def on_worker_init(sender=None, **kwargs):
"""worker_init 幂等兜底，仍处于进程池创建之前。"""
try:
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
initialize_worker_main_identity(
_worker_name_from_sender(sender, **kwargs) or None,
before_pool=True,
)
except Exception:
logger.exception("Celery Worker 主实例身份初始化失败。signal=worker_init")
@worker_ready.connect
def on_worker_ready(sender=None, **kwargs):
"""Celery worker 启动时做一次容灾恢复。
注意：
- 启动容灾只投递一个 recovery.startup_recovery_once 协调任务。
- Celery Beat 只用于每分钟触发轻量 generation.dispatch_due_poll_tasks，不跑完整启动容灾。
- 协调任务走独立 gen_recovery 队列，串行扫描并把真实业务任务投回原队列。
- 所有 worker 都尝试抢 Redis 投递锁，只有抢到锁的 worker 投递恢复任务。
"""
"""注册当前 Worker 主实例，并协调实例级与全局启动恢复。"""
if celery_app is None:
return
worker_name = _worker_name_from_sender(sender, **kwargs)
metadata = _worker_runtime_metadata(sender)
current_identity = None
try:
from app.services.celery_runtime.worker_service import (
initialize_worker_main_identity,
register_worker_instance,
)
# 正常情况下 token 已在 worker_init 前创建；这里仅做 late fallback。
initialize_worker_main_identity(worker_name or None, before_pool=False)
current_identity = run_async(
register_worker_instance(
worker_name=worker_name,
queues=metadata["queues"],
pool_type=metadata["pool_type"],
configured_concurrency=metadata["configured_concurrency"],
)
)
except Exception:
# Redis 或身份注册失败不能阻塞 Worker 启动，任务级执行锁仍会 fail-closed。
logger.exception("Celery Worker 主实例注册失败。worker_name=%s", worker_name)
if current_identity is not None:
try:
from app.services.celery_runtime.recovery_service import (
mark_stale_worker_instance_candidates,
)
run_async(
mark_stale_worker_instance_candidates(
worker_name=current_identity.worker_name,
current_worker_instance_id=current_identity.worker_instance_id,
supports_targeted_recovery=current_identity.supports_targeted_recovery,
)
)
except Exception:
logger.exception(
"Worker 旧主实例任务候选标记失败。worker_name=%s worker_instance_id=%s",
current_identity.worker_name,
current_identity.worker_instance_id,
)
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
return
@@ -232,14 +398,15 @@ def on_worker_ready(sender=None, **kwargs):
if not run_async(_try_acquire_startup_recovery_lock()):
return
except Exception:
# Redis 不可用时不阻塞 worker 启动，避免影响稳定生成链路。
logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递")
return
try:
from app.services.celery_runtime.recovery_service import set_startup_barrier
from app.tasks.generation_recovery_tasks import startup_recovery_once
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
run_async(set_startup_barrier())
countdown = max(0, int(settings.CELERY_STARTUP_RECOVERY_DELAY_SECONDS or 30))
startup_recovery_once.apply_async(
countdown=countdown,
queue=RECOVERY_QUEUE,
@@ -254,9 +421,66 @@ def on_worker_ready(sender=None, **kwargs):
logger.exception("启动容灾恢复协调任务投递失败")
@heartbeat_sent.connect
def on_worker_heartbeat_sent(sender=None, **kwargs):
"""刷新主实例 heartbeat，并低频扫描已到期的旧主实例。"""
try:
from app.services.celery_runtime.worker_service import (
claim_worker_heartbeat_slot,
claim_worker_stale_scan_slot,
heartbeat_current_worker_instance,
registered_worker_identity,
)
heartbeat_ok = True
if claim_worker_heartbeat_slot():
heartbeat_ok = bool(run_async(heartbeat_current_worker_instance()))
if not heartbeat_ok or not claim_worker_stale_scan_slot():
return
identity = registered_worker_identity()
if identity is None:
return
from app.services.celery_runtime.recovery_service import (
mark_stale_worker_instance_candidates,
)
run_async(
mark_stale_worker_instance_candidates(
worker_name=identity.worker_name,
current_worker_instance_id=identity.worker_instance_id,
supports_targeted_recovery=identity.supports_targeted_recovery,
emit_duplicate_log=False,
)
)
except Exception:
logger.warning("Celery Worker 主实例 heartbeat/旧实例扫描失败", exc_info=True)
@worker_shutdown.connect
def on_worker_shutdown(sender=None, **kwargs):
"""优雅退出时撤销活跃 Worker key；旧任务集合保留给恢复流程。"""
try:
from app.services.celery_runtime.worker_service import unregister_current_worker_instance
run_async(unregister_current_worker_instance())
except Exception:
logger.debug("Celery Worker 主实例注销失败", exc_info=True)
finally:
close_loop()
@worker_process_init.connect
def on_worker_process_init(**kwargs):
"""Linux prefork 子进程启动后丢弃 fork 前可能继承的连接池状态。"""
"""prefork 子进程保留主 token，同时重建执行进程缓存和异步连接。"""
try:
from app.services.celery_runtime.worker_service import reset_process_identity_cache
reset_process_identity_cache()
except Exception:
pass
try:
run_async(engine.dispose())
except Exception:
video-gen-api/app/tasks/celery_runtime_tasks.py
+157
View File
@@ -0,0 +1,157 @@
from __future__ import annotations
import logging
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeEvent
from app.services.celery_runtime.recovery_service import (
garbage_collect_registry_pair,
garbage_collect_worker_registry,
guard_periodic_recovery,
)
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
async def _run_reconcile_once() -> dict[str, Any]:
guarded = await guard_periodic_recovery(check_global_lock=False)
if guarded:
return guarded
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="celery_runtime_reconcile",
renew_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
)
if lease is None:
return {"skipped": "global_recovery_lock_held"}
try:
from app.tasks.generation_recovery_tasks import (
_run_create_once,
_run_download_once,
_run_module_async_once,
_run_shot_analysis_once,
_run_shot_split_once,
)
results: dict[str, Any] = {}
for name, runner in (
("create", _run_create_once),
("download", _run_download_once),
("module_async", _run_module_async_once),
("shot_analysis", _run_shot_analysis_once),
("shot_split", _run_shot_split_once),
):
try:
results[name] = await runner()
except Exception as exc:
logger.exception("Celery runtime reconcile step failed. step=%s", name)
results[name] = {"error": str(exc)}
await lease.ensure_owned()
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.REGISTRY_RECONCILE_DONE.value,
event_status="success",
source="recovery",
detail={"steps": results},
)
return {"steps": results}
finally:
await lease.close()
async def _run_registry_gc_once() -> dict[str, Any]:
guarded = await guard_periodic_recovery()
if guarded:
return guarded
pairs = {
"generation_create": (
settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
),
"poll": (settings.POLL_ACTIVE_REDIS_HASH_KEY, settings.POLL_ACTIVE_REDIS_ZSET_KEY),
"download": (settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY),
"module_async": (
settings.MODULE_ASYNC_ACTIVE_REDIS_HASH_KEY,
settings.MODULE_ASYNC_ACTIVE_REDIS_ZSET_KEY,
),
"shot_analysis": (
settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
),
"shot_split": (
settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
),
"video_upscale": (
settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
),
"private_portrait_poll": (
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
),
"private_portrait_delete": (
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
),
}
results = {}
for name, (hash_key, zset_key) in pairs.items():
results[name] = await garbage_collect_registry_pair(
hash_key=hash_key,
zset_key=zset_key,
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
)
results["worker_registry_v2"] = await garbage_collect_worker_registry(
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.REGISTRY_GC_DONE.value,
event_status="success",
source="recovery",
detail={"results": results},
)
return {"results": results}
if celery_app:
@celery_app.task(
name=CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def reconcile_once(self) -> dict[str, Any]:
return run_async(_run_reconcile_once())
@celery_app.task(
name=CeleryTaskName.CELERY_RUNTIME_GC.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def registry_gc_once(self) -> dict[str, Any]:
return run_async(_run_registry_gc_once())
else:
class _DisabledTask:
def delay(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
def apply_async(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
reconcile_once = _DisabledTask()
registry_gc_once = _DisabledTask()
video-gen-api/app/tasks/generation_create_tasks.py
+39 -11
View File
@@ -2,11 +2,13 @@ from __future__ import annotations
import asyncio
import json
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import (
ALLOWED_GENERATION_MODES,
@@ -34,6 +36,7 @@ from app.services.generation.pipeline.owner_service import (
owner_mode,
owner_provider_task_id,
set_owner_provider_task_id,
renew_generation_owner_claim_lease,
)
from app.services.generation.poll_schedule_service import ensure_video_poll_fields
from app.services.generation.provider_service import create_provider_task
@@ -41,10 +44,8 @@ from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot,
sync_generation_record_media_token_snapshot,
)
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
)
from app.services.redis_registry_service import RedisExecutionLockError
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
@@ -257,13 +258,36 @@ async def _run(
if effective_attempt is None:
return
lease = await RedisExecutionLockLease.acquire(
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
log_context="generation_create",
renew_interval_seconds=int(
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
lease_token = uuid.uuid4().hex
async def _renew_db_claim(token: str) -> bool:
return await renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="provider_create_claim_token",
lease_field="provider_create_lease_until",
token=token,
lease_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
)
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_CREATE.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.CHATAPI_CREATE.value,
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
token=lease_token,
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
db_heartbeat=_renew_db_claim,
)
if lease is None:
return
@@ -320,6 +344,10 @@ async def _run(
_stage(owner, ChatGenerationPipelineStage.PREPARING),
_stage(owner, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK),
}
if is_image_main:
allowed_stages.add(
_stage(owner, ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED)
)
if owner.pipeline_stage not in allowed_stages:
return
video-gen-api/app/tasks/generation_download_tasks.py
+34 -6
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import logging
import asyncio
import errno
import math
@@ -9,6 +10,8 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import (
ALLOWED_GENERATION_MODES,
@@ -47,6 +50,7 @@ from app.services.generation.pipeline.owner_service import (
owner_type_of,
redis_owner_item_id,
set_owner_completed,
renew_generation_owner_claim_lease,
)
from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot,
@@ -58,14 +62,16 @@ from app.services.resource_accounting_service import (
)
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
ensure_aware_utc,
)
from app.services.video_upscale.media_service import probe_video
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
DOWNLOAD_QUEUE = "gen_result_download"
logger = logging.getLogger("video_gen")
DOWNLOAD_QUEUE = CeleryQueue.GEN_RESULT_DOWNLOAD.value
DOWNLOAD_STAGE_QUEUED = ChatGenerationPipelineStage.DOWNLOAD_QUEUED.value
DOWNLOAD_STAGE_DOWNLOADING = ChatGenerationPipelineStage.DOWNLOADING.value
@@ -573,12 +579,34 @@ async def _run(
if effective_attempt is None:
return
lease = await RedisExecutionLockLease.acquire(
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_DOWNLOAD.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.DOWNLOAD_GENERATION_RESULT.value,
queue=DOWNLOAD_QUEUE,
registry_item_id=redis_owner_item_id(
normalized_owner_type, task_id, effective_attempt
),
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.GENERATION_DOWNLOAD_LOCK_TTL_SECONDS or 600),
log_context="generation_download",
renew_interval_seconds=int(
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.DOWNLOADING.value,
db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="download_claim_token",
lease_field="download_lease_until",
token=owned_token,
lease_seconds=int(settings.DOWNLOAD_TASK_LEASE_SECONDS or 600),
),
)
if lease is None:
video-gen-api/app/tasks/generation_poll_tasks.py
+53 -13
View File
@@ -1,11 +1,14 @@
from __future__ import annotations
import logging
import json
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import (
ALLOWED_GENERATION_MODES,
@@ -36,6 +39,7 @@ from app.services.generation.pipeline.owner_service import (
owner_provider_task_id,
owner_type_of,
redis_owner_item_id,
renew_generation_owner_claim_lease,
)
from app.services.generation.poll_schedule_service import (
build_default_poll_schedule,
@@ -52,16 +56,19 @@ from app.services.media_token_usage_snapshot_service import (
)
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
datetime_to_epoch,
ensure_aware_utc,
redis_get_registry_payloads,
redis_remove_registry_item,
redis_upsert_registry_item,
utc_now,
)
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
@@ -183,16 +190,27 @@ async def register_poll_active(
reason: str,
next_poll_at: datetime | None = None,
) -> None:
item_id = _registry_id(owner)
payload = _build_poll_active_payload(
owner,
reason=reason,
next_poll_at=next_poll_at,
check_at=check_at,
)
existing = await redis_get_registry_payloads(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
item_ids=[item_id],
log_context="poll_active",
)
if item_id in existing:
merged = dict(existing[item_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=_registry_id(owner),
payload=_build_poll_active_payload(
owner,
reason=reason,
next_poll_at=next_poll_at,
check_at=check_at,
),
item_id=item_id,
payload=payload,
check_at=check_at,
log_context="poll_active",
)
@@ -388,12 +406,34 @@ async def _run(
if effective_attempt is None:
return
lease = await RedisExecutionLockLease.acquire(
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_POLL.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.POLL_GENERATION.value,
queue=POLL_QUEUE,
registry_item_id=redis_owner_item_id(
normalized_owner_type, task_id, effective_attempt
),
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.GENERATION_POLL_LOCK_TTL_SECONDS or 300),
log_context="generation_poll",
renew_interval_seconds=int(
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.POLLING.value,
db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="poll_claim_token",
lease_field="poll_lease_until",
token=owned_token,
lease_seconds=int(settings.POLL_TASK_LEASE_SECONDS or 300),
),
)
if lease is None:
video-gen-api/app/tasks/generation_recovery_tasks.py
+55 -52
View File
@@ -7,10 +7,10 @@ from typing import Any, Awaitable, Callable, Dict
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.models.base import async_session
from app.services.celery_runtime.recovery_service import clear_startup_barrier, guard_periodic_recovery
from app.services.redis_registry_service import (
RedisExecutionLockLease,
get_registry_redis,
redis_acquire_execution_lock,
)
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
@@ -203,6 +203,13 @@ async def _run_shot_split_once() -> Dict[str, Any]:
return await recover_shot_split_tasks_once(db)
async def _run_shot_analysis_once() -> Dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
async with async_session() as db:
return await recover_shot_analysis_tasks_once(db)
async def _run_video_upscale_once() -> Dict[str, Any]:
from app.services.video_upscale.task_service import recover_video_upscale_tasks_once
@@ -265,45 +272,43 @@ async def _run_due_poll_dispatch_with_guard() -> Dict[str, Any]:
return await _run_due_poll_dispatch_once()
async def _acquire_download_recovery_loop_lock() -> tuple[bool, str]:
"""下载恢复循环调度锁；Redis 不可用直接抛错，不做 DB 降级。"""
token = await redis_acquire_execution_lock(
lock_key=settings.DOWNLOAD_RECOVERY_LOOP_LOCK_KEY,
ttl_seconds=int(settings.DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS or 55),
log_context="download_recovery_loop",
async def _run_periodic_with_guard(
*,
lock_key: str,
log_context: str,
runner: RecoveryRunner,
ttl_seconds: int | None = None,
) -> Dict[str, Any]:
guarded = await guard_periodic_recovery()
if guarded:
return guarded
return await _run_with_execution_lock(
lock_key=lock_key,
log_context=log_context,
runner=runner,
ttl_seconds=ttl_seconds,
)
return (bool(token), "lock_acquired" if token else "lock_held")
def _schedule_next_download_recovery_loop() -> None:
if not celery_app or not bool(getattr(settings, "DOWNLOAD_RECOVERY_LOOP_ENABLED", False)):
return
try:
recover_download_tasks_once.apply_async(
countdown=max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
queue=RECOVERY_QUEUE,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
except Exception:
logger.exception("下载恢复循环下一轮投递失败")
async def _run_startup_recovery_once() -> Dict[str, Any]:
"""启动容灾协调器：串行跑恢复扫描。
"""启动容灾协调器；全局锁隔离周期对账，完成或失败后释放 barrier。"""
async def _run_startup_locked() -> Dict[str, Any]:
return await _run_with_execution_lock(
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
log_context="startup_recovery_once",
runner=_run_startup_recovery_steps,
)
真实业务任务仍投递回原队列：
- 创建/提词/视频分析 -> gen_chatapi_create
- provider poll -> gen_provider_poll
- 下载/ffmpeg 切片 -> gen_result_download
- 本地视频超分 -> gen_video_upscale_local
- 火山视频超分 -> gen_video_upscale_remote
恢复扫描本身只走 gen_recovery，避免堵住业务 worker。
"""
return await _run_with_execution_lock(
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
log_context="startup_recovery_once",
runner=_run_startup_recovery_steps,
)
try:
return await _run_with_execution_lock(
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
log_context="startup_global_recovery",
runner=_run_startup_locked,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
)
finally:
await clear_startup_barrier()
async def _run_startup_recovery_steps() -> Dict[str, Any]:
@@ -316,6 +321,12 @@ async def _run_startup_recovery_steps() -> Dict[str, Any]:
"module_async_recovery",
_run_module_async_once,
),
(
"shot_analysis",
settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
"shot_analysis_recovery",
_run_shot_analysis_once,
),
(
"shot_split",
settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
@@ -375,21 +386,13 @@ if celery_app:
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
)
def recover_download_tasks_once(self) -> Dict[str, Any]:
acquired, reason = run_async(_acquire_download_recovery_loop_lock())
if not acquired:
return {"skipped": reason}
try:
result = run_async(
_run_with_execution_lock(
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
log_context="download_recovery",
runner=_run_download_once,
)
return run_async(
_run_periodic_with_guard(
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
log_context="download_recovery",
runner=_run_download_once,
)
result["loop_lock"] = reason
return result
finally:
_schedule_next_download_recovery_loop()
)
@celery_app.task(
@@ -400,7 +403,7 @@ if celery_app:
)
def recover_generation_tasks_once(self) -> Dict[str, Any]:
return run_async(
_run_with_execution_lock(
_run_periodic_with_guard(
lock_key=settings.GENERATION_RECOVERY_LOCK_KEY,
log_context="generation_recovery",
runner=_run_generation_once,
@@ -416,7 +419,7 @@ if celery_app:
)
def recover_create_tasks_once(self) -> Dict[str, Any]:
return run_async(
_run_with_execution_lock(
_run_periodic_with_guard(
lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create",
log_context="generation_create_recovery",
runner=_run_create_once,
@@ -433,7 +436,7 @@ if celery_app:
)
def dispatch_due_poll_tasks(self) -> Dict[str, Any]:
return run_async(
_run_with_execution_lock(
_run_periodic_with_guard(
lock_key=settings.POLL_DUE_DISPATCH_LOCK_KEY,
log_context="due_poll_dispatch",
runner=_run_due_poll_dispatch_with_guard,
video-gen-api/app/tasks/hot_opening_replicate_tasks.py
+21 -2
View File
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
TASK_HOT_VIDEO_PROMPT,
acquire_object_lock,
cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started,
release_object_lock,
register_module_step_task,
@@ -38,7 +39,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_image_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
@@ -64,7 +74,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_video_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
video-gen-api/app/tasks/module_async_recovery_tasks.py
+16 -21
View File
@@ -5,35 +5,30 @@ from typing import Any
from app.config import settings
from app.models.base import async_session
from app.services.module_async_recovery_service import recover_module_async_tasks_once
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
async def _run_recover_module_async_tasks_once() -> dict[str, Any]:
redis = await get_registry_redis()
token: str | None = None
if redis is not None:
token = await redis_acquire_lock(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="module_async_recovery",
)
if not token:
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
try:
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="module_async_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_module_async_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
return result
finally:
if token:
await redis_release_lock(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
token=token,
log_context="module_async_recovery",
)
result["execution_lock"] = "lock_acquired"
return result
if celery_app:
video-gen-api/app/tasks/module_generation_v2_tasks.py
+7 -1
View File
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
TASK_MODULE_V2_VIDEO_PROMPT,
acquire_object_lock,
cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started,
register_module_step_task,
release_object_lock,
@@ -46,7 +47,12 @@ async def _run_video_prompt(project_id: str, step_id: str) -> None:
task_name=TASK_MODULE_V2_VIDEO_PROMPT,
)
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
await run_video_prompt_optimize_v2(db, project_id=project_id, step_id=step_id)
await run_video_prompt_optimize_v2(
db,
project_id=project_id,
step_id=step_id,
execution_guard=lambda: ensure_object_lock_owned(token=lock_token),
)
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
finally:
await release_object_lock(
video-gen-api/app/tasks/private_portrait_asset_tasks.py
+297 -73
View File
@@ -1,36 +1,58 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.private_portrait import (
PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE,
PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE,
PrivatePortraitAssetStatus,
PrivatePortraitEventSource,
PrivatePortraitEventType,
PrivatePortraitRemoteDeleteStatus,
)
from app.models import async_session
from app.services.operation_log_service import log_operation_error
from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAssetGroup
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.asset_service import (
DOMAIN,
delete_asset_group_remote,
delete_asset_remote,
delete_project_remote,
poll_due_assets_once,
recover_remote_deletes_once,
sync_asset_status,
)
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger(__name__)
QUEUE = CeleryQueue.GEN_PRIVATE_PORTRAIT.value
def _now() -> datetime:
return datetime.now(timezone.utc)
def _retry_countdown(retries: int) -> int:
return min(300, 30 * (2 ** max(0, retries)))
async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, detail: dict[str, Any] | None = None, **kwargs: Any):
async def _rollback_and_reraise(
db,
*,
event_type: str,
exc: BaseException,
detail: dict[str, Any] | None = None,
**kwargs: Any,
):
await db.rollback()
log_operation_error(
domain=DOMAIN,
@@ -43,12 +65,58 @@ async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, deta
raise exc
@celery_app.task(name="private_portrait.poll_asset_status", queue="gen_private_portrait", bind=True, max_retries=5, default_retry_delay=30)
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
async def _inner():
async def _acquire_runtime(
*,
domain: str,
owner_type: str,
owner_id: str,
task_name: str,
hash_key: str,
zset_key: str,
lock_prefix: str,
) -> CeleryRuntimeLease | None:
token = uuid.uuid4().hex
return await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=domain,
owner_type=owner_type,
owner_id=owner_id,
attempt_no=1,
task_name=task_name,
queue=QUEUE,
),
lock_key=f"{lock_prefix}:{owner_type}:{owner_id}:attempt:1",
hash_key=hash_key,
zset_key=zset_key,
token=token,
ttl_seconds=max(60, int(settings.PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS or 180)),
heartbeat_interval_seconds=max(10, int(settings.PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS or 30)),
pipeline_stage="processing",
)
async def _run_poll_asset(asset_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_POLL.value,
owner_type="asset",
owner_id=asset_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value,
hash_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
async with async_session() as db:
try:
await sync_asset_status(db, user_id=None, asset_id=asset_id)
await sync_asset_status(
db,
user_id=None,
asset_id=asset_id,
execution_guard=lease.ensure_owned,
)
await lease.ensure_owned()
await db.commit()
except Exception as exc:
logger.exception("poll private portrait asset failed: %s", asset_id)
@@ -57,97 +125,253 @@ def poll_private_portrait_asset_status(self, asset_id: str) -> None:
event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value,
exc=exc,
asset_id=asset_id,
detail={"celery_task": "private_portrait.poll_asset_status"},
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value},
)
finally:
await lease.close()
async def _dispatch_due_assets() -> int:
barrier = await guard_periodic_recovery()
if barrier is not None:
return 0
lock = await RedisExecutionLockLease.acquire(
lock_key=settings.PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY,
ttl_seconds=55,
renew_interval_seconds=20,
log_context="private_portrait_poll_dispatch",
)
if lock is None:
return 0
async with lock:
async with async_session() as db:
now = _now()
rows = await db.execute(
select(PrivatePortraitAsset)
.where(
PrivatePortraitAsset.deleted_at.is_(None),
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
PrivatePortraitAsset.next_poll_at.is_not(None),
PrivatePortraitAsset.next_poll_at <= now,
)
.order_by(PrivatePortraitAsset.next_poll_at.asc(), PrivatePortraitAsset.id.asc())
.limit(PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
.with_for_update(skip_locked=True)
)
assets = list(rows.scalars().all())
dispatches: list[tuple[str, int]] = []
queue_hold_until = now + timedelta(seconds=120)
for asset in assets:
dispatches.append((str(asset.id), int(asset.poll_count or 0) + 1))
asset.next_poll_at = queue_hold_until
await db.commit()
for asset_id, poll_no in dispatches:
poll_private_portrait_asset_status.apply_async(
args=[asset_id],
queue=QUEUE,
countdown=0,
task_id=f"private-portrait-poll:{asset_id}:attempt:{poll_no}",
)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value,
event_status="success",
source=PrivatePortraitEventSource.CELERY.value,
detail={"matched_count": len(dispatches), "dispatched_count": len(dispatches)},
)
return len(dispatches)
async def _run_delete_asset(asset_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="asset",
owner_id=asset_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.sync_due_assets", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def sync_private_portrait_due_assets(self) -> int:
async def _inner() -> int:
async with async_session() as db:
try:
count = await poll_due_assets_once(db, limit=PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
await db.commit()
return count
except Exception as exc:
logger.exception("sync private portrait due assets failed")
await _rollback_and_reraise(
await delete_asset_remote(
db,
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_FAILED.value,
exc=exc,
detail={"celery_task": "private_portrait.sync_due_assets"},
asset_id=asset_id,
execution_guard=lease.ensure_owned,
)
try:
return run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.delete_asset_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
async def _inner():
async with async_session() as db:
try:
await delete_asset_remote(db, asset_id=asset_id)
await lease.ensure_owned()
await db.commit()
except Exception as exc:
logger.exception("delete private portrait asset remote failed: %s", asset_id)
await _rollback_and_reraise(
db,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
exc=exc,
asset_id=asset_id,
detail={"celery_task": "private_portrait.delete_asset_remote"},
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value},
)
finally:
await lease.close()
async def _run_delete_group(group_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="group",
owner_id=group_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.delete_project_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_project_remote(self, project_id: str) -> None:
async def _inner():
async with async_session() as db:
try:
await delete_project_remote(db, project_id=project_id)
await delete_asset_group_remote(
db,
group_id=group_id,
execution_guard=lease.ensure_owned,
)
await lease.ensure_owned()
await db.commit()
except Exception as exc:
logger.exception("delete private portrait project remote failed: %s", project_id)
await _rollback_and_reraise(
db,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
exc=exc,
project_id=project_id,
detail={"celery_task": "private_portrait.delete_project_remote"},
group_id=group_id,
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value},
)
finally:
await lease.close()
async def _dispatch_project_deletes(project_id: str) -> int:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="project",
owner_id=project_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return 0
try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.recover_remote_deletes", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
async def _inner() -> dict[str, int]:
async with async_session() as db:
try:
result = await recover_remote_deletes_once(db, limit=PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
await db.commit()
return result
except Exception as exc:
logger.exception("recover private portrait remote deletes failed")
await _rollback_and_reraise(
db,
event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_FAILED.value,
exc=exc,
detail={"celery_task": "private_portrait.recover_remote_deletes"},
asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.project_id == project_id))
group_rows = await db.execute(select(PrivatePortraitAssetGroup.id).where(PrivatePortraitAssetGroup.project_id == project_id))
asset_ids = [str(value) for value in asset_rows.scalars().all()]
group_ids = [str(value) for value in group_rows.scalars().all()]
await db.rollback()
await lease.ensure_owned()
for asset_id in asset_ids:
delete_private_portrait_asset_remote.apply_async(
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
)
for group_id in group_ids:
delete_private_portrait_group_remote.apply_async(
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
)
return len(asset_ids) + len(group_ids)
finally:
await lease.close()
async def _dispatch_remote_delete_recovery() -> dict[str, int]:
barrier = await guard_periodic_recovery()
if barrier is not None:
return {"asset_count": 0, "group_count": 0, "total_count": 0}
lock = await RedisExecutionLockLease.acquire(
lock_key=settings.PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY,
ttl_seconds=240,
renew_interval_seconds=30,
log_context="private_portrait_delete_recovery",
)
if lock is None:
return {"asset_count": 0, "group_count": 0, "total_count": 0}
async with lock:
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value]
async with async_session() as db:
asset_rows = await db.execute(
select(PrivatePortraitAsset.id)
.where(PrivatePortraitAsset.remote_delete_status.in_(statuses))
.order_by(PrivatePortraitAsset.updated_at.asc(), PrivatePortraitAsset.id.asc())
.limit(PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
)
asset_ids = [str(value) for value in asset_rows.scalars().all()]
remaining = max(0, PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE - len(asset_ids))
group_ids: list[str] = []
if remaining:
group_rows = await db.execute(
select(PrivatePortraitAssetGroup.id)
.where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses))
.order_by(PrivatePortraitAssetGroup.updated_at.asc(), PrivatePortraitAssetGroup.id.asc())
.limit(remaining)
)
group_ids = [str(value) for value in group_rows.scalars().all()]
await db.rollback()
for asset_id in asset_ids:
delete_private_portrait_asset_remote.apply_async(
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
)
for group_id in group_ids:
delete_private_portrait_group_remote.apply_async(
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
)
return {"asset_count": len(asset_ids), "group_count": len(group_ids), "total_count": len(asset_ids) + len(group_ids)}
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value, queue=QUEUE, bind=True, max_retries=5, default_retry_delay=30)
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
try:
return run_async(_inner())
return run_async(_run_poll_asset(asset_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def sync_private_portrait_due_assets(self) -> int:
try:
return run_async(_dispatch_due_assets())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
try:
return run_async(_run_delete_asset(asset_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_group_remote(self, group_id: str) -> None:
try:
return run_async(_run_delete_group(group_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_project_remote(self, project_id: str) -> int:
try:
return run_async(_dispatch_project_deletes(project_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
try:
return run_async(_dispatch_remote_delete_recovery())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
video-gen-api/app/tasks/shot_replicate_flow_tasks.py
+21 -2
View File
@@ -10,6 +10,7 @@ from app.services.module_async_recovery_service import (
TASK_SHOT_VIDEO_PROMPT,
acquire_object_lock,
cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started,
release_object_lock,
register_module_step_task,
@@ -37,7 +38,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_image_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
@@ -63,7 +73,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_video_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
video-gen-api/app/tasks/shot_replicate_tasks.py
+482 -196
View File
@@ -1,13 +1,16 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from sqlalchemy import select, update
from app.config import settings
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.shot_replicate import (
ModuleCodeEnum,
ShotAnalysisStatusEnum,
@@ -21,25 +24,18 @@ from app.models.base import async_session
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
from app.services.module_async_recovery_service import (
OBJECT_SHOT_SEGMENT_ANALYSIS,
OBJECT_SHOT_SPLIT_SEGMENT,
OBJECT_SHOT_TASK_SET_ANALYSIS,
acquire_object_lock,
cleanup_active_if_terminal,
mark_active_started,
postpone_active_task,
register_shot_segment_analysis_task,
register_shot_split_task,
register_shot_task_set_analysis_task,
release_object_lock,
remove_active_task,
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
redis_acquire_lock,
redis_release_lock,
)
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
from app.services.generation.billing_service import charge_shot_video_analysis_usage
from app.services.shot_video_split_service import split_video_segment_async
from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async
from app.services.upload_video_asset_service import validate_split_range
from app.services.upload_resource import record_shot_segment_upload_resource
from app.tasks.async_runner import run_async
@@ -48,8 +44,8 @@ from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
SPLIT_QUEUE = "gen_result_download"
ANALYSIS_QUEUE = "gen_chatapi_create"
SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value
ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value
def _now() -> datetime:
@@ -82,12 +78,80 @@ async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> Non
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.id == task_set_id,
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_attempt_no == attempt_no,
ShotReplicateTaskSet.analysis_claim_token == token,
ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.analysis_attempt_no == attempt_no,
ShotReplicateSegment.analysis_claim_token == token,
ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str:
return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}"
async def _run_analyze_original_video(task_set_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if not lock_token:
token = uuid.uuid4().hex
attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_task_set",
owner_id=task_set_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token),
)
if lease is None:
return
await register_shot_task_set_analysis_task(task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
task_set_user_id: str | None = None
video_url: str | None = None
@@ -100,16 +164,28 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await db.rollback()
return
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
current_lease = task_set.analysis_lease_until
if (
task_set.analysis_claim_token
and task_set.analysis_claim_token != token
and current_lease
and current_lease > _now()
):
await db.rollback()
return
task_set_user_id = task_set.user_id
video_url = task_set.video_url
if int(task_set.analysis_attempt_no or 1) != attempt_no:
await db.rollback()
return
task_set_user_id = str(task_set.user_id)
video_url = str(task_set.video_url)
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
task_set.analysis_claim_token = token
task_set.analysis_started_at = _now()
task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
task_set.analysis_error_message = None
await db.commit()
@@ -119,21 +195,40 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析开始",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
detail={
"task_set_id": task_set_id,
"video_url": video_url,
"analysis_mode": "full_breakdown",
"analysis_attempt_no": attempt_no,
"queue": ANALYSIS_QUEUE,
},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}")
result = await db.execute(
async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(
call_db,
video_url or "",
user_id=task_set_user_id,
mode="full_breakdown",
task_set_id=task_set_id,
trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await db.rollback()
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if (
not task_set
or int(task_set.analysis_attempt_no or 1) != attempt_no
or task_set.analysis_claim_token != token
or str(task_set.video_url) != str(video_url)
or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return
result_json = analyzed.result
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -144,9 +239,11 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
task_set.analysis_result_json = result_json
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = None
await charge_shot_video_analysis_usage(
db,
call_db,
user_id=task_set.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
owner_id=task_set.id,
@@ -154,9 +251,9 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
description="拆镜复刻-原视频分析",
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
source_project_id=task_set.id,
attempt_no=attempt_no,
)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
await call_db.commit()
log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
@@ -175,8 +272,10 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析成功",
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "token_usage": analyzed.usage},
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no},
)
except RedisExecutionLockError:
raise
except Exception as exc:
async with async_session() as db:
result = await db.execute(
@@ -186,34 +285,69 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1)
)
task_set = result.scalar_one_or_none()
if task_set:
task_set_user_id = task_set_user_id or task_set.user_id
if (
task_set
and int(task_set.analysis_attempt_no or 1) == attempt_no
and task_set.analysis_claim_token == token
and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value
):
task_set_user_id = task_set_user_id or str(task_set.user_id)
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = str(exc)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
else:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
await db.rollback()
log_module_error(
module=MODULE,
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析失败",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc,
)
finally:
await release_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id, token=lock_token)
await lease.close()
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if not lock_token:
token = uuid.uuid4().hex
attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token),
)
if lease is None:
return
await register_shot_segment_analysis_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
user_id: str | None = None
task_set_id: str | None = None
@@ -227,17 +361,23 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or not segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await db.rollback()
return
if segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
current_lease = segment.analysis_lease_until
if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now():
await db.rollback()
return
user_id = segment.user_id
task_set_id = segment.task_set_id
video_url = segment.segment_video_url
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id)
if int(segment.analysis_attempt_no or 1) != attempt_no:
await db.rollback()
return
user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
video_url = str(segment.segment_video_url)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
segment.analysis_claim_token = token
segment.analysis_started_at = _now()
segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
segment.analysis_error_message = None
await db.commit()
@@ -248,21 +388,35 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析开始",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}")
result = await db.execute(
async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(
call_db,
video_url or "",
user_id=user_id,
mode="summary_only",
task_set_id=task_set_id,
segment_id=segment_id,
trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await db.rollback()
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if (
not segment
or int(segment.analysis_attempt_no or 1) != attempt_no
or segment.analysis_claim_token != token
or str(segment.segment_video_url) != str(video_url)
or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return
result_json = analyzed.result
segment.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -273,9 +427,11 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
segment.segment_audience = segment.original_video_audience
segment.analysis_json = result_json
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = None
await charge_shot_video_analysis_usage(
db,
call_db,
user_id=segment.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
owner_id=segment.id,
@@ -284,9 +440,9 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
source_project_id=segment.task_set_id,
source_step_id=segment.id,
attempt_no=attempt_no,
)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
await call_db.commit()
log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
@@ -306,8 +462,10 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析成功",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "token_usage": analyzed.usage},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no},
)
except RedisExecutionLockError:
raise
except Exception as exc:
async with async_session() as db:
result = await db.execute(
@@ -317,15 +475,21 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if segment:
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
if (
segment
and int(segment.analysis_attempt_no or 1) == attempt_no
and segment.analysis_claim_token == token
and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = str(exc)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
else:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
await db.rollback()
log_module_error(
module=MODULE,
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
@@ -333,49 +497,90 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析失败",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc,
)
finally:
await release_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id, token=lock_token)
await lease.close()
async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.split_retry_count == attempt_no,
ShotReplicateSegment.split_claim_token == token,
ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value,
)
.values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _run_split_one_segment(segment_id: str) -> None:
segment_lock_key = f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}"
segment_lock_token = await redis_acquire_lock(
lock_key=segment_lock_key,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
log_context="shot_split_segment_lock",
)
if not segment_lock_token:
return
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial:
return
if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url:
return
attempt = int(initial.split_retry_count or 0) + 1
await db.rollback()
await register_shot_split_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_SPLIT.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt,
task_name=CeleryTaskName.SHOT_SPLIT_ONE.value,
queue=SPLIT_QUEUE,
),
lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}",
hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ShotSplitStatusEnum.PROCESSING.value,
db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token),
)
if lease is None:
return
semaphore_key: str | None = None
user_id: str | None = None
task_set_id: str | None = None
source_path: str | None = None
split_result = None
try:
semaphore_key = await _acquire_split_semaphore(segment_id)
semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}")
if not semaphore_key:
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
step_id=segment_id,
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
detail={"segment_id": segment_id, "reason": "semaphore_full"},
)
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=delay,
reason="semaphore_full",
detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt},
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL)
split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
)
return
async with async_session() as db:
@@ -387,10 +592,10 @@ async def _run_split_one_segment(segment_id: str) -> None:
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
user_id = segment.user_id
task_set_id = segment.task_set_id
user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
@@ -399,10 +604,18 @@ async def _run_split_one_segment(segment_id: str) -> None:
)
task_set = task_set_result.scalar_one_or_none()
if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if (
segment.split_claim_token
and segment.split_claim_token != token
and segment.split_lease_until
and segment.split_lease_until > _now()
):
await db.rollback()
return
validate_split_range(
@@ -410,25 +623,21 @@ async def _run_split_one_segment(segment_id: str) -> None:
end_second=segment.end_second,
video_duration_seconds=task_set.video_duration_seconds,
)
now = _now()
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
segment.split_claim_token = token
segment.split_started_at = now
segment.split_lease_until = _lease_until(now)
segment.split_retry_count = int(segment.split_retry_count or 0) + 1
segment.split_retry_count = attempt
segment.split_next_retry_at = None
segment.split_last_error = None
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.commit()
await register_shot_split_task(segment_id, task_set_id=segment.task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
source_path = task_set.video_path
source_path = str(task_set.video_path)
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
start_second = segment.start_second
end_second = segment.end_second
attempt = segment.split_retry_count
start_second = float(segment.start_second)
end_second = float(segment.end_second)
await db.commit()
log_module_event_file(
module=MODULE,
@@ -444,16 +653,20 @@ async def _run_split_one_segment(segment_id: str) -> None:
"start_second": start_second,
"end_second": end_second,
"attempt": attempt,
"queue": SPLIT_QUEUE,
},
)
split_result = await split_video_segment_async(
source_path=source_path,
source_path=source_path or "",
segment_id=segment_id,
start_second=start_second,
end_second=end_second,
date_dir=date_dir,
attempt_key=f"attempt-{attempt}-{token[-8:]}",
finalize=False,
)
await lease.ensure_owned()
async with async_session() as db:
result = await db.execute(
@@ -463,9 +676,16 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
if (
not segment
or int(segment.split_retry_count or 0) != attempt
or segment.split_claim_token != token
or segment.split_status != ShotSplitStatusEnum.PROCESSING.value
):
await db.rollback()
cleanup_split_result(split_result)
return
split_result = finalize_split_result(split_result)
segment.segment_video_url = split_result.url
segment.segment_video_path = split_result.path
await record_shot_segment_upload_resource(
@@ -476,35 +696,46 @@ async def _run_split_one_segment(segment_id: str) -> None:
file_size_bytes=split_result.file_size_bytes,
)
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
segment.split_claim_token = None
segment.split_completed_at = _now()
segment.split_lease_until = None
segment.split_next_retry_at = None
segment.split_last_error = None
source_mode = str(segment.source_mode)
final_task_set_id = str(segment.task_set_id)
final_user_id = str(segment.user_id)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=segment.task_set_id,
step_id=segment.id,
user_id=segment.user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment.id,
"task_set_id": segment.task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": segment.source_mode,
},
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=final_task_set_id,
step_id=segment_id,
user_id=final_user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment_id,
"task_set_id": final_task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": source_mode,
"attempt": attempt,
},
)
if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
analyze_custom_segment_video.apply_async(
args=[segment_id],
queue=ANALYSIS_QUEUE,
countdown=0,
task_id=f"shot-analysis:segment:{segment_id}:attempt:1",
)
if segment.source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
analyze_custom_segment_video.apply_async(args=[segment.id], queue=ANALYSIS_QUEUE, countdown=0)
except RedisExecutionLockError:
cleanup_split_result(split_result)
raise
except Exception as exc:
cleanup_split_result(split_result)
next_retry_delay: int | None = None
final_failed = False
async with async_session() as db:
@@ -515,37 +746,41 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
return
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
attempt = int(segment.split_retry_count or 0)
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
if (
segment
and int(segment.split_retry_count or 0) == attempt
and segment.split_claim_token == token
and segment.split_status == ShotSplitStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.split_claim_token = None
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
next_retry_delay = max(
1,
int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()),
)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
if segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
next_retry_delay = max(1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=next_retry_delay,
reason="split_retry_waiting",
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
elif final_failed:
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if next_retry_delay and celery_app:
split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=next_retry_delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
log_module_error(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
@@ -559,61 +794,111 @@ async def _run_split_one_segment(segment_id: str) -> None:
"source_path": source_path,
"next_retry_delay_seconds": next_retry_delay,
"final_failed": final_failed,
"attempt": attempt,
},
exc=exc,
)
finally:
await _release_split_semaphore(semaphore_key, segment_id)
await redis_release_lock(lock_key=segment_lock_key, token=segment_lock_token, log_context="shot_split_segment_lock")
await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}")
await lease.close()
async def _run_recover_split_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
redis = await get_registry_redis()
token: str | None = None
if redis is not None:
token = await redis_acquire_lock(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="shot_split_recovery",
)
if not token:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
try:
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_split_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_split_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
return result
finally:
if token:
await redis_release_lock(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
token=token,
log_context="shot_split_recovery",
)
result["execution_lock"] = "lock_acquired"
return result
async def _run_recover_analysis_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_analysis_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_analysis_tasks_once(db)
result["execution_lock"] = "lock_acquired"
return result
if celery_app:
@celery_app.task(name="shot_replicate.analyze_original_video")
def analyze_original_video(task_set_id: str) -> None:
return run_async(_run_analyze_original_video(task_set_id))
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_original_video(self, task_set_id: str) -> None:
try:
return run_async(_run_analyze_original_video(task_set_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(name="shot_replicate.split_one_segment", bind=True, max_retries=0)
@celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True)
def split_one_segment(self, segment_id: str) -> None:
return run_async(_run_split_one_segment(segment_id))
@celery_app.task(name="shot_replicate.analyze_custom_segment_video")
def analyze_custom_segment_video(segment_id: str) -> None:
return run_async(_run_analyze_custom_segment_video(segment_id))
try:
return run_async(_run_split_one_segment(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=30)
@celery_app.task(
name="shot_replicate.recover_split_tasks_once",
name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_custom_segment_video(self, segment_id: str) -> None:
try:
return run_async(_run_analyze_custom_segment_video(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
)
def recover_analysis_tasks_once(self) -> dict[str, Any]:
return run_async(_run_recover_analysis_tasks_once())
@celery_app.task(
name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
@@ -634,3 +919,4 @@ else:
split_one_segment = _DisabledTask()
analyze_custom_segment_video = _DisabledTask()
recover_split_tasks_once = _DisabledTask()
recover_analysis_tasks_once = _DisabledTask()
video-gen-api/app/tasks/video_upscale_tasks.py
+65 -14
View File
@@ -1,10 +1,14 @@
from __future__ import annotations
import uuid
from collections.abc import Awaitable, Callable
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.models.base import async_session
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.redis_registry_service import RedisExecutionLockLease
from app.services.video_upscale.task_service import (
recover_video_upscale_tasks_once,
@@ -21,13 +25,30 @@ from app.tasks.celery_app import celery_app
async def _run_with_execution_lock(
upscale_task_id: str,
callback: Callable[[str, str, Callable[[], Awaitable[None]]], Awaitable[None]],
*,
task_name: str,
queue: str,
pipeline_stage: str,
) -> None:
lock_key = f"{settings.VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX}:{upscale_task_id}"
lease = await RedisExecutionLockLease.acquire(
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.VIDEO_UPSCALE.value,
owner_type="video_upscale_task",
owner_id=upscale_task_id,
attempt_no=1,
task_name=task_name,
queue=queue,
registry_item_id=f"video_upscale:{upscale_task_id}",
),
lock_key=lock_key,
hash_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=max(30, int(settings.VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS or 900)),
log_context="video_upscale_execution",
renew_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
heartbeat_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
pipeline_stage=pipeline_stage,
)
if lease is None:
# 重复消息已有其他 Worker 推进，不属于业务失败。
@@ -43,7 +64,13 @@ async def _run_local(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_local_upscale(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
pipeline_stage="execute_local",
)
async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> None:
@@ -52,28 +79,52 @@ async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> No
await run_remote_submit(
db, task_id, count_attempt=count_attempt, execution_token=token, execution_guard=guard
)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="submit_remote",
)
async def _run_poll(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_remote_poll(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="poll_remote",
)
async def _run_download(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_remote_result_download(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="download_remote_result",
)
async def _run_finalize(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_finalize_upscale(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
pipeline_stage="finalize",
)
async def _run_recovery() -> dict[str, Any]:
@@ -96,7 +147,7 @@ async def _run_recovery() -> dict[str, Any]:
if celery_app:
@celery_app.task(name="video_upscale.execute_local", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value, bind=True, max_retries=2)
def execute_local(self, upscale_task_id: str) -> None:
try:
return run_async(_run_local(upscale_task_id))
@@ -104,7 +155,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.submit_remote", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value, bind=True, max_retries=2)
def submit_remote(self, upscale_task_id: str, count_attempt: bool = True) -> None:
try:
return run_async(_run_submit(upscale_task_id, count_attempt=count_attempt))
@@ -112,7 +163,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.poll_remote", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value, bind=True, max_retries=2)
def poll_remote(self, upscale_task_id: str) -> None:
try:
return run_async(_run_poll(upscale_task_id))
@@ -120,7 +171,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.download_remote_result", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value, bind=True, max_retries=2)
def download_remote_result(self, upscale_task_id: str) -> None:
try:
return run_async(_run_download(upscale_task_id))
@@ -128,7 +179,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.finalize", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value, bind=True, max_retries=2)
def finalize(self, upscale_task_id: str) -> None:
try:
return run_async(_run_finalize(upscale_task_id))
@@ -136,7 +187,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.recover_once", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_RECOVER.value, bind=True, max_retries=2)
def recover_once(self) -> dict[str, Any]:
try:
return run_async(_run_recovery())