90 lines
3.1 KiB
Python
90 lines
3.1 KiB
Python
# app/tasks/generation_recovery_tasks.py
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any, Dict
|
|
|
|
from app.config import settings
|
|
from app.models.base import async_session
|
|
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock
|
|
from app.tasks.async_runner import run_async
|
|
from app.tasks.celery_app import celery_app
|
|
|
|
logger = logging.getLogger("video_gen")
|
|
|
|
|
|
async def _run_download_once() -> Dict[str, Any]:
|
|
from app.services.generation_recovery_service import recover_download_tasks_once
|
|
|
|
async with async_session() as db:
|
|
return await recover_download_tasks_once(db)
|
|
|
|
|
|
async def _run_generation_once() -> Dict[str, Any]:
|
|
from app.services.generation_recovery_service import recover_generation_tasks_once
|
|
|
|
async with async_session() as db:
|
|
return await recover_generation_tasks_once(db)
|
|
|
|
|
|
async def _acquire_download_recovery_loop_lock() -> tuple[bool, str]:
|
|
"""下载恢复循环锁。
|
|
|
|
Redis 不可用时降级为直接执行 DB fallback，避免恢复能力彻底失效；
|
|
Redis 可用但锁被其他 worker 持有时，本轮跳过，不再重复投递下一轮。
|
|
"""
|
|
redis = await get_registry_redis()
|
|
if redis is None:
|
|
return True, "redis_unavailable_run_db_fallback"
|
|
token = await redis_acquire_lock(
|
|
lock_key=settings.DOWNLOAD_RECOVERY_LOOP_LOCK_KEY,
|
|
ttl_seconds=int(settings.DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS or 55),
|
|
log_context="download_recovery_loop",
|
|
)
|
|
return (bool(token), "lock_acquired" if token else "lock_held")
|
|
|
|
|
|
def _schedule_next_download_recovery_loop() -> None:
|
|
if not celery_app or not bool(getattr(settings, "DOWNLOAD_RECOVERY_LOOP_ENABLED", True)):
|
|
return
|
|
try:
|
|
recover_download_tasks_once.apply_async(
|
|
countdown=max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
|
|
queue="gen_result_download",
|
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
|
)
|
|
except Exception:
|
|
logger.exception("下载恢复循环下一轮投递失败")
|
|
|
|
|
|
if celery_app:
|
|
|
|
@celery_app.task(name="generation.recover_download_tasks_once", bind=True)
|
|
def recover_download_tasks_once(self) -> Dict[str, Any]:
|
|
acquired, reason = run_async(_acquire_download_recovery_loop_lock())
|
|
if not acquired:
|
|
return {"skipped": reason}
|
|
try:
|
|
result = run_async(_run_download_once())
|
|
result["loop_lock"] = reason
|
|
return result
|
|
finally:
|
|
_schedule_next_download_recovery_loop()
|
|
|
|
|
|
@celery_app.task(name="generation.recover_generation_tasks_once")
|
|
def recover_generation_tasks_once() -> Dict[str, Any]:
|
|
return run_async(_run_generation_once())
|
|
|
|
else:
|
|
|
|
class _DisabledTask:
|
|
def delay(self, *args: Any, **kwargs: Any) -> None:
|
|
raise RuntimeError("Celery is disabled")
|
|
|
|
def apply_async(self, *args: Any, **kwargs: Any) -> None:
|
|
raise RuntimeError("Celery is disabled")
|
|
|
|
recover_download_tasks_once = _DisabledTask()
|
|
recover_generation_tasks_once = _DisabledTask()
|