celery 容灾升级

video-gen-api/app/tasks/module_async_recovery_tasks.py

@@ -5,35 +5,30 @@ from typing import Any
 from app.config import settings
 from app.models.base import async_session
 from app.services.module_async_recovery_service import recover_module_async_tasks_once
-from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
+from app.services.celery_runtime.recovery_service import guard_periodic_recovery
+from app.services.redis_registry_service import RedisExecutionLockLease
 from app.tasks.async_runner import run_async
 from app.tasks.celery_app import celery_app
 
 
 async def _run_recover_module_async_tasks_once() -> dict[str, Any]:
-    redis = await get_registry_redis()
-    token: str | None = None
-    if redis is not None:
-        token = await redis_acquire_lock(
-            lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
-            ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
-            log_context="module_async_recovery",
-        )
-        if not token:
-            return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
+    barrier = await guard_periodic_recovery()
+    if barrier is not None:
+        return barrier
 
-    try:
+    lease = await RedisExecutionLockLease.acquire(
+        lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
+        ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
+        renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
+        log_context="module_async_recovery",
+    )
+    if lease is None:
+        return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
+    async with lease:
         async with async_session() as db:
             result = await recover_module_async_tasks_once(db)
-            result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
-            return result
-    finally:
-        if token:
-            await redis_release_lock(
-                lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
-                token=token,
-                log_context="module_async_recovery",
-            )
+        result["execution_lock"] = "lock_acquired"
+        return result
 
 
 if celery_app: