celery异步恢复任务独立队列|扩增celery子进程连接池上限配置

video-gen-api/app/tasks/celery_app.py

@@ -26,6 +26,9 @@ CELERY_TASK_IMPORTS = (
 )
 
 
+RECOVERY_QUEUE = settings.CELERY_RECOVERY_QUEUE or "gen_recovery"
+
+
 def _derive_redis_db(url: str, db_no: int) -> str:
     if not url:
         return url
@@ -73,10 +76,12 @@ if broker_url:
             "shot_replicate.split_one_segment": {"queue": "gen_result_download"},
             "shot_replicate.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
             "shot_replicate.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
-            "shot_replicate.recover_split_tasks_once": {"queue": "gen_result_download"},
-            "generation.recover_download_tasks_once": {"queue": "gen_result_download"},
-            "generation.recover_generation_tasks_once": {"queue": "gen_result_download"},
-            "module_async.recover_module_async_tasks_once": {"queue": "gen_result_download"},
+            # 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。
+            "recovery.startup_recovery_once": {"queue": RECOVERY_QUEUE},
+            "shot_replicate.recover_split_tasks_once": {"queue": RECOVERY_QUEUE},
+            "generation.recover_download_tasks_once": {"queue": RECOVERY_QUEUE},
+            "generation.recover_generation_tasks_once": {"queue": RECOVERY_QUEUE},
+            "module_async.recover_module_async_tasks_once": {"queue": RECOVERY_QUEUE},
             "user_oauth.update_oauth_accounts": {"queue": "default"},
             "app.tasks.cleanup.*": {"queue": "default"},
         },
@@ -86,7 +91,7 @@ else:
 
 
 async def _try_acquire_startup_recovery_lock() -> bool:
-    """任意 worker 启动时都可尝试抢恢复锁，避免依赖 hostname 命名。"""
+    """任意 worker 启动时都可尝试抢恢复投递锁，避免依赖 hostname 命名。"""
     from app.services.redis_registry_service import redis_acquire_lock
 
     token = await redis_acquire_lock(
@@ -103,9 +108,9 @@ def on_worker_ready(sender=None, **kwargs):
 
     注意：
     - 不启用 Celery beat。
-    - 不要求新增第四条启动命令。
-    - 不再依赖 worker hostname 是否包含 gen_result_download。
-    - 所有 worker 都尝试抢 Redis 锁，只有抢到锁的 worker 投递恢复任务。
+    - 启动容灾保留，但只投递一个 recovery.startup_recovery_once 协调任务。
+    - 协调任务走独立 gen_recovery 队列，串行扫描并把真实业务任务投回原队列。
+    - 所有 worker 都尝试抢 Redis 投递锁，只有抢到锁的 worker 投递恢复任务。
     """
     if celery_app is None:
         return
@@ -122,39 +127,21 @@ def on_worker_ready(sender=None, **kwargs):
         return
 
     try:
-        from app.tasks.generation_recovery_tasks import (
-            recover_download_tasks_once,
-            recover_generation_tasks_once,
-        )
-        from app.tasks.shot_replicate_tasks import recover_split_tasks_once
-        from app.tasks.module_async_recovery_tasks import recover_module_async_tasks_once_task
+        from app.tasks.generation_recovery_tasks import startup_recovery_once
 
         countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
-
-        recover_generation_tasks_once.apply_async(
+        startup_recovery_once.apply_async(
             countdown=countdown,
-            queue="gen_result_download",
+            queue=RECOVERY_QUEUE,
             priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
         )
-        recover_download_tasks_once.apply_async(
-            countdown=countdown + 5,
-            queue="gen_result_download",
-            priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
+        logger.info(
+            "启动容灾恢复协调任务已投递。queue=%s countdown=%s",
+            RECOVERY_QUEUE,
+            countdown,
         )
-        recover_split_tasks_once.apply_async(
-            countdown=countdown + 10,
-            queue="gen_result_download",
-            priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
-        )
-        recover_module_async_tasks_once_task.apply_async(
-            countdown=countdown + 15,
-            queue="gen_result_download",
-            priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
-        )
-
-        logger.info("启动容灾恢复任务已投递。countdown=%s", countdown)
     except Exception:
-        logger.exception("启动容灾恢复任务投递失败")
+        logger.exception("启动容灾恢复协调任务投递失败")
 
 
 @worker_process_init.connect
@@ -181,4 +168,4 @@ def on_worker_process_shutdown(**kwargs):
     except Exception:
         pass
     finally:
-        close_loop()
+        close_loop()