拆镜复刻开发完成
This commit is contained in:
@@ -51,6 +51,12 @@ if broker_url:
|
||||
"generation.download_generation_result_task": {"queue": "gen_result_download"},
|
||||
"hot_opening.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
|
||||
"hot_opening.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
|
||||
"shot_replicate.analyze_original_video": {"queue": "gen_chatapi_create"},
|
||||
"shot_replicate.analyze_custom_segment_video": {"queue": "gen_chatapi_create"},
|
||||
"shot_replicate.split_one_segment": {"queue": "gen_result_download"},
|
||||
"shot_replicate.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
|
||||
"shot_replicate.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
|
||||
"shot_replicate.recover_split_tasks_once": {"queue": "gen_result_download"},
|
||||
"generation.recover_download_tasks_once": {"queue": "gen_result_download"},
|
||||
"generation.recover_generation_tasks_once": {"queue": "gen_result_download"},
|
||||
"app.tasks.cleanup.*": {"queue": "default"},
|
||||
@@ -61,6 +67,18 @@ else:
|
||||
celery_app = None
|
||||
|
||||
|
||||
async def _try_acquire_startup_recovery_lock() -> bool:
|
||||
"""任意 worker 启动时都可尝试抢恢复锁,避免依赖 hostname 命名。"""
|
||||
from app.services.redis_registry_service import redis_acquire_lock
|
||||
|
||||
token = await redis_acquire_lock(
|
||||
lock_key=settings.CELERY_STARTUP_RECOVERY_LOCK_KEY,
|
||||
ttl_seconds=int(settings.CELERY_STARTUP_RECOVERY_LOCK_TTL_SECONDS or 120),
|
||||
log_context="celery_startup_recovery",
|
||||
)
|
||||
return bool(token)
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def on_worker_ready(sender=None, **kwargs):
|
||||
"""Celery worker 启动时做一次容灾恢复。
|
||||
@@ -68,13 +86,21 @@ def on_worker_ready(sender=None, **kwargs):
|
||||
注意:
|
||||
- 不启用 Celery beat。
|
||||
- 不要求新增第四条启动命令。
|
||||
- 只让 gen_result_download worker 投递恢复任务,避免三个 worker 同时重复扫描。
|
||||
- 不再依赖 worker hostname 是否包含 gen_result_download。
|
||||
- 所有 worker 都尝试抢 Redis 锁,只有抢到锁的 worker 投递恢复任务。
|
||||
"""
|
||||
if celery_app is None:
|
||||
return
|
||||
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
|
||||
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
|
||||
return
|
||||
|
||||
hostname = str(getattr(sender, "hostname", "") or "")
|
||||
if "gen_result_download" not in hostname:
|
||||
try:
|
||||
if not run_async(_try_acquire_startup_recovery_lock()):
|
||||
return
|
||||
except Exception:
|
||||
# Redis 不可用时不阻塞 worker 启动,避免影响稳定生成链路。
|
||||
logger.exception("启动容灾恢复锁获取失败,已跳过本次自动恢复投递")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -82,6 +108,7 @@ def on_worker_ready(sender=None, **kwargs):
|
||||
recover_download_tasks_once,
|
||||
recover_generation_tasks_once,
|
||||
)
|
||||
from app.tasks.shot_replicate_tasks import recover_split_tasks_once
|
||||
|
||||
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
|
||||
|
||||
@@ -95,6 +122,13 @@ def on_worker_ready(sender=None, **kwargs):
|
||||
queue="gen_result_download",
|
||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||
)
|
||||
recover_split_tasks_once.apply_async(
|
||||
countdown=countdown + 10,
|
||||
queue="gen_result_download",
|
||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||
)
|
||||
|
||||
logger.info("启动容灾恢复任务已投递。countdown=%s", countdown)
|
||||
except Exception:
|
||||
logger.exception("启动容灾恢复任务投递失败")
|
||||
|
||||
@@ -110,14 +144,14 @@ def on_worker_process_init(**kwargs):
|
||||
|
||||
@worker_process_shutdown.connect
|
||||
def on_worker_process_shutdown(**kwargs):
|
||||
"""子进程退出前关闭连接池和 event loop。"""
|
||||
"""子进程退出前关闭连接池、Redis 注册表连接和 event loop。"""
|
||||
try:
|
||||
run_async(engine.dispose())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
from app.services.celery_download_recovery_service import close_registry_redis
|
||||
from app.services.redis_registry_service import close_registry_redis
|
||||
|
||||
run_async(close_registry_redis())
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user