拆镜复刻开发完成

This commit is contained in:
2026-06-11 17:54:40 +08:00
parent a6b6e5f822
commit 74266a126c
34 changed files with 7105 additions and 470 deletions
+39 -5
View File
@@ -51,6 +51,12 @@ if broker_url:
"generation.download_generation_result_task": {"queue": "gen_result_download"},
"hot_opening.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
"hot_opening.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
"shot_replicate.analyze_original_video": {"queue": "gen_chatapi_create"},
"shot_replicate.analyze_custom_segment_video": {"queue": "gen_chatapi_create"},
"shot_replicate.split_one_segment": {"queue": "gen_result_download"},
"shot_replicate.start_image_prompt_optimize": {"queue": "gen_chatapi_create"},
"shot_replicate.start_video_prompt_optimize": {"queue": "gen_chatapi_create"},
"shot_replicate.recover_split_tasks_once": {"queue": "gen_result_download"},
"generation.recover_download_tasks_once": {"queue": "gen_result_download"},
"generation.recover_generation_tasks_once": {"queue": "gen_result_download"},
"app.tasks.cleanup.*": {"queue": "default"},
@@ -61,6 +67,18 @@ else:
celery_app = None
async def _try_acquire_startup_recovery_lock() -> bool:
"""任意 worker 启动时都可尝试抢恢复锁,避免依赖 hostname 命名。"""
from app.services.redis_registry_service import redis_acquire_lock
token = await redis_acquire_lock(
lock_key=settings.CELERY_STARTUP_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_STARTUP_RECOVERY_LOCK_TTL_SECONDS or 120),
log_context="celery_startup_recovery",
)
return bool(token)
@worker_ready.connect
def on_worker_ready(sender=None, **kwargs):
"""Celery worker 启动时做一次容灾恢复。
@@ -68,13 +86,21 @@ def on_worker_ready(sender=None, **kwargs):
注意:
- 不启用 Celery beat。
- 不要求新增第四条启动命令。
- 只让 gen_result_download worker 投递恢复任务,避免三个 worker 同时重复扫描
- 不再依赖 worker hostname 是否包含 gen_result_download
- 所有 worker 都尝试抢 Redis 锁,只有抢到锁的 worker 投递恢复任务。
"""
if celery_app is None:
return
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
return
hostname = str(getattr(sender, "hostname", "") or "")
if "gen_result_download" not in hostname:
try:
if not run_async(_try_acquire_startup_recovery_lock()):
return
except Exception:
# Redis 不可用时不阻塞 worker 启动,避免影响稳定生成链路。
logger.exception("启动容灾恢复锁获取失败,已跳过本次自动恢复投递")
return
try:
@@ -82,6 +108,7 @@ def on_worker_ready(sender=None, **kwargs):
recover_download_tasks_once,
recover_generation_tasks_once,
)
from app.tasks.shot_replicate_tasks import recover_split_tasks_once
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
@@ -95,6 +122,13 @@ def on_worker_ready(sender=None, **kwargs):
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
recover_split_tasks_once.apply_async(
countdown=countdown + 10,
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
logger.info("启动容灾恢复任务已投递。countdown=%s", countdown)
except Exception:
logger.exception("启动容灾恢复任务投递失败")
@@ -110,14 +144,14 @@ def on_worker_process_init(**kwargs):
@worker_process_shutdown.connect
def on_worker_process_shutdown(**kwargs):
"""子进程退出前关闭连接池和 event loop。"""
"""子进程退出前关闭连接池、Redis 注册表连接和 event loop。"""
try:
run_async(engine.dispose())
except Exception:
pass
try:
from app.services.celery_download_recovery_service import close_registry_redis
from app.services.redis_registry_service import close_registry_redis
run_async(close_registry_redis())
except Exception: