celery 容灾升级
This commit is contained in:
@@ -119,6 +119,8 @@ class Settings(BaseSettings):
|
||||
VIDEO_UPSCALE_RECOVERY_LOCK_KEY: str = "vg:celery:video_upscale_recovery_lock"
|
||||
VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX: str = "vg:lock:upscale:execute"
|
||||
VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS: int = 30 * 60
|
||||
VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:video_upscale:active"
|
||||
VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:video_upscale:active_index"
|
||||
VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS: int = 2
|
||||
|
||||
VIDEO_COVER_SEEK_TIME: str = "00:00:01"
|
||||
@@ -146,6 +148,27 @@ class Settings(BaseSettings):
|
||||
CELERY_ASYNC_RUNNER_MODE: str = "single_loop"
|
||||
CELERY_DB_USE_NULLPOOL: bool = False
|
||||
CELERY_STARTUP_RECOVERY_ENABLED: bool = True
|
||||
CELERY_STARTUP_RECOVERY_DELAY_SECONDS: int = 30
|
||||
CELERY_RUNTIME_STARTUP_BARRIER_KEY: str = "vg:celery:recovery:startup_barrier"
|
||||
CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS: int = 120
|
||||
CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY: str = "vg:celery:recovery:global"
|
||||
# Celery Runtime V2 Worker identity。
|
||||
# worker_instance_id 只由 worker_name + host_boot_id + 主进程启动 token 构成，
|
||||
# 与 prefork 子进程 PID、threads 线程数和 --concurrency 完全解耦。
|
||||
CELERY_RUNTIME_SCHEMA_VERSION: int = 2
|
||||
CELERY_RUNTIME_WORKER_INSTANCE_PREFIX: str = "vg:celery:v2:worker_instance"
|
||||
CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX: str = "vg:celery:v2:worker_name_instances"
|
||||
CELERY_RUNTIME_WORKER_TASK_SET_PREFIX: str = "vg:celery:v2:worker_tasks"
|
||||
CELERY_RUNTIME_LOCATION_HASH_KEY: str = "vg:celery:runtime:locations"
|
||||
CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS: int = 30
|
||||
CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS: int = 90
|
||||
CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS: int = 24 * 60 * 60
|
||||
CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS: int = 120
|
||||
CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS: int = 120
|
||||
CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD: int = 3
|
||||
CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS: int = 300
|
||||
CELERY_RUNTIME_GC_INTERVAL_SECONDS: int = 600
|
||||
CELERY_RUNTIME_GC_BATCH_SIZE: int = 500
|
||||
|
||||
CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180
|
||||
CHATAPI_VIDEO_FPS: float = 0.5
|
||||
@@ -186,18 +209,15 @@ class Settings(BaseSettings):
|
||||
DOWNLOAD_TASK_LEASE_SECONDS: int = 10 * 60
|
||||
DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS: int = 5 * 60
|
||||
DOWNLOAD_RECOVERY_BATCH_SIZE: int = 20
|
||||
DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS: int = 3
|
||||
# 下载恢复自循环：不依赖 Celery beat，不新增 worker；由 gen_result_download 队列周期扫描 DB/Redis。
|
||||
DOWNLOAD_RECOVERY_LOOP_ENABLED: bool = False
|
||||
DOWNLOAD_RECOVERY_INTERVAL_SECONDS: int = 60
|
||||
DOWNLOAD_RECOVERY_LOOP_LOCK_KEY: str = "vg:celery:download_recovery_loop_lock"
|
||||
DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS: int = 55
|
||||
DOWNLOAD_RETRY_COUNTDOWN_EXTRA_SECONDS: int = 1
|
||||
DOWNLOAD_NON_RETRYABLE_LOCAL_ERRORS: bool = True
|
||||
DOWNLOAD_EVENT_VERBOSE_ENABLED: bool = True
|
||||
MEDIA_TOKEN_SNAPSHOT_ENABLED: bool = True
|
||||
DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active"
|
||||
DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index"
|
||||
GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:generation_create:active"
|
||||
GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:generation_create:active_index"
|
||||
|
||||
# Celery 生成链路 / provider poll 容灾配置。
|
||||
# 说明：
|
||||
@@ -225,6 +245,7 @@ class Settings(BaseSettings):
|
||||
DOWNLOAD_RECOVERY_LOCK_KEY: str = "vg:celery:download_recovery_lock"
|
||||
MODULE_ASYNC_RECOVERY_LOCK_KEY: str = "vg:celery:module_async_recovery_lock"
|
||||
SHOT_SPLIT_RECOVERY_LOCK_KEY: str = "vg:celery:shot_split_recovery_lock"
|
||||
SHOT_ANALYSIS_RECOVERY_LOCK_KEY: str = "vg:celery:shot_analysis_recovery_lock"
|
||||
|
||||
# 视频到期轮询调度。
|
||||
# Celery Beat 每分钟投递轻量 dispatcher 到 gen_recovery；dispatcher 只扫描 next_poll_at 到期的视频任务。
|
||||
@@ -276,6 +297,15 @@ class Settings(BaseSettings):
|
||||
# 拆镜复刻配置。
|
||||
# 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。
|
||||
SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 3600
|
||||
SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS: int = 3720
|
||||
SHOT_ANALYSIS_TIME_LIMIT_SECONDS: int = 3900
|
||||
SHOT_ANALYSIS_QUEUE: str = "gen_shot_analysis"
|
||||
SHOT_ANALYSIS_LOCK_TTL_SECONDS: int = 180
|
||||
SHOT_ANALYSIS_LEASE_SECONDS: int = 180
|
||||
SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS: int = 30
|
||||
SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_analysis:active"
|
||||
SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_analysis:active_index"
|
||||
SHOT_ANALYSIS_LOCK_KEY_PREFIX: str = "vg:lock:shot_analysis"
|
||||
SHOT_ANALYSIS_TEMPERATURE: float = 0.1
|
||||
SHOT_ANALYSIS_MAX_TOKENS: int = 5000
|
||||
SHOT_ANALYSIS_VIDEO_FPS: float = 1.0
|
||||
@@ -291,7 +321,10 @@ class Settings(BaseSettings):
|
||||
SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20
|
||||
FFPROBE_BIN: str = ""
|
||||
|
||||
# 继续复用 gen_result_download 队列，但限制 ffmpeg 并发，避免拖慢 Chat 下载。
|
||||
# 拆镜切片使用独立队列，避免 FFmpeg 占用用户结果下载 worker。
|
||||
SHOT_SPLIT_QUEUE: str = "gen_shot_split"
|
||||
SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_split:active"
|
||||
SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_split:active_index"
|
||||
SHOT_SPLIT_MAX_CONCURRENT: int = 1
|
||||
SHOT_SPLIT_MAX_RETRY_COUNT: int = 3
|
||||
SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30
|
||||
@@ -301,6 +334,18 @@ class Settings(BaseSettings):
|
||||
SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock"
|
||||
SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore"
|
||||
|
||||
# 私域素材 Celery Runtime。轮询与远程删除使用对象级原子锁，Redis 不可用时停止外部调用。
|
||||
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_poll:active"
|
||||
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_poll:active_index"
|
||||
PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:poll"
|
||||
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_delete:active"
|
||||
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_delete:active_index"
|
||||
PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:delete"
|
||||
PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS: int = 180
|
||||
PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS: int = 30
|
||||
PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY: str = "vg:celery:private_portrait:dispatch_lock"
|
||||
PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY: str = "vg:celery:private_portrait:delete_recovery_lock"
|
||||
|
||||
SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4
|
||||
SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16"
|
||||
SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p"
|
||||
|
||||
Reference in New Issue
Block a user