1

video-gen-api/app/tasks/generation_recovery_tasks.py

@@ -40,6 +40,7 @@ async def _recover_generation_records_once(*, include_create: bool, include_poll
                             args=[ref.owner_id],
                             kwargs={"owner_type": ref.owner_type, "generation_attempt_no": ref.generation_attempt_no},
                             queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
+                            task_id=f"generation-create:{ref.owner_type}:{ref.owner_id}:attempt:{ref.generation_attempt_no}",
                         )
                         counts["create"] += 1
                     except Exception as exc:
@@ -116,6 +117,17 @@ async def _run_generation_once() -> Dict[str, Any]:
     return {"chat_generation_task": chat_result, "generation_record": record_result}
 
 
+async def _run_create_once() -> Dict[str, Any]:
+    from app.services.generation.recovery_service import recover_stale_create_tasks_once
+
+    async with async_session() as db:
+        chat_result = await recover_stale_create_tasks_once(db)
+    record_result = await _recover_generation_records_once(
+        include_create=True, include_poll=False, include_download=False
+    )
+    return {"chat_generation_task": chat_result, "generation_record": record_result}
+
+
 async def _run_due_poll_dispatch_once() -> Dict[str, Any]:
     from datetime import datetime, timedelta, timezone
 
@@ -396,6 +408,23 @@ if celery_app:
         )
 
 
+    @celery_app.task(
+        name="generation.recover_create_tasks_once",
+        bind=True,
+        soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
+        time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
+    )
+    def recover_create_tasks_once(self) -> Dict[str, Any]:
+        return run_async(
+            _run_with_execution_lock(
+                lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create",
+                log_context="generation_create_recovery",
+                runner=_run_create_once,
+                ttl_seconds=max(55, int(settings.GENERATION_CREATE_RECOVERY_INTERVAL_SECONDS or 60) - 5),
+            )
+        )
+
+
     @celery_app.task(
         name="generation.dispatch_due_poll_tasks",
         bind=True,
@@ -424,4 +453,5 @@ else:
     startup_recovery_once = _DisabledTask()
     recover_download_tasks_once = _DisabledTask()
     recover_generation_tasks_once = _DisabledTask()
+    recover_create_tasks_once = _DisabledTask()
     dispatch_due_poll_tasks = _DisabledTask()