This commit is contained in:
2026-07-20 14:01:22 +08:00
parent 53e01f6f74
commit a2e98ef3a3
75 changed files with 5912 additions and 2666 deletions
+2 -2
View File
@@ -81,7 +81,7 @@ async def calc_video_credits(
if not engine_id:
video_engines_result = await db.execute(
select(VideoEngine.id)
.where(VideoEngine.is_active == True)
.where(VideoEngine.is_active == True, VideoEngine.deleted_at.is_(None))
.order_by(VideoEngine.priority.desc())
.limit(1)
)
@@ -144,7 +144,7 @@ async def calc_image_credits(
if not engine_id:
image_engines_result = await db.execute(
select(ImageEngine.id)
.where(ImageEngine.is_active == True)
.where(ImageEngine.is_active == True, ImageEngine.deleted_at.is_(None))
.order_by(ImageEngine.priority.desc())
.limit(1)
)
@@ -50,7 +50,7 @@ def normalize_generation_count(value: int | None) -> int:
async def get_image_engine(db: AsyncSession, engine_id: str | None) -> ImageEngine:
query = select(ImageEngine).where(ImageEngine.is_active == True)
query = select(ImageEngine).where(ImageEngine.is_active == True, ImageEngine.deleted_at.is_(None))
if engine_id:
query = query.where(ImageEngine.id == engine_id)
else:
@@ -63,7 +63,7 @@ async def get_image_engine(db: AsyncSession, engine_id: str | None) -> ImageEngi
async def get_video_engine(db: AsyncSession, engine_id: str | None) -> VideoEngine:
query = select(VideoEngine).where(VideoEngine.is_active == True)
query = select(VideoEngine).where(VideoEngine.is_active == True, VideoEngine.deleted_at.is_(None))
if engine_id:
query = query.where(VideoEngine.id == engine_id)
else:
@@ -4,7 +4,7 @@ import json
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from uuid import uuid4
from typing import Awaitable, Callable
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -18,6 +18,7 @@ from app.enums.generation_task import (
GenerationType,
)
from app.models.chat_generation_task import ChatGenerationTask
from app.services.generation.pipeline.db_lock_service import execute_with_lock_timeout
from app.services.generation.ai.task_group_service import aggregate_main_task_status, load_children_map
from app.services.generation.log_service import log_task_event
from app.services.generation.provider_service import (
@@ -27,6 +28,7 @@ from app.services.generation.provider_service import (
from app.services.generation.refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.image_gen import ImageProviderError
from app.services.operation_log_service import build_exception_detail, log_operation_event
from app.services.redis_registry_service import RedisExecutionLockError
from app.utils.id_gen import generate_id
@@ -84,8 +86,15 @@ def _task_snapshot(main: ChatGenerationTask) -> SimpleNamespace:
)
async def _claim_image_main_batch(db: AsyncSession, main_task_id: str) -> ImageBatchClaim:
result = await db.execute(
async def _claim_image_main_batch(
db: AsyncSession,
main_task_id: str,
*,
execution_token: str,
) -> ImageBatchClaim:
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == main_task_id,
@@ -146,7 +155,7 @@ async def _claim_image_main_batch(db: AsyncSession, main_task_id: str) -> ImageB
await db.commit()
return ImageBatchClaim(False, main_task_id, reason="deadline_expired")
claim_token = uuid4().hex
claim_token = execution_token
main.provider_create_claim_token = claim_token
main.provider_create_started_at = now
main.provider_create_lease_until = now + timedelta(seconds=IMAGE_PROVIDER_CLAIM_LEASE_SECONDS)
@@ -232,7 +241,9 @@ async def _fail_claimed_main(
await db.rollback()
except Exception:
pass
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == main_task_id,
@@ -296,7 +307,9 @@ async def _split_children(
provider_result: dict,
provider_items: list[dict],
) -> list[str]:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == main_task_id,
@@ -343,8 +356,12 @@ async def _split_children(
index = int(item.get("generation_index") or 0)
if index < 1 or index > expected_count:
raise RuntimeError(f"无效的图片生成序号: {index}")
child_created_at = datetime.now(timezone.utc)
child = ChatGenerationTask(
id=generate_id(),
created_at=child_created_at,
resource_generation_started_at=child_created_at,
generation_attempt_no=1,
user_id=main.user_id,
original_prompt=main.original_prompt,
optimized_prompt=main.optimized_prompt,
@@ -433,13 +450,23 @@ async def _enqueue_child_downloads(db: AsyncSession, child_ids: list[str]) -> di
return {"enqueued": enqueued, "failed": failed}
async def run_image_main_batch(db: AsyncSession, main_task: ChatGenerationTask) -> list[str]:
async def run_image_main_batch(
db: AsyncSession,
main_task: ChatGenerationTask,
*,
execution_token: str,
execution_guard: Callable[[], Awaitable[None]],
) -> list[str]:
"""单次同步组图,全部成功后原子拆分 child。
绝不在组图 API 失败后退化为 N 次单图请求。
"""
main_task_id = str(main_task.id)
claim = await _claim_image_main_batch(db, main_task_id)
claim = await _claim_image_main_batch(
db,
main_task_id,
execution_token=execution_token,
)
if claim.existing_child_ids is not None:
await _enqueue_child_downloads(db, claim.existing_child_ids)
return claim.existing_child_ids
@@ -463,6 +490,7 @@ async def run_image_main_batch(db: AsyncSession, main_task: ChatGenerationTask)
claim.runtime_engine,
generation_count=generation_count,
)
await execution_guard()
provider_items = _validate_provider_batch(provider_result, generation_count)
log_operation_event(
domain="generation_ai_batch",
@@ -480,7 +508,10 @@ async def run_image_main_batch(db: AsyncSession, main_task: ChatGenerationTask)
"fallback_to_single_requests": False,
},
)
except RedisExecutionLockError:
raise
except Exception as exc:
await execution_guard()
message = exc.safe_message if isinstance(exc, ImageProviderError) else str(exc)
await _fail_claimed_main(
db,
@@ -493,6 +524,7 @@ async def run_image_main_batch(db: AsyncSession, main_task: ChatGenerationTask)
return []
try:
await execution_guard()
child_ids = await _split_children(
db,
main_task_id=main_task_id,
@@ -500,7 +532,10 @@ async def run_image_main_batch(db: AsyncSession, main_task: ChatGenerationTask)
provider_result=provider_result,
provider_items=provider_items,
)
except RedisExecutionLockError:
raise
except Exception as exc:
await execution_guard()
await _fail_claimed_main(
db,
main_task_id=main_task_id,
@@ -95,12 +95,12 @@ async def list_generation_ai_engine_options(db: AsyncSession) -> GenerationAIEng
"""获取当前启用的图片/视频生成引擎,供前端创建任务时选择 engine_id。"""
image_result = await db.execute(
select(ImageEngine)
.where(ImageEngine.is_active == True)
.where(ImageEngine.is_active == True, ImageEngine.deleted_at.is_(None))
.order_by(ImageEngine.priority.desc())
)
video_result = await db.execute(
select(VideoEngine)
.where(VideoEngine.is_active == True)
.where(VideoEngine.is_active == True, VideoEngine.deleted_at.is_(None))
.order_by(VideoEngine.priority.desc())
)
@@ -274,7 +274,7 @@ def record_to_out(
text_tokens_used=task.text_tokens_used or 0,
image_tokens_used=task.image_tokens_used or 0,
video_tokens_used=task.video_tokens_used or 0,
retry_count=task.retry_count or 0,
retry_count=task.manual_retry_count or 0,
poll_count=task.poll_count or 0,
error_message=task.error_message if is_main else _resolve_error_message(task.error_message),
created_at=task.created_at,
@@ -142,8 +142,16 @@ def _base_task_kwargs(
credits_cost: float = 0.0,
idempotency_key: str | None = None,
) -> dict[str, Any]:
resource_started_at = (
deadline_at - timedelta(minutes=int(settings.CHATAPI_ASYNC_IMAGE_DEADLINE_MINUTES or 30))
if gen_type == GenerationType.IMAGE.value
else deadline_at - timedelta(hours=int(settings.CHATAPI_ASYNC_VIDEO_FINAL_DEADLINE_HOURS or 24))
)
return {
"id": task_id,
"created_at": resource_started_at,
"resource_generation_started_at": resource_started_at,
"generation_attempt_no": 1,
"user_id": user_id,
"original_prompt": req.original_prompt,
"gen_type": gen_type,
@@ -515,12 +523,10 @@ async def enqueue_created_generation_tasks(
) -> list[str]:
"""在业务事务提交后投递任务;返回投递失败的任务ID。
投递失败会在补偿事务中将对应任务置为失败并幂等退款,视频子任务
同时触发主任务状态汇总。调用方不应在初始事务提交前调用本函数
投递失败时保留 queued 状态和已提交的计费/快照,交由 generation recovery
补投,避免 broker 短暂故障被误判为业务生成失败并提前退款
"""
from app.services.generation.ai.task_group_service import aggregate_parent_for_child
from app.services.generation.log_service import log_task_event
from app.services.generation.refund_service import mark_chat_generation_task_failed_and_refund_once
from app.tasks.generation_create_tasks import chatapi_create_generation_task
normalized_ids = list(dict.fromkeys(str(item) for item in task_ids if item))
@@ -530,6 +536,7 @@ async def enqueue_created_generation_tasks(
ChatGenerationTask.user_id,
ChatGenerationTask.parent_task_id,
ChatGenerationTask.generation_index,
ChatGenerationTask.generation_attempt_no,
).where(ChatGenerationTask.id.in_(normalized_ids))
) if normalized_ids else None
task_meta = {
@@ -537,6 +544,7 @@ async def enqueue_created_generation_tasks(
"user_id": str(row.user_id),
"parent_task_id": str(row.parent_task_id) if row.parent_task_id else None,
"generation_index": row.generation_index,
"generation_attempt_no": int(row.generation_attempt_no or 1),
}
for row in (meta_result.all() if meta_result is not None else [])
}
@@ -555,7 +563,11 @@ async def enqueue_created_generation_tasks(
detail={"generation_index": meta.get("generation_index")},
)
try:
chatapi_create_generation_task.delay(task_id)
chatapi_create_generation_task.apply_async(
args=[task_id],
kwargs={"owner_type": "chat_generation_task", "generation_attempt_no": int(meta.get("generation_attempt_no") or 1)},
queue="gen_chatapi_create",
)
await log_task_event(
task_id=task_id,
event_type="CHILD_ENQUEUE_SUCCESS",
@@ -575,33 +587,25 @@ async def enqueue_created_generation_tasks(
)
except Exception as exc:
failed_ids.append(task_id)
await db.rollback()
failed_task = await mark_chat_generation_task_failed_and_refund_once(
db,
task_id=task_id,
error_message=f"任务队列投递失败: {exc}",
pipeline_stage="failed",
)
await aggregate_parent_for_child(db, failed_task)
await db.commit()
await log_task_event(
task_id=task_id,
generation_attempt_no=int(meta.get("generation_attempt_no") or 1),
event_type="CHILD_ENQUEUE_FAILED",
to_status="failed",
to_stage="failed",
to_status="generating",
to_stage="queued",
message=str(exc),
detail={"task_id": task_id},
detail={"task_id": task_id, "recoverable": True},
)
log_operation_event(
domain="generation_ai_batch",
event_type="CHILD_ENQUEUE_FAILED",
event_status="failed",
source="api",
user_id=getattr(failed_task, "user_id", None),
group_id=getattr(failed_task, "parent_task_id", None) or task_id,
user_id=meta.get("user_id"),
group_id=meta.get("parent_task_id") or task_id,
task_id=task_id,
message=str(exc),
detail={"physical_files_deleted": False},
detail={"recoverable": True, "physical_files_deleted": False},
error=str(exc),
)
return failed_ids
@@ -14,6 +14,7 @@ from app.enums.generation_task import (
GenerationMode,
)
from app.models.chat_generation_task import ChatGenerationTask
from app.services.generation.pipeline.db_lock_service import execute_with_lock_timeout
from app.services.operation_log_service import log_operation_event
from app.services.video_upscale.guard_service import assert_no_recoverable_failed_upscale_tasks
from app.services.resource_accounting_service import (
@@ -157,7 +158,9 @@ async def aggregate_main_task_status(
*,
parent_task_id: str,
) -> ChatGenerationTask | None:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == parent_task_id,
@@ -229,8 +232,10 @@ async def aggregate_main_task_status(
main.text_tokens_used = sum(int(child.text_tokens_used or 0) for child in children)
main.image_tokens_used = sum(int(child.image_tokens_used or 0) for child in children)
main.video_tokens_used = sum(int(child.video_tokens_used or 0) for child in children)
main.retry_count = sum(int(child.retry_count or 0) for child in children)
# main 的手动重试次数只代表 main 自身,不能累加 child 的轮询/重试次数。
main.retry_count = int(main.manual_retry_count or 0)
main.poll_count = sum(int(child.poll_count or 0) for child in children)
main.poll_error_count = sum(int(child.poll_error_count or 0) for child in children)
await db.flush()
log_operation_event(
@@ -272,7 +277,9 @@ async def soft_delete_child_tasks_batch(
if not ids:
return 0
deleted_at = deleted_at or datetime.now(timezone.utc)
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id.in_(ids),
@@ -374,7 +381,9 @@ async def soft_delete_top_level_task_group(
deleted_at: datetime | None = None,
) -> int:
deleted_at = deleted_at or datetime.now(timezone.utc)
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == task_id,
@@ -558,6 +558,7 @@ async def charge_generation_media_for_record(
project_name: str | None = None,
description_prefix: str = "AI创作-",
attempt_no: int | None = None,
engine_id: str | None = None,
) -> BillingSummary:
return await charge_generation_media_by_params(
db,
@@ -567,6 +568,7 @@ async def charge_generation_media_for_record(
image_size=record.image_size,
duration=record.duration,
resolution=record.resolution,
engine_id=engine_id or getattr(record, "engine_id", None),
project_name=project_name,
description_prefix=description_prefix,
owner_type=OWNER_GENERATION_RECORD,
@@ -1,12 +1,15 @@
from __future__ import annotations
import json
import os
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Awaitable, Callable
from urllib.parse import urlparse
from app.config import settings
from app.models.chat_generation_task import ChatGenerationTask
from app.services.generation.pipeline.owner_service import GenerationOwner
from app.services.image_gen import download_image
from app.services.provider_limit import provider_limit
from app.services.resource_accounting_service import safe_file_size
@@ -34,7 +37,7 @@ def _to_aware_utc(value: datetime | None) -> datetime | None:
return value.astimezone(timezone.utc)
def _build_storage_date_dir(record: ChatGenerationTask) -> str:
def _build_storage_date_dir(record: GenerationOwner) -> str:
fixed = (getattr(record, "download_storage_date_dir", None) or "").strip().strip("/")
if fixed:
return fixed
@@ -42,6 +45,32 @@ def _build_storage_date_dir(record: ChatGenerationTask) -> str:
return created_at.strftime("%Y/%m/%d")
def _normalize_image_extension(record: GenerationOwner) -> str:
output_format = ""
try:
snapshot = json.loads(getattr(record, "engine_snapshot_json", None) or "{}")
if isinstance(snapshot, dict):
output_format = str(snapshot.get("output_format") or "").strip().lower()
except Exception:
output_format = ""
if output_format in {"jpg", "jpeg"}:
return "jpg"
if output_format in {"png", "webp"}:
return output_format
remote_url = str(getattr(record, "remote_result_url", None) or "")
try:
suffix = os.path.splitext(urlparse(remote_url).path or "")[1].lower().lstrip(".")
except Exception:
suffix = ""
if suffix in {"jpg", "jpeg"}:
return "jpg"
if suffix in {"png", "webp"}:
return suffix
return "jpg"
def _make_part_path(final_path: str) -> str:
return f"{final_path}.{uuid.uuid4().hex}.part"
@@ -65,16 +94,27 @@ def _safe_remove(path: str | None) -> None:
pass
async def _download_image_atomically(remote_url: str, final_path: str) -> str:
async def _download_image_atomically(
remote_url: str,
final_path: str,
*,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> str:
if _is_valid_file(final_path):
return final_path
os.makedirs(os.path.dirname(final_path), exist_ok=True)
part_path = _make_part_path(final_path)
try:
await download_image(remote_url, part_path)
await download_image(
remote_url,
part_path,
execution_guard=execution_guard,
)
if not _is_valid_file(part_path):
raise RuntimeError("图片下载完成但临时文件为空")
if execution_guard is not None:
await execution_guard()
os.replace(part_path, final_path)
return final_path
except Exception:
@@ -82,16 +122,27 @@ async def _download_image_atomically(remote_url: str, final_path: str) -> str:
raise
async def _download_video_atomically(remote_url: str, final_path: str) -> str:
async def _download_video_atomically(
remote_url: str,
final_path: str,
*,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> str:
if _is_valid_file(final_path):
return final_path
os.makedirs(os.path.dirname(final_path), exist_ok=True)
part_path = _make_part_path(final_path)
try:
await download_video(remote_url, part_path)
await download_video(
remote_url,
part_path,
execution_guard=execution_guard,
)
if not _is_valid_file(part_path):
raise RuntimeError("视频下载完成但临时文件为空")
if execution_guard is not None:
await execution_guard()
os.replace(part_path, final_path)
return final_path
except Exception:
@@ -101,7 +152,11 @@ async def _download_video_atomically(remote_url: str, final_path: str) -> str:
async def download_video_upscale_source(record: ChatGenerationTask) -> DownloadedGenerationResult:
async def download_video_upscale_source(
record: GenerationOwner,
*,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> DownloadedGenerationResult:
"""下载超分源视频。
源视频只供后处理使用,不生成封面,也不作为用户 GeneratedResource。
@@ -119,10 +174,16 @@ async def download_video_upscale_source(record: ChatGenerationTask) -> Downloade
part_path = build_part_mp4_path(dest)
try:
async with provider_limit("result_download", settings.RESULT_DOWNLOAD_MAX_CONCURRENCY):
await download_video(record.remote_result_url, part_path)
await download_video(
record.remote_result_url,
part_path,
execution_guard=execution_guard,
)
if not _is_valid_file(part_path):
raise RuntimeError("超分源视频下载完成但临时文件为空")
await probe_video(part_path)
if execution_guard is not None:
await execution_guard()
os.replace(part_path, dest)
except Exception:
_safe_remove(part_path)
@@ -138,7 +199,11 @@ async def download_video_upscale_source(record: ChatGenerationTask) -> Downloade
)
async def download_generation_result(record: ChatGenerationTask) -> DownloadedGenerationResult:
async def download_generation_result(
record: GenerationOwner,
*,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> DownloadedGenerationResult:
if not record.remote_result_url:
raise ValueError("缺少远程结果URL")
@@ -147,13 +212,18 @@ async def download_generation_result(record: ChatGenerationTask) -> DownloadedGe
if record.gen_type == "image":
dest_dir = os.path.join(settings.STORAGE_IMAGE_LOCAL_PATH, date_dir)
os.makedirs(dest_dir, exist_ok=True)
dest = os.path.join(dest_dir, f"{record.id}.png")
extension = _normalize_image_extension(record)
dest = os.path.join(dest_dir, f"{record.id}.{extension}")
async with provider_limit("result_download", settings.RESULT_DOWNLOAD_MAX_CONCURRENCY):
await _download_image_atomically(record.remote_result_url if record.remote_result_url else "", dest)
await _download_image_atomically(
record.remote_result_url if record.remote_result_url else "",
dest,
execution_guard=execution_guard,
)
return DownloadedGenerationResult(
url=f"/generate/images/{date_dir}/{record.id}.png",
url=f"/generate/images/{date_dir}/{record.id}.{extension}",
storage_path=dest,
file_size_bytes=safe_file_size(dest),
resource_type="image",
@@ -164,14 +234,20 @@ async def download_generation_result(record: ChatGenerationTask) -> DownloadedGe
dest = os.path.join(dest_dir, f"{record.id}.mp4")
async with provider_limit("result_download", settings.RESULT_DOWNLOAD_MAX_CONCURRENCY):
await _download_video_atomically(record.remote_result_url if record.remote_result_url else "", dest)
await _download_video_atomically(
record.remote_result_url if record.remote_result_url else "",
dest,
execution_guard=execution_guard,
)
cover_url, cover_storage_path = create_video_cover_for_local_video(
record_id=record.id,
video_path=dest,
date_dir=date_dir,
log_prefix=f"ChatGenerationTask视频封面生成 task_id={record.id}",
log_prefix=f"生成资源视频封面 task_id={record.id}",
)
if execution_guard is not None:
await execution_guard()
return DownloadedGenerationResult(
url=f"/generate/videos/{date_dir}/{record.id}.mp4",
@@ -23,6 +23,7 @@ from app.models.module_generation_step import ModuleGenerationStep
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.user import User
from app.schemas.generation_ai import GenerationAIHistoryBatchDeleteOut
from app.services.generation.pipeline.db_lock_service import execute_with_lock_timeout
from app.services.generation.ai.task_group_service import soft_delete_child_tasks_batch
from app.services.module_generation_flow_base_service import is_active_chat_generation_task
from app.services.module_generation_log_service import log_module_event_file
@@ -105,7 +106,9 @@ async def _load_chat_tasks_for_steps(
task_ids = _task_id_list(steps)
if not task_ids:
return {}
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id.in_(task_ids),
@@ -194,7 +197,9 @@ async def _delete_generation_records(
ids: list[str],
deleted_at: datetime,
) -> GenerationAIHistoryBatchDeleteOut:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(GenerationRecord)
.where(
GenerationRecord.id.in_(ids),
@@ -238,7 +243,9 @@ async def _delete_chat_tasks(
ids: list[str],
deleted_at: datetime,
) -> GenerationAIHistoryBatchDeleteOut:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id.in_(ids),
@@ -325,7 +332,9 @@ async def _load_module_projects_by_ids(
source: GenerationHistorySourceEnum,
project_ids: list[str],
) -> list[ModuleGenerationProject]:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id.in_(project_ids),
@@ -351,7 +360,8 @@ async def _soft_delete_module_projects(
if not project_ids:
return [], [], 0
step_result = await db.execute(
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.project_id.in_(project_ids),
@@ -452,7 +462,9 @@ async def _delete_shot_segments(
ids: list[str],
deleted_at: datetime,
) -> GenerationAIHistoryBatchDeleteOut:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ShotReplicateSegment)
.where(
ShotReplicateSegment.id.in_(ids),
@@ -4,9 +4,13 @@ import hashlib
import json
from typing import Any
from app.enums.generation_task import GenerationMode, GenerationOwnerType
from app.models.base import async_session
from app.models.chat_generation_task import ChatGenerationTask
from app.models.chat_generation_task_event import ChatGenerationTaskEvent
from app.models.chat_provider_call_log import ChatProviderCallLog
from app.models.generation_record import GenerationRecord
from app.services.operation_log_service import build_exception_detail, log_operation_event, sanitize_log_value
from app.utils.id_gen import generate_id
MAX_EXCERPT_CHARS = 2000
@@ -22,12 +26,9 @@ def _safe_json(data: Any) -> str | None:
def _excerpt(data: Any, limit: int = MAX_EXCERPT_CHARS) -> str | None:
text = _safe_json(data)
text = _safe_json(sanitize_log_value(data))
if text is None:
return None
# Avoid storing secrets in logs.
text = text.replace("Authorization", "Authorization-REDACTED")
text = text.replace("api_key", "api_key_REDACTED")
if len(text) > limit:
return text[:limit] + "...[truncated]"
return text
@@ -40,12 +41,74 @@ def _hash(data: Any) -> str | None:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def _owner_fields(
obj: Any | None,
*,
owner_type: str | None,
owner_id: str | None,
task_id: str | None,
record_id: str | None,
generation_attempt_no: int | None,
generation_mode: str | None,
) -> dict[str, Any] | None:
if isinstance(obj, GenerationRecord) or record_id:
resolved_owner_type = GenerationOwnerType.GENERATION_RECORD.value
resolved_owner_id = record_id or owner_id or getattr(obj, "id", None)
resolved_task_id = None
resolved_record_id = resolved_owner_id
resolved_mode = generation_mode or GenerationMode.GENERATION_RECORD.value
elif isinstance(obj, ChatGenerationTask) or task_id:
resolved_owner_type = GenerationOwnerType.CHAT_GENERATION_TASK.value
resolved_owner_id = task_id or owner_id or getattr(obj, "id", None)
resolved_task_id = resolved_owner_id
resolved_record_id = None
resolved_mode = generation_mode or getattr(obj, "generation_mode", GenerationMode.CHATAPI_ASYNC.value)
else:
resolved_owner_type = owner_type or GenerationOwnerType.CHAT_GENERATION_TASK.value
resolved_owner_id = owner_id
if resolved_owner_type == GenerationOwnerType.GENERATION_RECORD.value:
resolved_task_id = None
resolved_record_id = resolved_owner_id
resolved_mode = generation_mode or GenerationMode.GENERATION_RECORD.value
else:
resolved_task_id = resolved_owner_id
resolved_record_id = None
resolved_mode = generation_mode or GenerationMode.CHATAPI_ASYNC.value
if not resolved_owner_id:
return None
return {
"owner_type": resolved_owner_type,
"owner_id": str(resolved_owner_id),
"task_id": str(resolved_task_id) if resolved_task_id else None,
"generation_record_id": str(resolved_record_id) if resolved_record_id else None,
"generation_attempt_no": int(generation_attempt_no or getattr(obj, "generation_attempt_no", 1) or 1),
"generation_mode": str(resolved_mode or ""),
}
def _fallback_log(event_type: str, fields: dict[str, Any] | None, exc: Exception) -> None:
fields = fields or {}
log_operation_event(
domain="generation_pipeline",
event_type="PIPELINE_DB_LOG_FAILED",
event_status="failed",
task_id=fields.get("owner_id"),
message=f"数据库生成日志写入失败: {event_type}",
detail=build_exception_detail(exc, fields),
error=str(exc),
)
async def log_task_event(
task: Any | None = None,
*,
record: Any | None = None,
owner_type: str | None = None,
owner_id: str | None = None,
task_id: str | None = None,
record_id: str | None = None,
generation_attempt_no: int | None = None,
generation_mode: str | None = None,
event_type: str,
from_status: str | None = None,
to_status: str | None = None,
@@ -54,17 +117,52 @@ async def log_task_event(
message: str | None = None,
detail: Any = None,
) -> None:
"""Write task event in a separate transaction; failure must not affect main flow."""
"""Write an owner-scoped event in a separate transaction."""
obj = task or record
fields = _owner_fields(
obj,
owner_type=owner_type,
owner_id=owner_id,
task_id=task_id,
record_id=record_id,
generation_attempt_no=generation_attempt_no,
generation_mode=generation_mode,
)
if not fields:
return
upper_event = str(event_type or "").upper()
failed_event = any(marker in upper_event for marker in ("FAILED", "TIMEOUT", "ERROR"))
log_operation_event(
domain="generation_pipeline",
event_type=event_type,
event_status="failed" if failed_event else "success",
source="pipeline",
user_id=str(getattr(obj, "user_id", "") or "") or None,
project_id=str(getattr(obj, "project_id", "") or "") or None,
task_id=fields["owner_id"],
message=message,
detail={
"owner_type": fields["owner_type"],
"owner_id": fields["owner_id"],
"generation_attempt_no": fields["generation_attempt_no"],
"generation_mode": fields["generation_mode"],
"from_status": from_status,
"to_status": to_status,
"from_stage": from_stage,
"to_stage": to_stage,
"detail_excerpt": _excerpt(detail),
},
error=message if failed_event else None,
)
try:
obj = task or record
tid = task_id or record_id or (obj.id if obj else None)
if not tid:
return
async with async_session() as db:
db.add(ChatGenerationTaskEvent(
id=generate_id(),
task_id=tid,
generation_mode=getattr(obj, "generation_mode", "chatapi_async"),
owner_type=fields["owner_type"],
task_id=fields["task_id"],
generation_record_id=fields["generation_record_id"],
generation_attempt_no=fields["generation_attempt_no"],
generation_mode=fields["generation_mode"],
event_type=event_type,
from_status=from_status,
to_status=to_status,
@@ -74,16 +172,20 @@ async def log_task_event(
detail_json=_excerpt(detail),
))
await db.commit()
except Exception:
return
except Exception as exc:
_fallback_log(event_type, fields, exc)
async def log_provider_call(
task: Any | None = None,
*,
record: Any | None = None,
owner_type: str | None = None,
owner_id: str | None = None,
task_id: str | None = None,
record_id: str | None = None,
generation_attempt_no: int | None = None,
generation_mode: str | None = None,
provider: str | None,
api_type: str,
model: str | None = None,
@@ -100,17 +202,28 @@ async def log_provider_call(
error_code: str | None = None,
error_message: str | None = None,
) -> None:
"""Write provider call log in a separate transaction; failure must not affect main flow."""
"""Write an owner-scoped provider call log in a separate transaction."""
obj = task or record
fields = _owner_fields(
obj,
owner_type=owner_type,
owner_id=owner_id,
task_id=task_id,
record_id=record_id,
generation_attempt_no=generation_attempt_no,
generation_mode=generation_mode,
)
if not fields:
return
try:
obj = task or record
tid = task_id or record_id or (obj.id if obj else None)
if not tid:
return
async with async_session() as db:
db.add(ChatProviderCallLog(
id=generate_id(),
task_id=tid,
generation_mode=getattr(obj, "generation_mode", "chatapi_async"),
owner_type=fields["owner_type"],
task_id=fields["task_id"],
generation_record_id=fields["generation_record_id"],
generation_attempt_no=fields["generation_attempt_no"],
generation_mode=fields["generation_mode"],
provider=provider,
api_type=api_type,
model=model,
@@ -130,5 +243,5 @@ async def log_provider_call(
error_message=error_message,
))
await db.commit()
except Exception:
return
except Exception as exc:
_fallback_log(api_type, fields, exc)
@@ -0,0 +1 @@
"""Shared Celery generation pipeline for ChatGenerationTask and GenerationRecord."""
@@ -0,0 +1,84 @@
from __future__ import annotations
from typing import Any
from fastapi import HTTPException
from sqlalchemy import text
from sqlalchemy.exc import DBAPIError, OperationalError
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
class DatabaseRowLockBusy(HTTPException, RuntimeError):
"""A short-lived PostgreSQL row/table lock could not be acquired in time."""
def __init__(self, message: str = "当前任务正在被其他流程处理,请稍后重试") -> None:
super().__init__(status_code=409, detail=message)
_LOCK_NOT_AVAILABLE_SQLSTATE = "55P03"
def _sqlstate_from_exception(exc: BaseException | None) -> str | None:
current: BaseException | None = exc
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
for attr in ("sqlstate", "pgcode"):
value = getattr(current, attr, None)
if value:
return str(value)
current = getattr(current, "orig", None) or getattr(current, "__cause__", None)
return None
def is_postgres_lock_timeout(exc: BaseException) -> bool:
if _sqlstate_from_exception(exc) == _LOCK_NOT_AVAILABLE_SQLSTATE:
return True
message = str(exc).lower()
return "lock timeout" in message or "could not obtain lock" in message
def raise_if_database_lock_busy(exc: BaseException) -> None:
if is_postgres_lock_timeout(exc):
raise DatabaseRowLockBusy("数据库任务行正在被其他事务处理,请稍后重试") from exc
async def apply_short_lock_timeout(
db: AsyncSession,
*,
seconds: int | None = None,
) -> None:
"""Apply a transaction-local PostgreSQL lock wait limit.
It deliberately does not change the global database configuration and is a
no-op on non-PostgreSQL test/development databases.
"""
bind = db.get_bind()
if bind is None or bind.dialect.name != "postgresql":
return
timeout_seconds = max(
1,
int(
seconds
if seconds is not None
else getattr(settings, "GENERATION_DB_LOCK_TIMEOUT_SECONDS", 5)
or 5
),
)
await db.execute(text(f"SET LOCAL lock_timeout = '{timeout_seconds}s'"))
async def execute_with_lock_timeout(
db: AsyncSession,
statement: Any,
*,
seconds: int | None = None,
):
await apply_short_lock_timeout(db, seconds=seconds)
try:
return await db.execute(statement)
except (OperationalError, DBAPIError) as exc:
raise_if_database_lock_busy(exc)
raise
@@ -0,0 +1,39 @@
from __future__ import annotations
from app.enums.celery_queue import CeleryQueue
from app.enums.generation_task import ChatGenerationTaskEventType
from app.services.generation.log_service import log_task_event
from app.services.generation.pipeline.owner_service import GenerationOwner, owner_type_of
async def enqueue_generation_create(
owner: GenerationOwner,
*,
reason: str,
) -> None:
"""Commit caller-owned state before invoking this function."""
from app.tasks.generation_create_tasks import chatapi_create_generation_task
owner_type = owner_type_of(owner)
attempt_no = int(getattr(owner, "generation_attempt_no", 1) or 1)
try:
chatapi_create_generation_task.apply_async(
args=[str(owner.id)],
kwargs={"owner_type": owner_type, "generation_attempt_no": attempt_no},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
task_id=f"generation-create:{owner_type}:{owner.id}:attempt:{attempt_no}",
)
await log_task_event(
owner,
event_type=ChatGenerationTaskEventType.GENERATION_RECORD_ENQUEUE_SUCCESS.value,
message="资源生成创建任务已投递",
detail={"reason": reason, "queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
)
except Exception as exc:
await log_task_event(
owner,
event_type=ChatGenerationTaskEventType.GENERATION_RECORD_ENQUEUE_FAILED.value,
message=str(exc),
detail={"reason": reason, "queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
)
raise
@@ -0,0 +1,59 @@
from __future__ import annotations
import json
from datetime import datetime, timezone
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.generation_status import GenerationRecordPipelineStage, GenerationStatus
from app.models.generation_record import GenerationRecord
from app.models.image_engine import ImageEngine
from app.models.video_engine import VideoEngine
from app.services.generation.ai.engine_service import build_image_snapshot, build_video_snapshot
from app.services.generation.pipeline.enqueue_service import enqueue_generation_create
from app.services.generation.pipeline.lifecycle_service import reset_execution_fields
def _json(data: dict) -> str:
return json.dumps(data, ensure_ascii=False, default=str)
def prepare_generation_record_execution(
record: GenerationRecord,
*,
engine: ImageEngine | VideoEngine,
attempt_no: int,
) -> None:
now = datetime.now(timezone.utc)
reset_execution_fields(record, started_at=now, attempt_no=attempt_no)
record.engine_id = engine.id
if record.gen_type == "image":
record.engine_snapshot_json = _json(build_image_snapshot(
engine,
record.image_size or getattr(engine, "default_size", "2K") or "2K",
record.image_proportion or "1:1",
record.image_px or "2048x2048",
))
else:
record.engine_snapshot_json = _json(build_video_snapshot(
engine,
record.aspect_ratio or "16:9",
record.resolution or "480p",
int(record.duration or 4),
))
record.status = GenerationStatus.generating.value
record.pipeline_stage = GenerationRecordPipelineStage.QUEUED.value
async def commit_and_enqueue_generation_record(
db: AsyncSession,
record: GenerationRecord,
*,
reason: str,
) -> None:
await db.commit()
try:
await enqueue_generation_create(record, reason=reason)
except Exception:
# queued stage and all execution metadata are already committed; recovery will retry.
return
@@ -0,0 +1,149 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import ChatGenerationPipelineStage, GenerationType
from app.models.chat_generation_task import ChatGenerationTask
from app.models.generation_record import GenerationRecord
from app.models.base import async_session
from app.services.generation.pipeline.db_lock_service import DatabaseRowLockBusy
from app.services.generation.pipeline.owner_service import GenerationOwner
from app.services.generation.refund_service import (
mark_chat_generation_task_failed_and_refund_once,
mark_generation_record_failed_and_refund_once,
)
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def generation_deadline(*, gen_type: str, started_at: datetime) -> datetime:
if str(gen_type or "").lower() == GenerationType.IMAGE.value:
return started_at + timedelta(minutes=max(1, int(settings.CHATAPI_ASYNC_IMAGE_DEADLINE_MINUTES or 30)))
return started_at + timedelta(hours=max(1, int(settings.CHATAPI_ASYNC_VIDEO_FINAL_DEADLINE_HOURS or 24)))
def reset_execution_fields(owner: GenerationOwner, *, started_at: datetime, attempt_no: int) -> None:
owner.generation_attempt_no = max(1, int(attempt_no or 1))
owner.resource_generation_started_at = started_at
owner.deadline_at = generation_deadline(gen_type=owner.gen_type, started_at=started_at)
owner.error_message = None
owner.seedance_task_id = None
if hasattr(owner, "provider_task_id"):
owner.provider_task_id = None
owner.remote_result_url = None
owner.provider_response_json = None
owner.provider_create_claim_token = None
owner.provider_create_lease_until = None
owner.provider_create_started_at = None
owner.retry_count = int(getattr(owner, "manual_retry_count", 0) or 0)
owner.poll_error_count = 0
owner.poll_count = 0
owner.last_poll_at = None
owner.poll_started_at = None
owner.next_poll_at = None
owner.poll_interval_seconds = 0
owner.poll_claim_token = None
owner.poll_lease_until = None
owner.download_celery_task_id = None
owner.download_enqueued_at = None
owner.download_started_at = None
owner.download_claim_token = None
owner.download_lease_until = None
owner.download_next_retry_at = None
owner.download_attempt_count = 0
owner.download_last_error = None
owner.download_storage_date_dir = None
owner.generated_at = None
owner.image_url = None
owner.video_url = None
owner.video_cover_url = None
async def notify_owner_finished(db: AsyncSession, owner: GenerationOwner) -> None:
"""Run module hooks after the generation owner transaction has committed.
Module project/step rows have their own short locks. A transient lock conflict
must not turn an already completed generation into a download/provider retry,
so hooks use a fresh short transaction with a small local retry window.
"""
_ = db
if not isinstance(owner, ChatGenerationTask):
return
from app.services.generation.module_hook_service import notify_chat_generation_task_finished
from app.services.generation.ai.task_group_service import aggregate_parent_for_child
from app.services.operation_log_service import log_operation_event
task_id = str(owner.id)
attempt_no = int(owner.generation_attempt_no or 1)
last_error: Exception | None = None
for retry_index in range(3):
try:
async with async_session() as hook_db:
result = await hook_db.execute(
select(ChatGenerationTask).where(
ChatGenerationTask.id == task_id,
ChatGenerationTask.deleted_at.is_(None),
).limit(1)
)
fresh_task = result.scalar_one_or_none()
if fresh_task is None:
return
await notify_chat_generation_task_finished(hook_db, fresh_task)
await aggregate_parent_for_child(hook_db, fresh_task)
await hook_db.commit()
return
except DatabaseRowLockBusy as exc:
last_error = exc
await asyncio.sleep(1 + retry_index)
except Exception as exc:
last_error = exc
break
log_operation_event(
domain="generation_pipeline",
event_type="MODULE_HOOK_DEFERRED_MANUAL_CHECK",
event_status="failed",
source="pipeline",
task_id=task_id,
message="生成任务已进入终态,但模块状态回填失败,需要人工排查",
detail={
"generation_attempt_no": attempt_no,
"retry_count": 3,
},
error=str(last_error or "unknown module hook error"),
)
async def mark_owner_failed_and_refund_once(
db: AsyncSession,
owner: GenerationOwner,
*,
error_message: str,
pipeline_stage: str,
) -> GenerationOwner:
if isinstance(owner, ChatGenerationTask):
await mark_chat_generation_task_failed_and_refund_once(
db,
task=owner,
error_message=error_message,
pipeline_stage=pipeline_stage,
generation_attempt_no=int(owner.generation_attempt_no or 1),
)
return owner
owner.pipeline_stage = pipeline_stage if pipeline_stage in {item.value for item in GenerationRecordPipelineStage} else GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(
db,
record=owner,
error_message=error_message,
generation_attempt_no=int(owner.generation_attempt_no or 1),
)
return owner
@@ -0,0 +1,192 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from typing import TypeAlias
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.generation_status import GenerationStatus
from app.enums.generation_task import (
ChatGenerationTaskStatus,
GenerationMode,
GenerationOwnerType,
)
from app.models.chat_generation_task import ChatGenerationTask
from app.models.generation_record import GenerationRecord
from app.services.generation.pipeline.db_lock_service import (
DatabaseRowLockBusy,
apply_short_lock_timeout,
raise_if_database_lock_busy,
)
GenerationOwner: TypeAlias = ChatGenerationTask | GenerationRecord
@dataclass(frozen=True, slots=True)
class GenerationOwnerRef:
owner_type: str
owner_id: str
generation_attempt_no: int | None = None
@classmethod
def from_owner(cls, owner: GenerationOwner) -> "GenerationOwnerRef":
return cls(
owner_type=owner_type_of(owner),
owner_id=str(owner.id),
generation_attempt_no=int(getattr(owner, "generation_attempt_no", 1) or 1),
)
def normalize_owner_type(value: str | GenerationOwnerType | None) -> str:
if isinstance(value, GenerationOwnerType):
return value.value
text = str(value or "").strip().lower()
if not text:
return GenerationOwnerType.CHAT_GENERATION_TASK.value
if text not in {item.value for item in GenerationOwnerType}:
raise ValueError(f"不支持的生成任务所有者类型: {value}")
return text
def owner_type_of(owner: GenerationOwner) -> str:
if isinstance(owner, ChatGenerationTask):
return GenerationOwnerType.CHAT_GENERATION_TASK.value
if isinstance(owner, GenerationRecord):
return GenerationOwnerType.GENERATION_RECORD.value
raise TypeError(f"不支持的生成任务对象: {type(owner)!r}")
def owner_mode(owner: GenerationOwner) -> str:
if isinstance(owner, ChatGenerationTask):
return str(owner.generation_mode or GenerationMode.CHATAPI_ASYNC.value)
return GenerationMode.GENERATION_RECORD.value
def owner_provider_task_id(owner: GenerationOwner) -> str | None:
return str(getattr(owner, "provider_task_id", None) or getattr(owner, "seedance_task_id", None) or "") or None
def set_owner_provider_task_id(owner: GenerationOwner, value: str | None) -> None:
if hasattr(owner, "provider_task_id"):
owner.provider_task_id = value
owner.seedance_task_id = value
def owner_is_generating(owner: GenerationOwner) -> bool:
if isinstance(owner, ChatGenerationTask):
return owner.status == ChatGenerationTaskStatus.GENERATING.value
return owner.status == GenerationStatus.generating.value
def owner_is_completed(owner: GenerationOwner) -> bool:
if isinstance(owner, ChatGenerationTask):
return owner.status == ChatGenerationTaskStatus.COMPLETED.value
return owner.status == GenerationStatus.completed.value
def owner_is_failed(owner: GenerationOwner) -> bool:
if isinstance(owner, ChatGenerationTask):
return owner.status == ChatGenerationTaskStatus.FAILED.value
return owner.status == GenerationStatus.failed.value
def set_owner_generating(owner: GenerationOwner) -> None:
owner.status = ChatGenerationTaskStatus.GENERATING.value if isinstance(owner, ChatGenerationTask) else GenerationStatus.generating.value
def set_owner_completed(owner: GenerationOwner) -> None:
owner.status = ChatGenerationTaskStatus.COMPLETED.value if isinstance(owner, ChatGenerationTask) else GenerationStatus.completed.value
def set_owner_failed(owner: GenerationOwner) -> None:
owner.status = ChatGenerationTaskStatus.FAILED.value if isinstance(owner, ChatGenerationTask) else GenerationStatus.failed.value
def is_attempt_current(owner: GenerationOwner, attempt_no: int | None) -> bool:
if attempt_no is None:
return True
return int(getattr(owner, "generation_attempt_no", 1) or 1) == int(attempt_no)
async def load_generation_owner(
db: AsyncSession,
*,
owner_type: str | GenerationOwnerType | None,
owner_id: str,
for_update: bool = False,
include_deleted: bool = False,
) -> GenerationOwner | None:
normalized = normalize_owner_type(owner_type)
if normalized == GenerationOwnerType.CHAT_GENERATION_TASK.value:
query = select(ChatGenerationTask).where(ChatGenerationTask.id == owner_id)
if not include_deleted:
query = query.where(ChatGenerationTask.deleted_at.is_(None))
else:
query = select(GenerationRecord).where(GenerationRecord.id == owner_id)
if not include_deleted:
query = query.where(GenerationRecord.deleted_at.is_(None))
if for_update:
await apply_short_lock_timeout(db)
query = query.with_for_update().execution_options(populate_existing=True)
try:
result = await db.execute(query.limit(1))
except Exception as exc:
raise_if_database_lock_busy(exc)
raise
return result.scalar_one_or_none()
async def load_generation_owner_for_update_retry(
db: AsyncSession,
*,
owner_type: str | GenerationOwnerType | None,
owner_id: str,
attempts: int = 3,
) -> GenerationOwner | None:
"""Retry a short owner row lock inside the same Worker.
This is intended after an external call/download has already completed so a
transient row lock does not force the Worker to repeat that external side effect.
"""
last_error: DatabaseRowLockBusy | None = None
for retry_index in range(max(1, int(attempts or 1))):
try:
return await load_generation_owner(
db,
owner_type=owner_type,
owner_id=owner_id,
for_update=True,
)
except DatabaseRowLockBusy as exc:
last_error = exc
await db.rollback()
if retry_index + 1 < max(1, int(attempts or 1)):
await asyncio.sleep(1 + retry_index)
raise last_error or DatabaseRowLockBusy()
def redis_owner_item_id(owner_type: str | GenerationOwnerType | None, owner_id: str, attempt_no: int | None = None) -> str:
normalized = normalize_owner_type(owner_type)
if attempt_no is None:
return f"{normalized}:{owner_id}"
return f"{normalized}:{owner_id}:attempt:{int(attempt_no)}"
def parse_redis_owner_item_id(value: str) -> GenerationOwnerRef:
text = str(value or "").strip()
for owner_type in (GenerationOwnerType.CHAT_GENERATION_TASK.value, GenerationOwnerType.GENERATION_RECORD.value):
prefix = f"{owner_type}:"
if text.startswith(prefix):
rest = text[len(prefix):]
marker = ":attempt:"
if marker in rest:
owner_id, attempt = rest.rsplit(marker, 1)
try:
return GenerationOwnerRef(owner_type, owner_id, int(attempt))
except ValueError:
return GenerationOwnerRef(owner_type, owner_id, None)
return GenerationOwnerRef(owner_type, rest, None)
# Historical Redis/Celery identifiers always belonged to ChatGenerationTask.
return GenerationOwnerRef(GenerationOwnerType.CHAT_GENERATION_TASK.value, text, None)
@@ -0,0 +1,148 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.generation_status import GenerationRecordPipelineStage, GenerationStatus
from app.enums.generation_task import GenerationOwnerType
from app.models.generation_record import GenerationRecord
from app.services.generation.pipeline.owner_service import GenerationOwnerRef
from app.services.redis_registry_service import ensure_aware_utc
@dataclass(frozen=True, slots=True)
class GenerationRecordRecoveryCursor:
owner_id: str
@dataclass(slots=True)
class GenerationRecordRecoveryBatch:
create: list[GenerationOwnerRef]
poll: list[GenerationOwnerRef]
download: list[GenerationOwnerRef]
next_cursor: GenerationRecordRecoveryCursor | None
async def find_generation_record_recovery_batch(
db: AsyncSession,
*,
limit: int,
cursor: GenerationRecordRecoveryCursor | None = None,
) -> GenerationRecordRecoveryBatch:
"""按稳定游标读取恢复分流所需列,避免大字段加载和 offset 扫描。"""
stages = {
GenerationRecordPipelineStage.QUEUED.value,
GenerationRecordPipelineStage.PREPARING.value,
GenerationRecordPipelineStage.CREATING_PROVIDER_TASK.value,
GenerationRecordPipelineStage.WAITING_REMOTE.value,
GenerationRecordPipelineStage.POLLING.value,
GenerationRecordPipelineStage.RESULT_READY.value,
GenerationRecordPipelineStage.DOWNLOAD_QUEUED.value,
GenerationRecordPipelineStage.DOWNLOADING.value,
GenerationRecordPipelineStage.RETRY_WAITING.value,
}
page_size = max(1, int(limit))
now = datetime.now(timezone.utc)
queue_timeout = timedelta(
seconds=max(1, int(settings.DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS or 300))
)
query = select(
GenerationRecord.id,
GenerationRecord.generation_attempt_no,
GenerationRecord.seedance_task_id,
GenerationRecord.remote_result_url,
GenerationRecord.pipeline_stage,
GenerationRecord.provider_create_lease_until,
GenerationRecord.next_poll_at,
GenerationRecord.poll_lease_until,
GenerationRecord.download_enqueued_at,
GenerationRecord.download_lease_until,
GenerationRecord.download_next_retry_at,
).where(
GenerationRecord.deleted_at.is_(None),
GenerationRecord.status == GenerationStatus.generating.value,
GenerationRecord.pipeline_stage.in_(stages),
)
if cursor is not None:
query = query.where(GenerationRecord.id > cursor.owner_id)
result = await db.execute(
query.order_by(GenerationRecord.id.asc()).limit(page_size)
)
rows = list(result.all())
create: list[GenerationOwnerRef] = []
poll: list[GenerationOwnerRef] = []
download: list[GenerationOwnerRef] = []
for (
owner_id,
attempt_no,
provider_task_id,
remote_result_url,
pipeline_stage,
provider_create_lease_until,
next_poll_at,
poll_lease_until,
download_enqueued_at,
download_lease_until,
download_next_retry_at,
) in rows:
ref = GenerationOwnerRef(
GenerationOwnerType.GENERATION_RECORD.value,
str(owner_id),
int(attempt_no or 1),
)
stage = str(pipeline_stage or "")
if str(remote_result_url or "").strip():
if stage == GenerationRecordPipelineStage.RESULT_READY.value:
download.append(ref)
elif stage == GenerationRecordPipelineStage.DOWNLOAD_QUEUED.value:
checked_enqueued_at = ensure_aware_utc(download_enqueued_at)
if (
checked_enqueued_at is None
or checked_enqueued_at + queue_timeout <= now
):
download.append(ref)
elif stage == GenerationRecordPipelineStage.DOWNLOADING.value:
if (
ensure_aware_utc(download_lease_until) is None
or ensure_aware_utc(download_lease_until) <= now
):
download.append(ref)
elif stage == GenerationRecordPipelineStage.RETRY_WAITING.value:
if (
ensure_aware_utc(download_next_retry_at) is None
or ensure_aware_utc(download_next_retry_at) <= now
):
download.append(ref)
elif str(provider_task_id or "").strip():
checked_next_poll_at = ensure_aware_utc(next_poll_at)
checked_poll_lease_until = ensure_aware_utc(poll_lease_until)
if (
(checked_next_poll_at is None or checked_next_poll_at <= now)
and (
checked_poll_lease_until is None
or checked_poll_lease_until <= now
)
):
poll.append(ref)
else:
if (
ensure_aware_utc(provider_create_lease_until) is None
or ensure_aware_utc(provider_create_lease_until) <= now
):
create.append(ref)
next_cursor = None
if len(rows) == page_size:
last = rows[-1]
next_cursor = GenerationRecordRecoveryCursor(owner_id=str(last.id))
return GenerationRecordRecoveryBatch(
create=create,
poll=poll,
download=download,
next_cursor=next_cursor,
)
@@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone
from app.config import settings
from app.enums.generation_task import GenerationType
from app.models.chat_generation_task import ChatGenerationTask
from app.services.generation.pipeline.owner_service import GenerationOwner
from app.services.redis_registry_service import ensure_aware_utc
@@ -23,7 +23,7 @@ def utc_now() -> datetime:
return datetime.now(timezone.utc)
def is_video_generation_task(task: ChatGenerationTask) -> bool:
def is_video_generation_task(task: GenerationOwner) -> bool:
return str(getattr(task, "gen_type", "") or "").lower() == GenerationType.VIDEO.value
@@ -33,7 +33,7 @@ def video_final_deadline_from(now: datetime | None = None) -> datetime:
return current_time + timedelta(hours=hours)
def ensure_video_poll_fields(task: ChatGenerationTask, *, now: datetime | None = None) -> None:
def ensure_video_poll_fields(task: GenerationOwner, *, now: datetime | None = None) -> None:
"""补齐视频轮询调度字段,兼容历史任务。"""
if not is_video_generation_task(task):
return
@@ -42,17 +42,22 @@ def ensure_video_poll_fields(task: ChatGenerationTask, *, now: datetime | None =
if ensure_aware_utc(getattr(task, "poll_started_at", None)) is None:
task.poll_started_at = current_time
if ensure_aware_utc(getattr(task, "deadline_at", None)) is None:
task.deadline_at = video_final_deadline_from(current_time)
resource_started_at = (
ensure_aware_utc(getattr(task, "resource_generation_started_at", None))
or ensure_aware_utc(getattr(task, "created_at", None))
or current_time
)
task.deadline_at = video_final_deadline_from(resource_started_at)
if getattr(task, "poll_interval_seconds", None) is None:
task.poll_interval_seconds = 0
def is_final_poll_due(task: ChatGenerationTask, *, now: datetime | None = None) -> bool:
def is_final_poll_due(task: GenerationOwner, *, now: datetime | None = None) -> bool:
deadline_at = ensure_aware_utc(getattr(task, "deadline_at", None))
return bool(deadline_at and deadline_at <= (now or utc_now()))
def is_poll_not_due(task: ChatGenerationTask, *, now: datetime | None = None, tolerance_seconds: int = 1) -> bool:
def is_poll_not_due(task: GenerationOwner, *, now: datetime | None = None, tolerance_seconds: int = 1) -> bool:
"""判断当前 poll 任务是否早于 next_poll_at。只对视频降频轮询生效。"""
if not is_video_generation_task(task):
return False
@@ -73,7 +78,7 @@ def _clamp_positive_seconds(value: int | float | None, default: int) -> int:
def build_video_pending_poll_schedule(
task: ChatGenerationTask,
task: GenerationOwner,
*,
now: datetime | None = None,
) -> PollScheduleDecision:
@@ -133,7 +138,7 @@ def build_video_pending_poll_schedule(
def build_default_poll_schedule(
task: ChatGenerationTask,
task: GenerationOwner,
*,
now: datetime | None = None,
delay_seconds: int | None = None,
@@ -75,7 +75,7 @@ async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | Non
async def _get_model_config(db: AsyncSession) -> ModelConfig:
result = await db.execute(
select(ModelConfig)
.where(ModelConfig.is_active == True)
.where(ModelConfig.is_active == True, ModelConfig.deleted_at.is_(None))
.order_by(ModelConfig.priority.desc())
.limit(1)
)
@@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.models.chat_generation_task import ChatGenerationTask
from app.services.generation.pipeline.owner_service import GenerationOwner, owner_provider_task_id
from app.models.image_engine import ImageEngine
from app.models.video_engine import VideoEngine
from app.services.generation.log_service import log_provider_call
@@ -39,7 +40,7 @@ def _try_json(value: Any) -> Any:
return None
async def get_runtime_engine(db: AsyncSession, task: ChatGenerationTask) -> Any:
async def get_runtime_engine(db: AsyncSession, task: GenerationOwner) -> Any:
"""使用任务快照冻结历史参数,只从当前引擎记录读取密钥。"""
snapshot = _loads(task.engine_snapshot_json)
if not task.engine_id:
@@ -50,7 +51,7 @@ async def get_runtime_engine(db: AsyncSession, task: ChatGenerationTask) -> Any:
result = await db.execute(select(VideoEngine).where(VideoEngine.id == task.engine_id).limit(1))
engine = result.scalar_one_or_none()
if not engine:
raise ValueError("引擎不存在或已删除")
raise ValueError("任务绑定的引擎不存在")
return SimpleNamespace(
id=task.engine_id,
name=snapshot.get("name") or engine.name,
@@ -89,7 +90,7 @@ async def get_runtime_engine(db: AsyncSession, task: ChatGenerationTask) -> Any:
)
async def create_provider_task(db: AsyncSession, task: ChatGenerationTask) -> dict:
async def create_provider_task(db: AsyncSession, task: GenerationOwner) -> dict:
if task.gen_type == "video":
return await _create_video_task(db, task)
if task.gen_type == "image":
@@ -97,12 +98,14 @@ async def create_provider_task(db: AsyncSession, task: ChatGenerationTask) -> di
raise ValueError(f"不支持的生成类型: {task.gen_type}")
async def _create_video_task(db: AsyncSession, task: ChatGenerationTask) -> dict:
async def _create_video_task(db: AsyncSession, task: GenerationOwner) -> dict:
engine = await get_runtime_engine(db, task)
# Close the engine lookup transaction before the long provider HTTP call.
await db.commit()
started = time.perf_counter()
async with provider_limit("ark_video_create", settings.ARK_VIDEO_CREATE_MAX_CONCURRENCY):
try:
provider_task_id = await submit_video_task(None, engine, task, include_media_references=True)
provider_task_id = await submit_video_task(None, engine, task, include_media_references=isinstance(task, ChatGenerationTask))
response = {"task_id": provider_task_id}
await log_provider_call(
task,
@@ -132,11 +135,13 @@ async def _create_video_task(db: AsyncSession, task: ChatGenerationTask) -> dict
async def create_image_sync_batch_result(
db: AsyncSession,
task: ChatGenerationTask,
task: GenerationOwner,
*,
generation_count: int,
) -> ImageProviderBatchResult:
engine = await get_runtime_engine(db, task)
# Do not keep a database transaction open while the synchronous provider call runs.
await db.commit()
return await create_image_sync_batch_result_with_engine(
task,
engine,
@@ -145,7 +150,7 @@ async def create_image_sync_batch_result(
async def create_image_sync_batch_result_with_engine(
task: ChatGenerationTask,
task: GenerationOwner,
engine: Any,
*,
generation_count: int,
@@ -164,7 +169,7 @@ async def create_image_sync_batch_result_with_engine(
None,
engine,
task,
include_media_references=True,
include_media_references=isinstance(task, ChatGenerationTask),
generation_count=count,
)
response_data = result.get("response_data") or result
@@ -197,7 +202,7 @@ async def create_image_sync_batch_result_with_engine(
raise
async def create_image_sync_result(db: AsyncSession, task: ChatGenerationTask) -> dict:
async def create_image_sync_result(db: AsyncSession, task: GenerationOwner) -> dict:
result = await create_image_sync_batch_result(db, task, generation_count=1)
items = result.get("items") or []
if len(items) != 1:
@@ -216,9 +221,13 @@ async def create_image_sync_result(db: AsyncSession, task: ChatGenerationTask) -
}
async def poll_provider_task(db: AsyncSession, task: ChatGenerationTask) -> dict:
async def poll_provider_task(db: AsyncSession, task: GenerationOwner) -> dict:
engine = await get_runtime_engine(db, task)
task_id = task.seedance_task_id or task.provider_task_id
# Polling may block on the remote provider; release the lookup transaction first.
await db.commit()
task_id = owner_provider_task_id(task)
if not task_id:
raise ValueError("缺少供应商任务ID")
if task.gen_type == "video":
async with provider_limit("ark_video_poll", settings.ARK_VIDEO_POLL_MAX_CONCURRENCY):
return await poll_task_status(engine, task_id)
@@ -16,6 +16,7 @@ from app.enums.generation_task import (
ChatGenerationTaskEventType,
ChatGenerationTaskStatus,
GenerationMode,
GenerationOwnerType,
GenerationType,
)
from app.models.chat_generation_task import ChatGenerationTask
@@ -27,9 +28,15 @@ from app.services.celery_download_recovery_service import (
remove_download_active,
)
from app.services.generation.log_service import log_task_event
from app.services.generation.module_hook_service import notify_chat_generation_task_finished
from app.services.generation.pipeline.db_lock_service import apply_short_lock_timeout
from app.services.generation.pipeline.lifecycle_service import notify_owner_finished
from app.services.generation.poll_schedule_service import ensure_video_poll_fields, is_poll_not_due, is_video_generation_task
from app.services.generation.refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.generation.pipeline.owner_service import (
load_generation_owner,
parse_redis_owner_item_id,
redis_owner_item_id,
)
from app.services.redis_registry_service import (
redis_get_due_registry_ids,
redis_get_registry_payloads,
@@ -42,6 +49,26 @@ logger = logging.getLogger("video_gen")
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
def _chat_registry_id(task: ChatGenerationTask) -> str:
return redis_owner_item_id(
GenerationOwnerType.CHAT_GENERATION_TASK.value,
str(task.id),
int(getattr(task, "generation_attempt_no", 1) or 1),
)
async def _load_chat_task_for_update(
db: AsyncSession, task_id: str
) -> ChatGenerationTask | None:
owner = await load_generation_owner(
db,
owner_type=GenerationOwnerType.CHAT_GENERATION_TASK.value,
owner_id=str(task_id),
for_update=True,
)
return owner if isinstance(owner, ChatGenerationTask) else None
def _now() -> datetime:
return datetime.now(timezone.utc)
@@ -140,26 +167,47 @@ async def recover_one_download_task(
if not task:
return "skip_missing_task"
if task.generation_mode not in ALLOWED_GENERATION_MODES:
await remove_download_active(task.id)
await remove_download_active(_chat_registry_id(task))
return "clean_invalid_mode"
if _is_final_task_state(task):
await remove_download_active(task.id)
await remove_download_active(_chat_registry_id(task))
return "clean_final_state"
if task.status != ChatGenerationTaskStatus.GENERATING.value:
await remove_download_active(task.id)
task_id = str(task.id)
generation_attempt_no = int(task.generation_attempt_no or 1)
generation_mode = str(task.generation_mode or "")
status = task.status
stage = task.pipeline_stage
registry_id = _chat_registry_id(task)
await db.rollback()
await remove_download_active(registry_id)
await log_task_event(
task,
owner_type=GenerationOwnerType.CHAT_GENERATION_TASK.value,
owner_id=task_id,
task_id=task_id,
generation_attempt_no=generation_attempt_no,
generation_mode=generation_mode,
event_type=ChatGenerationTaskEventType.DOWNLOAD_SKIP_NOT_GENERATING.value,
message=f"{source} 下载恢复跳过:任务不是 generating",
detail={"status": task.status, "stage": task.pipeline_stage},
detail={"status": status, "stage": stage},
)
return "clean_not_generating"
if not task.remote_result_url:
task_id = str(task.id)
generation_attempt_no = int(task.generation_attempt_no or 1)
generation_mode = str(task.generation_mode or "")
status = task.status
stage = task.pipeline_stage
await db.rollback()
await log_task_event(
task,
owner_type=GenerationOwnerType.CHAT_GENERATION_TASK.value,
owner_id=task_id,
task_id=task_id,
generation_attempt_no=generation_attempt_no,
generation_mode=generation_mode,
event_type=ChatGenerationTaskEventType.DOWNLOAD_SKIP_NO_REMOTE_RESULT_URL.value,
message=f"{source} 下载恢复跳过:缺少 remote_result_url",
detail={"status": task.status, "stage": task.pipeline_stage},
detail={"status": status, "stage": stage},
)
return "skip_no_remote_result_url"
@@ -167,38 +215,38 @@ async def recover_one_download_task(
redis_payload = payload or {}
if stage == ChatGenerationPipelineStage.RESULT_READY.value:
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 result_ready 未完成下载,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_result_ready",
)
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 result_ready 未完成下载,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
return "recover_result_ready"
if stage == DOWNLOAD_STAGE_QUEUED:
if _is_queue_timeout(task, current_time):
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 download_queued 长时间未消费,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_download_queued_timeout",
)
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 download_queued 长时间未消费,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
return "recover_queued_timeout"
await postpone_download_active_check(
record_id=task.id,
record_id=_chat_registry_id(task),
payload=payload,
check_at=_queue_timeout_at(task, current_time),
)
@@ -206,22 +254,22 @@ async def recover_one_download_task(
if stage == DOWNLOAD_STAGE_DOWNLOADING:
if _is_expired(task.download_lease_until, current_time):
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 downloading lease 过期,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_downloading_lease_expired",
)
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 downloading lease 过期,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
return "recover_downloading_expired"
await postpone_download_active_check(
record_id=task.id,
record_id=_chat_registry_id(task),
payload=payload,
check_at=task.download_lease_until,
)
@@ -229,22 +277,22 @@ async def recover_one_download_task(
if stage == DOWNLOAD_STAGE_RETRY_WAITING:
if _is_expired(task.download_next_retry_at, current_time):
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 retry_waiting 到期,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_retry_waiting_due",
)
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.DOWNLOAD_RECOVERY_ENQUEUE.value,
message=f"{source} 发现 retry_waiting 到期,启动时恢复投递下载任务",
detail={"payload": redis_payload},
)
return "recover_retry_due"
await postpone_download_active_check(
record_id=task.id,
record_id=_chat_registry_id(task),
payload=payload,
check_at=task.download_next_retry_at,
)
@@ -267,33 +315,42 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
)
payloads = await get_download_active_payloads(due_ids)
for task_id in due_ids:
result = await db.execute(
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == task_id,
ChatGenerationTask.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
task = result.scalar_one_or_none()
download_refs = {
registry_item_id: parse_redis_owner_item_id(registry_item_id)
for registry_item_id in due_ids
}
for registry_item_id, ref in download_refs.items():
if ref.owner_type != GenerationOwnerType.CHAT_GENERATION_TASK.value:
continue
task = await _load_chat_task_for_update(db, ref.owner_id)
if task is None:
await remove_download_active(task_id)
await db.rollback()
await remove_download_active(registry_item_id)
action = "clean_missing_task"
elif (
ref.generation_attempt_no is not None
and int(task.generation_attempt_no or 1)
!= int(ref.generation_attempt_no)
):
await db.rollback()
await remove_download_active(registry_item_id)
action = "clean_stale_download_attempt"
else:
current_registry_id = _chat_registry_id(task)
if registry_item_id != current_registry_id:
await remove_download_active(registry_item_id)
checked_ids.add(task.id)
action = await recover_one_download_task(
db,
task,
payload=payloads.get(task_id),
payload=payloads.get(registry_item_id),
source="startup_redis",
)
results[action] = results.get(action, 0) + 1
# DB fallback:不依赖 Redis active 注册表。
fallback_result = await db.execute(
select(ChatGenerationTask)
select(ChatGenerationTask.id)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode.in_(list(ALLOWED_GENERATION_MODES)),
@@ -310,12 +367,15 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
)
.order_by(ChatGenerationTask.updated_at.asc())
.limit(int(settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100))
.with_for_update(skip_locked=True)
)
fallback_tasks = fallback_result.scalars().all()
fallback_ids = [str(value) for value in fallback_result.scalars().all()]
for task in fallback_tasks:
if task.id in checked_ids:
for task_id in fallback_ids:
if task_id in checked_ids:
continue
task = await _load_chat_task_for_update(db, task_id)
if task is None:
await db.rollback()
continue
action = await recover_one_download_task(
db,
@@ -324,7 +384,7 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
source="startup_db",
)
results[action] = results.get(action, 0) + 1
checked_ids.add(task.id)
checked_ids.add(task_id)
return {"checked": len(checked_ids), "results": results}
@@ -341,11 +401,9 @@ async def _mark_timeout(
error_message=error_message,
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
)
await notify_chat_generation_task_finished(db, task)
from app.services.generation.ai.task_group_service import aggregate_parent_for_child
await aggregate_parent_for_child(db, task)
await db.commit()
await _remove_poll_active(task.id)
await notify_owner_finished(db, task)
await _remove_poll_active(_chat_registry_id(task))
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.TASK_TIMEOUT.value,
@@ -369,11 +427,9 @@ async def _mark_failed(
error_message=error_message,
pipeline_stage=ChatGenerationPipelineStage.FAILED.value,
)
await notify_chat_generation_task_finished(db, task)
from app.services.generation.ai.task_group_service import aggregate_parent_for_child
await aggregate_parent_for_child(db, task)
await db.commit()
await _remove_poll_active(task.id)
await notify_owner_finished(db, task)
await _remove_poll_active(_chat_registry_id(task))
await log_task_event(task, event_type=event_type, message=task.error_message, detail=detail)
return "mark_failed"
@@ -403,19 +459,19 @@ async def recover_one_generation_task(
if not task:
return "skip_missing_task"
if task.generation_mode not in ALLOWED_GENERATION_MODES:
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
return "clean_invalid_mode"
if _is_final_task_state(task):
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
return "clean_final_state"
if task.status != ChatGenerationTaskStatus.GENERATING.value:
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
return "clean_not_generating"
if bool(getattr(task, "video_upscale_enabled_snapshot", False)) and str(task.pipeline_stage or "").startswith("upscale_"):
# 原视频已经进入超分流水线,后续由 video_upscale 恢复扫描处理。
# 这里禁止再次投递原结果下载,避免覆盖保留的 source.mp4 或提前生成用户资源。
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
return "delegate_video_upscale_recovery"
has_remote_result = bool(str(task.remote_result_url or "").strip())
@@ -425,7 +481,13 @@ async def recover_one_generation_task(
# 最高优先级:只要远程结果 URL 已经落库,说明生成侧已经成功。
# 不管当前 pipeline_stage 是 queued/creating/waiting/result_ready/download_*,恢复时都不能重复 create 或 poll。
if has_remote_result:
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_has_remote_result_url",
)
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.GENERATION_RECOVERY_ENQUEUE.value,
@@ -436,12 +498,6 @@ async def recover_one_generation_task(
"deadline_expired": is_deadline_expired,
},
)
await enqueue_download_task(
db,
task,
recover=True,
reason=f"{source}_has_remote_result_url",
)
return "recover_download_has_remote_result"
# 已经过 deadline 且没有结果 URL
@@ -459,7 +515,7 @@ async def recover_one_generation_task(
)
poll_generation_task.apply_async(
args=[task.id],
kwargs={"force_due": True},
kwargs={"force_due": True, "owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": int(task.generation_attempt_no or 1)},
queue=POLL_QUEUE,
countdown=0,
)
@@ -470,13 +526,11 @@ async def recover_one_generation_task(
)
return "recover_deadline_final_poll"
await log_task_event(
return await _mark_timeout(
db,
task,
event_type=ChatGenerationTaskEventType.GENERATION_RECOVERY_TIMEOUT.value,
message=f"{source} 发现任务已到 deadline,且没有 remote_result_url/供应商任务ID,按超时失败处理",
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
error_message=f"{source} 发现任务已到 deadline,且没有远程结果或供应商任务ID",
)
return await _mark_timeout(db, task)
# 未过 deadline:有供应商任务 ID 才允许恢复到 poll 队列。
# 视频任务如果 next_poll_at 未到期,不提前 poll,只刷新 active 注册表等待 Beat dispatcher 到期投递。
@@ -523,7 +577,7 @@ async def recover_one_generation_task(
)
poll_generation_task.apply_async(
args=[task.id],
kwargs={"force_due": True},
kwargs={"force_due": True, "owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": int(task.generation_attempt_no or 1)},
queue=POLL_QUEUE,
countdown=0,
)
@@ -552,9 +606,11 @@ async def recover_one_generation_task(
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
):
task.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
await db.commit()
# Release the recovery row lock before writing an event through the
# independent logging session or talking to the broker.
await db.commit()
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.GENERATION_RECOVERY_ENQUEUE.value,
@@ -563,6 +619,7 @@ async def recover_one_generation_task(
)
chatapi_create_generation_task.apply_async(
args=[task.id],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": int(task.generation_attempt_no or 1)},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
@@ -572,7 +629,7 @@ async def recover_one_generation_task(
if task.pipeline_stage == ChatGenerationPipelineStage.RESULT_READY.value:
task.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
await db.commit()
await _remove_poll_active(task.id)
await _remove_poll_active(_chat_registry_id(task))
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.GENERATION_RECOVERY_ENQUEUE.value,
@@ -581,6 +638,7 @@ async def recover_one_generation_task(
)
chatapi_create_generation_task.apply_async(
args=[task.id],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": int(task.generation_attempt_no or 1)},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
@@ -621,28 +679,33 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
if image_main_cursor:
image_main_query = image_main_query.where(ChatGenerationTask.id > image_main_cursor)
image_main_result = await db.execute(
image_main_query.order_by(ChatGenerationTask.id.asc())
image_main_query.with_only_columns(ChatGenerationTask.id)
.order_by(ChatGenerationTask.id.asc())
.limit(image_main_batch_size)
.with_for_update()
)
image_mains = list(image_main_result.scalars().all())
if not image_mains:
image_main_ids = [str(value) for value in image_main_result.scalars().all()]
if not image_main_ids:
break
for main in image_mains:
main_id = str(main.id)
child_parent_result = await db.execute(
select(ChatGenerationTask.parent_task_id)
.where(
ChatGenerationTask.parent_task_id.in_(image_main_ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
)
.distinct()
)
split_parent_ids = {str(value) for value in child_parent_result.scalars().all() if value}
for main_id in image_main_ids:
image_main_cursor = main_id
checked_ids.add(main_id)
main = await _load_chat_task_for_update(db, main_id)
if main is None:
await db.rollback()
continue
child_result = await db.execute(
select(ChatGenerationTask.id)
.where(
ChatGenerationTask.parent_task_id == main_id,
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
)
.limit(1)
)
if child_result.scalar_one_or_none() is not None:
if main_id in split_parent_ids:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await db.commit()
@@ -653,7 +716,7 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
lease_until = ensure_aware_utc(main.provider_create_lease_until)
lease_alive = bool(main.provider_create_claim_token and lease_until and lease_until > now)
if lease_alive:
await db.commit()
await db.rollback()
results["image_main_claim_alive"] = results.get("image_main_claim_alive", 0) + 1
continue
@@ -670,19 +733,24 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
results["image_main_timeout"] = results.get("image_main_timeout", 0) + 1
continue
claim_expired = False
if main.provider_create_claim_token or main.provider_create_lease_until:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
claim_expired = True
attempt_no = int(main.generation_attempt_no or 1)
await db.commit()
if claim_expired:
await log_task_event(
main,
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
message="图片主任务供应商执行租约已过期,恢复重新投递",
)
await db.commit()
try:
chatapi_create_generation_task.apply_async(
args=[main_id],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": attempt_no},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
@@ -691,7 +759,7 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
logger.exception("恢复投递图片主任务失败 task_id=%s: %s", main_id, exc)
results["recover_image_main_enqueue_failed"] = results.get("recover_image_main_enqueue_failed", 0) + 1
if len(image_mains) < image_main_batch_size:
if len(image_main_ids) < image_main_batch_size:
break
due_poll_ids = await redis_get_due_registry_ids(
@@ -705,26 +773,34 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
log_context="poll_active",
)
for task_id in due_poll_ids:
result = await db.execute(
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == task_id,
ChatGenerationTask.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
task = result.scalar_one_or_none()
poll_refs = {
registry_item_id: parse_redis_owner_item_id(registry_item_id)
for registry_item_id in due_poll_ids
}
for registry_item_id, ref in poll_refs.items():
if ref.owner_type != GenerationOwnerType.CHAT_GENERATION_TASK.value:
continue
task = await _load_chat_task_for_update(db, ref.owner_id)
if task is None:
await _remove_poll_active(task_id)
await db.rollback()
await _remove_poll_active(registry_item_id)
action = "clean_missing_poll_task"
elif (
ref.generation_attempt_no is not None
and int(task.generation_attempt_no or 1) != int(ref.generation_attempt_no)
):
await db.rollback()
await _remove_poll_active(registry_item_id)
action = "clean_stale_poll_attempt"
else:
current_registry_id = _chat_registry_id(task)
if registry_item_id != current_registry_id:
await _remove_poll_active(registry_item_id)
checked_ids.add(task.id)
action = await recover_one_generation_task(
db,
task,
payload=poll_payloads.get(task_id),
payload=poll_payloads.get(registry_item_id),
source="startup_poll_redis",
)
results[action] = results.get(action, 0) + 1
@@ -735,7 +811,7 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
for _round in range(max_rounds):
query_result = await db.execute(
select(ChatGenerationTask)
select(ChatGenerationTask.id)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode.in_(list(ALLOWED_GENERATION_MODES)),
@@ -753,15 +829,18 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
)
.order_by(ChatGenerationTask.updated_at.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
tasks = query_result.scalars().all()
if not tasks:
task_ids = [str(value) for value in query_result.scalars().all()]
if not task_ids:
break
progressed_this_round = 0
for task in tasks:
if task.id in checked_ids:
for task_id in task_ids:
if task_id in checked_ids:
continue
task = await _load_chat_task_for_update(db, task_id)
if task is None:
await db.rollback()
continue
action = await recover_one_generation_task(
db,
@@ -770,11 +849,11 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
source="startup_db",
)
results[action] = results.get(action, 0) + 1
checked_ids.add(task.id)
checked_ids.add(task_id)
total_db_checked += 1
progressed_this_round += 1
if len(tasks) < batch_size or progressed_this_round <= 0:
if len(task_ids) < batch_size or progressed_this_round <= 0:
break
# 子任务可能在 worker 中断前已进入终态但主任务尚未汇总,按稳定游标完整重算全部主任务。
@@ -820,6 +899,7 @@ async def dispatch_due_poll_tasks_once(db: AsyncSession) -> dict[str, Any]:
batch_size = max(1, int(settings.POLL_DUE_DISPATCH_BATCH_SIZE or 100))
poll_lease_expired_at = current_time - timedelta(seconds=int(settings.POLL_TASK_LEASE_SECONDS or 300))
await apply_short_lock_timeout(db)
query_result = await db.execute(
select(ChatGenerationTask)
.where(
@@ -846,6 +926,7 @@ async def dispatch_due_poll_tasks_once(db: AsyncSession) -> dict[str, Any]:
dispatched_task_ids: list[str] = []
dispatched_due_next_poll_at_by_id: dict[str, datetime | None] = {}
dispatched_queue_hold_until_by_id: dict[str, datetime] = {}
post_commit_logs: list[dict[str, Any]] = []
for task in tasks:
action = "skip_unknown"
@@ -863,13 +944,14 @@ async def dispatch_due_poll_tasks_once(db: AsyncSession) -> dict[str, Any]:
continue
if not (task.seedance_task_id or task.provider_task_id):
# dispatcher 不负责重新 create;没有 provider id 的异常状态交给启动容灾或 create 任务处理。
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.POLL_DISPATCH_SKIP.value,
message="视频到期轮询调度跳过:缺少外部任务ID",
detail={"pipeline_stage": task.pipeline_stage, "next_poll_at": task.next_poll_at},
)
# Defer FK-backed event logging until the batch row locks are released.
post_commit_logs.append({
"task_id": str(task.id),
"generation_attempt_no": int(task.generation_attempt_no or 1),
"event_type": ChatGenerationTaskEventType.POLL_DISPATCH_SKIP.value,
"message": "视频到期轮询调度跳过:缺少外部任务ID",
"detail": {"pipeline_stage": task.pipeline_stage, "next_poll_at": task.next_poll_at},
})
action = "skip_no_provider_task_id"
continue
@@ -886,16 +968,28 @@ async def dispatch_due_poll_tasks_once(db: AsyncSession) -> dict[str, Any]:
except Exception as exc:
logger.exception("视频到期轮询调度单条处理失败。task_id=%s", getattr(task, "id", None))
action = "error"
await log_task_event(
task,
event_type=ChatGenerationTaskEventType.POLL_DISPATCH_SKIP.value,
message=f"视频到期轮询调度单条处理失败:{exc}",
)
post_commit_logs.append({
"task_id": str(getattr(task, "id", "") or ""),
"generation_attempt_no": int(getattr(task, "generation_attempt_no", 1) or 1),
"event_type": ChatGenerationTaskEventType.POLL_DISPATCH_SKIP.value,
"message": f"视频到期轮询调度单条处理失败:{exc}",
"detail": None,
})
finally:
results[action] = results.get(action, 0) + 1
await db.commit()
for item in post_commit_logs:
if item["task_id"]:
await log_task_event(
task_id=item["task_id"],
generation_attempt_no=item["generation_attempt_no"],
event_type=item["event_type"],
message=item["message"],
detail=item["detail"],
)
fresh_tasks = []
if dispatched_task_ids:
fresh_result = await db.execute(
@@ -931,7 +1025,7 @@ async def dispatch_due_poll_tasks_once(db: AsyncSession) -> dict[str, Any]:
)
poll_generation_task.apply_async(
args=[task.id],
kwargs={"force_due": True},
kwargs={"force_due": True, "owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": int(task.generation_attempt_no or 1)},
queue=POLL_QUEUE,
countdown=0,
)
@@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.chat_generation_task import ChatGenerationTask
from app.models.credit_record import CreditRecord
from app.models.generation_record import GenerationRecord
from app.services.generation.pipeline.db_lock_service import execute_with_lock_timeout
from app.services.credits import refund_credits
from app.services.credit_record_meta_service import build_refund_meta_from_charge
from app.services.generation.billing_service import (
@@ -25,46 +26,52 @@ def _round2(value: float | int | None) -> float:
return round(float(value or 0), 2)
async def _has_refund_for_biz_key(db: AsyncSession, *, user_id: str, charge_biz_key: str) -> bool:
result = await db.execute(
select(CreditRecord.id)
.where(
CreditRecord.user_id == user_id,
CreditRecord.type == "refund",
CreditRecord.refund_for_biz_key == charge_biz_key,
)
.limit(1)
)
return result.scalar_one_or_none() is not None
async def _find_unrefunded_media_charges(
db: AsyncSession,
*,
user_id: str,
owner_type: str,
owner_id: str,
generation_attempt_no: int | None = None,
) -> list[CreditRecord]:
"""查找当前任务下所有未退款的媒体生成扣费流水"""
pattern = f"{owner_type}:{owner_id}:attempt:%:{CHARGE_MEDIA}:charge"
"""批量查找指定任务、指定生成轮次下尚未退款的媒体扣费"""
result = await db.execute(
select(CreditRecord)
.where(
CreditRecord.user_id == user_id,
CreditRecord.related_id == owner_id,
CreditRecord.type == "consume",
CreditRecord.biz_key.like(pattern),
CreditRecord.biz_key.is_not(None),
)
.order_by(CreditRecord.created_at.asc())
)
charges = list(result.scalars().all())
unrefunded: list[CreditRecord] = []
for charge in charges:
if not charge.biz_key:
charges: list[CreditRecord] = []
for charge in result.scalars().all():
parsed = parse_credit_biz_key(charge.biz_key)
if not parsed:
continue
if not await _has_refund_for_biz_key(db, user_id=user_id, charge_biz_key=charge.biz_key):
unrefunded.append(charge)
return unrefunded
if parsed.get("owner_type") != owner_type or parsed.get("owner_id") != owner_id:
continue
if parsed.get("charge_kind") != CHARGE_MEDIA or parsed.get("action") != "charge":
continue
if generation_attempt_no is not None and int(parsed.get("attempt_no") or 0) != int(generation_attempt_no):
continue
charges.append(charge)
charge_keys = [str(charge.biz_key) for charge in charges if charge.biz_key]
if not charge_keys:
return []
refund_result = await db.execute(
select(CreditRecord.refund_for_biz_key).where(
CreditRecord.user_id == user_id,
CreditRecord.type == "refund",
CreditRecord.refund_for_biz_key.in_(charge_keys),
)
)
refunded_keys = {str(value) for (value,) in refund_result.all() if value}
return [charge for charge in charges if charge.biz_key not in refunded_keys]
async def refund_unrefunded_media_charges(
@@ -74,8 +81,9 @@ async def refund_unrefunded_media_charges(
owner_type: str,
owner_id: str,
description_prefix: str,
generation_attempt_no: int | None = None,
) -> float:
"""回退当前任务所有未退款媒体扣费流水。
"""回退当前生成轮次尚未退款媒体扣费流水。
容灾考虑:
- 不依赖 retry_count 推断当前轮次。
@@ -88,6 +96,7 @@ async def refund_unrefunded_media_charges(
user_id=user_id,
owner_type=owner_type,
owner_id=owner_id,
generation_attempt_no=generation_attempt_no,
)
for charge in charges:
parsed = parse_credit_biz_key(charge.biz_key)
@@ -124,6 +133,7 @@ async def mark_generation_record_failed_and_refund_once(
record_id: str | None = None,
record: GenerationRecord | None = None,
error_message: str | None = None,
generation_attempt_no: int | None = None,
) -> GenerationRecord | None:
"""把 GenerationRecord 标记为最终失败并幂等退回媒体生成积分。
@@ -132,7 +142,9 @@ async def mark_generation_record_failed_and_refund_once(
if record is None:
if not record_id:
return None
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(GenerationRecord)
.where(GenerationRecord.id == record_id, GenerationRecord.deleted_at.is_(None))
.with_for_update()
@@ -155,6 +167,7 @@ async def mark_generation_record_failed_and_refund_once(
owner_type=OWNER_GENERATION_RECORD,
owner_id=record.id,
description_prefix="生成记录",
generation_attempt_no=generation_attempt_no or int(getattr(record, "generation_attempt_no", 1) or 1),
)
if refunded_amount > 0:
# GenerationRecord.credits_cost 只代表视频/图片生成媒体积分。
@@ -171,12 +184,15 @@ async def mark_chat_generation_task_failed_and_refund_once(
task: ChatGenerationTask | None = None,
error_message: str | None = None,
pipeline_stage: str = "failed",
generation_attempt_no: int | None = None,
) -> ChatGenerationTask | None:
"""把 ChatGenerationTask 标记为最终失败并幂等退回媒体生成积分。"""
if task is None:
if not task_id:
return None
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == task_id,
@@ -204,6 +220,7 @@ async def mark_chat_generation_task_failed_and_refund_once(
owner_type=OWNER_CHAT_GENERATION_TASK,
owner_id=task.id,
description_prefix="任务生成",
generation_attempt_no=generation_attempt_no or int(getattr(task, "generation_attempt_no", 1) or 1),
)
if refunded_amount > 0:
@@ -132,6 +132,9 @@ async def create_chat_generation_task_for_module(
snapshot["generation_count"] = 1
task = ChatGenerationTask(
id=task_id,
created_at=now,
resource_generation_started_at=now,
generation_attempt_no=1,
user_id=current_user.id,
original_prompt=original_prompt,
optimized_prompt=optimized_prompt,
@@ -195,6 +198,9 @@ async def create_chat_generation_task_for_module(
snapshot["video_upscale_enabled_snapshot"] = upscale_enabled_snapshot
task = ChatGenerationTask(
id=task_id,
created_at=now,
resource_generation_started_at=now,
generation_attempt_no=1,
user_id=current_user.id,
original_prompt=original_prompt,
optimized_prompt=optimized_prompt,
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import json
from datetime import datetime
from typing import Any
@@ -42,6 +43,11 @@ from app.services.generation.ai.engine_service import (
)
from app.services.generation.billing_service import charge_module_prompt_usage
from app.services.generation.refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.generation.pipeline.db_lock_service import (
DatabaseRowLockBusy,
apply_short_lock_timeout,
execute_with_lock_timeout,
)
from app.services.generation.task_factory_service import create_chat_generation_task_for_module
from app.services.hot_opening_video_prompt_service import build_final_video_prompt, optimize_hot_opening_video_prompt, patch_video_prompt_schema_from_client
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
@@ -328,6 +334,18 @@ async def _soft_delete_steps_from_index(
start_index: int,
deleted_at: datetime | None = None,
) -> None:
processing_result = await db.execute(
select(ModuleGenerationStep.id).where(
ModuleGenerationStep.project_id == project.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.deleted_at.is_(None),
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.status == ModuleStepStatusEnum.PROCESSING.value,
ModuleGenerationStep.step_index >= start_index,
).limit(1)
)
if processing_result.scalar_one_or_none() is not None:
raise HTTPException(status_code=409, detail="当前步骤正在处理中,请等待完成后再操作")
await _base_soft_delete_steps_from_index(
db,
project=project,
@@ -750,6 +768,69 @@ async def update_hot_opening_video_prompt_schema(
)
async def _reload_prompt_context_for_update(
db: AsyncSession,
*,
project_id: str,
step_id: str,
step_code: str,
) -> tuple[ModuleGenerationProject | None, ModuleGenerationStep | None]:
last_error: DatabaseRowLockBusy | None = None
for retry_index in range(3):
try:
project_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id == project_id,
ModuleGenerationProject.module == MODULE,
ModuleGenerationProject.deleted_at.is_(None),
)
.with_for_update()
.execution_options(populate_existing=True)
.limit(1)
)
project = project_result.scalar_one_or_none()
if project is None:
return None, None
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id,
ModuleGenerationStep.project_id == project_id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.step_code == step_code,
ModuleGenerationStep.deleted_at.is_(None),
ModuleGenerationStep.is_current == True,
)
.with_for_update()
.execution_options(populate_existing=True)
.limit(1)
)
return project, step_result.scalar_one_or_none()
except DatabaseRowLockBusy as exc:
last_error = exc
await db.rollback()
if retry_index < 2:
await asyncio.sleep(1 + retry_index)
raise last_error or DatabaseRowLockBusy()
def _prompt_context_matches(
step: ModuleGenerationStep | None,
*,
expected_version: int,
expected_input_json: str,
) -> bool:
if step is None or step.status != ModuleStepStatusEnum.PROCESSING.value:
return False
if int(step.version or 1) != int(expected_version):
return False
current_input = json.dumps(step.input_json, ensure_ascii=False, sort_keys=True, default=str)
return current_input == expected_input_json
async def submit_image_prompt_optimize(
db: AsyncSession,
*,
@@ -782,6 +863,7 @@ async def submit_image_prompt_optimize(
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
.where(ModuleGenerationProject.id == project_id, ModuleGenerationProject.module == MODULE, ModuleGenerationProject.deleted_at.is_(None))
@@ -799,6 +881,7 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
return None
if step_id:
await apply_short_lock_timeout(db)
result = await db.execute(
select(ModuleGenerationStep)
.where(
@@ -845,24 +928,45 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
{"type": "video", "url": material.get("material_video_url"), "name": "参考素材视频"},
{"type": "image", "url": material.get("material_image_url"), "name": "新产品图片"},
]
project_id_value = str(project.id)
step_id_value = str(step.id)
user_id_value = str(project.user_id)
module_value = str(project.module)
expected_step_version = int(step.version or 1)
expected_input_json = json.dumps(step.input_json, ensure_ascii=False, sort_keys=True, default=str)
await db.commit()
try:
request_log = {"original_prompt": prompt_text, "references": references, "gen_type": "image"}
log_module_prompt_event(
event_type="module_prompt_request",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
module=project.module,
project_id=project_id_value,
step_id=step_id_value,
user_id=user_id_value,
module=module_value,
prompt_type=ModulePromptTypeEnum.IMAGE_PROMPT.value,
request=request_log,
)
optimized, token_usage = await optimize_prompt(
db,
original_prompt=prompt_text,
user_id=project.user_id,
user_id=user_id_value,
references=references,
gen_type="image",
)
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
billing = await charge_module_prompt_usage(
db,
user_id=project.user_id,
@@ -907,7 +1011,25 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
token_usage=usage,
)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.IMAGE_PROMPT_SUCCESS.value, message="图片 AI 提词生成成功")
await db.commit()
except DatabaseRowLockBusy:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
step.status = ModuleStepStatusEnum.FAILED.value
step.error_message = str(exc)
step.completed_at = _now()
@@ -925,6 +1047,7 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
)
_log_project_error(project=project, step=step, event_type="IMAGE_PROMPT_FAILED", message=project.error_message, exc=exc)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.IMAGE_PROMPT_FAILED.value, message=project.error_message)
await db.commit()
return step
@@ -1093,6 +1216,7 @@ async def submit_video_prompt_optimize(
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
.where(ModuleGenerationProject.id == project_id, ModuleGenerationProject.module == MODULE, ModuleGenerationProject.deleted_at.is_(None))
@@ -1112,6 +1236,7 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
return None
if step_id:
await apply_short_lock_timeout(db)
result = await db.execute(
select(ModuleGenerationStep)
.where(
@@ -1151,8 +1276,17 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step.error_message = "缺少新项目图片结果,不能生成视频提词"
project.status = ModuleProjectStatusEnum.FAILED.value
project.error_message = step.error_message
await db.commit()
return step
project_id_value = str(project.id)
step_id_value = str(step.id)
user_id_value = str(project.user_id)
module_value = str(project.module)
expected_step_version = int(step.version or 1)
expected_input_json = json.dumps(step.input_json, ensure_ascii=False, sort_keys=True, default=str)
await db.commit()
try:
request_log = {
"source_project_name": material.get("source_project_name") or "",
@@ -1165,10 +1299,10 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
}
log_module_prompt_event(
event_type="module_prompt_request",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
module=project.module,
project_id=project_id_value,
step_id=step_id_value,
user_id=user_id_value,
module=module_value,
prompt_type=ModulePromptTypeEnum.VIDEO_PROMPT.value,
request=request_log,
)
@@ -1176,7 +1310,7 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
request_log["schema_config_source"] = schema_config_snapshot.get("source")
prompt_schema, final_prompt, token_usage = await optimize_hot_opening_video_prompt(
db,
user_id=project.user_id,
user_id=user_id_value,
source_project_name=request_log["source_project_name"],
target_project_name=request_log["target_project_name"],
core_content_point=request_log["core_content_point"],
@@ -1185,11 +1319,24 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
video_config=video_config,
target_platform=target_platform,
schema_config_snapshot=schema_config_snapshot,
module=project.module,
project_id=project.id,
step_id=step.id,
trace_id=f"hot-video-prompt:{step.id}",
module=module_value,
project_id=project_id_value,
step_id=step_id_value,
trace_id=f"hot-video-prompt:{step_id_value}",
)
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
billing = await charge_module_prompt_usage(
db,
user_id=project.user_id,
@@ -1237,7 +1384,25 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
token_usage=usage,
)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.VIDEO_PROMPT_SUCCESS.value, message="视频 AI 提词生成成功")
await db.commit()
except DatabaseRowLockBusy:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
step.status = ModuleStepStatusEnum.FAILED.value
step.error_message = str(exc)
step.completed_at = _now()
@@ -1255,6 +1420,7 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
)
_log_project_error(project=project, step=step, event_type="VIDEO_PROMPT_FAILED", message=project.error_message, exc=exc)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.VIDEO_PROMPT_FAILED.value, message=project.error_message)
await db.commit()
return step
@@ -1366,9 +1532,37 @@ async def generate_video_from_prompt(
async def handle_chat_generation_task_completed(db: AsyncSession, task: ChatGenerationTask) -> None:
if not task or task.generation_mode != GENERATION_MODE:
return
result = await db.execute(
meta_result = await db.execute(
select(ModuleGenerationStep.id, ModuleGenerationStep.project_id).where(
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.deleted_at.is_(None),
).limit(1)
)
meta = meta_result.first()
if not meta:
return
step_id_value, project_id_value = str(meta.id), str(meta.project_id)
project_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id == project_id_value,
ModuleGenerationProject.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
project = project_result.scalar_one_or_none()
if not project:
return
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id_value,
ModuleGenerationStep.project_id == project_id_value,
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
@@ -1377,18 +1571,9 @@ async def handle_chat_generation_task_completed(db: AsyncSession, task: ChatGene
.with_for_update()
.limit(1)
)
step = result.scalar_one_or_none()
step = step_result.scalar_one_or_none()
if not step:
return
project_result = await db.execute(
select(ModuleGenerationProject)
.where(ModuleGenerationProject.id == step.project_id, ModuleGenerationProject.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
project = project_result.scalar_one_or_none()
if not project:
return
if step.step_code == HotOpeningStepCodeEnum.IMAGE_GENERATE.value:
step.status = ModuleStepStatusEnum.COMPLETED.value
@@ -1429,19 +1614,48 @@ async def handle_chat_generation_task_completed(db: AsyncSession, task: ChatGene
async def handle_chat_generation_task_failed(db: AsyncSession, task: ChatGenerationTask) -> None:
if not task or task.generation_mode != GENERATION_MODE:
return
result = await db.execute(
select(ModuleGenerationStep)
.where(ModuleGenerationStep.chat_task_id == task.id, ModuleGenerationStep.module == MODULE, ModuleGenerationStep.is_current == True, ModuleGenerationStep.deleted_at.is_(None))
meta_result = await db.execute(
select(ModuleGenerationStep.id, ModuleGenerationStep.project_id).where(
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.deleted_at.is_(None),
).limit(1)
)
meta = meta_result.first()
if not meta:
return
step_id_value, project_id_value = str(meta.id), str(meta.project_id)
project_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id == project_id_value,
ModuleGenerationProject.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
step = result.scalar_one_or_none()
if not step:
return
project_result = await db.execute(select(ModuleGenerationProject).where(ModuleGenerationProject.id == step.project_id).with_for_update().limit(1))
project = project_result.scalar_one_or_none()
if not project:
return
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id_value,
ModuleGenerationStep.project_id == project_id_value,
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
step = step_result.scalar_one_or_none()
if not step:
return
step.status = ModuleStepStatusEnum.FAILED.value
step.error_message = task.error_message
step.completed_at = _now()
@@ -1459,7 +1673,8 @@ async def mark_hot_opening_step_dispatch_failed(
error_message: str,
) -> None:
project = await _get_project_for_user(db, project_id=project_id, user=current_user, for_update=True)
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id,
@@ -1515,7 +1515,7 @@ def _log_video_prompt_ai_event(
)
async def _select_model_config(db: AsyncSession) -> ModelConfig | None:
result = await db.execute(select(ModelConfig).where(ModelConfig.is_active == True).order_by(ModelConfig.priority.desc()).limit(1))
result = await db.execute(select(ModelConfig).where(ModelConfig.is_active == True, ModelConfig.deleted_at.is_(None)).order_by(ModelConfig.priority.desc()).limit(1))
return result.scalar_one_or_none()
@@ -1557,6 +1557,9 @@ async def optimize_hot_opening_video_prompt(
# return result, build_final_video_prompt(result), {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
config = await _select_model_config(db)
# All module/project claims are committed by the caller. Release this
# configuration read transaction before the remote model request.
await db.commit()
if not config:
result = normalize_video_prompt_schema_from_ai(_mock_result(video_config, target_platform), video_config, schema_config_snapshot)
return result, build_final_video_prompt(result), {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
+13 -2
View File
@@ -107,7 +107,7 @@ def _log_image_response(record_id: str, response_data: dict, error: str | None =
async def get_active_image_engine(db: AsyncSession) -> ImageEngine:
result = await db.execute(
select(ImageEngine)
.where(ImageEngine.is_active == True)
.where(ImageEngine.is_active == True, ImageEngine.deleted_at.is_(None))
.order_by(ImageEngine.priority.desc())
.limit(1)
)
@@ -455,12 +455,23 @@ async def poll_image_task_status(engine: ImageEngine, task_id: str) -> dict:
}
async def download_image(image_url: str, dest_path: str) -> str:
async def download_image(
image_url: str,
dest_path: str,
*,
execution_guard=None,
) -> str:
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
async with httpx.AsyncClient(timeout=300) as client:
async with client.stream("GET", image_url) as response:
response.raise_for_status()
with open(dest_path, "wb") as file:
chunk_no = 0
async for chunk in response.aiter_bytes(chunk_size=8192):
file.write(chunk)
chunk_no += 1
if execution_guard is not None and chunk_no % 32 == 0:
await execution_guard()
if execution_guard is not None:
await execution_guard()
return dest_path
+5 -1
View File
@@ -102,10 +102,13 @@ async def optimize_prompt(
result = await db.execute(
select(ModelConfig)
.where(ModelConfig.is_active == True)
.where(ModelConfig.is_active == True, ModelConfig.deleted_at.is_(None))
.order_by(ModelConfig.priority.desc())
)
configs = list(result.scalars().all())
# Release the read transaction before the external LLM request. Callers
# must commit their business claim before invoking optimize_prompt.
await db.commit()
if configs:
# 按 priority 从大到小依次尝试,跳过 mock,失败则用下一个
@@ -178,6 +181,7 @@ async def _call_openai_compatible(
break
except (json.JSONDecodeError, TypeError):
pass
await db.commit()
if not system_prompt:
if gen_type == "image":
@@ -85,6 +85,7 @@ async def _find_latest_media_charge(
owner_type: str,
owner_id: str,
media_type: str | None,
attempt_no: int | None = None,
) -> CreditRecord | None:
query = (
select(CreditRecord)
@@ -97,6 +98,8 @@ async def _find_latest_media_charge(
)
if media_type:
query = query.where(CreditRecord.media_type == media_type)
if attempt_no is not None:
query = query.where(CreditRecord.attempt_no == int(attempt_no))
query = query.order_by(CreditRecord.attempt_no.desc().nullslast(), CreditRecord.created_at.desc()).limit(1)
result = await db.execute(query)
return result.scalar_one_or_none()
@@ -213,6 +216,7 @@ async def sync_chat_generation_task_media_token_snapshot(
owner_type=CreditRecordOwnerType.CHAT_GENERATION_TASK.value,
owner_id=task.id,
media_type=gen_type or None,
attempt_no=int(getattr(task, "generation_attempt_no", 1) or 1),
)
return await _sync_charge_snapshot(
db,
@@ -243,6 +247,7 @@ async def sync_generation_record_media_token_snapshot(
owner_type=CreditRecordOwnerType.GENERATION_RECORD.value,
owner_id=record.id,
media_type=gen_type or None,
attempt_no=int(getattr(record, "generation_attempt_no", 1) or 1),
)
return await _sync_charge_snapshot(
db,
@@ -16,6 +16,11 @@ from app.models.module_generation_step import ModuleGenerationStep
from app.models.user import User
from app.enums.module_generation_flow import ModuleGenerationFlowConfig
from app.services.module_generation_step_common_service import build_step_input, build_step_output, utc_now
from app.services.generation.pipeline.db_lock_service import (
apply_short_lock_timeout,
execute_with_lock_timeout,
raise_if_database_lock_busy,
)
from app.services.resource_accounting_service import (
SOURCE_MODEL_CHAT_TASK,
soft_delete_resources_by_source,
@@ -44,8 +49,13 @@ async def get_project_for_user(
if populate_existing:
query = query.execution_options(populate_existing=True)
if for_update:
await apply_short_lock_timeout(db)
query = query.with_for_update()
result = await db.execute(query.limit(1))
try:
result = await db.execute(query.limit(1))
except Exception as exc:
raise_if_database_lock_busy(exc)
raise
project = result.scalar_one_or_none()
if not project:
raise HTTPException(status_code=404, detail=config.project_not_found_message)
@@ -72,8 +82,13 @@ async def get_step_for_user(
if not user.is_admin:
query = query.where(ModuleGenerationStep.user_id == user.id)
if for_update:
await apply_short_lock_timeout(db)
query = query.with_for_update()
result = await db.execute(query.limit(1))
try:
result = await db.execute(query.limit(1))
except Exception as exc:
raise_if_database_lock_busy(exc)
raise
step = result.scalar_one_or_none()
if not step:
raise HTTPException(status_code=404, detail=config.step_not_found_message)
@@ -278,7 +293,9 @@ async def load_chat_tasks_for_steps(
)
if for_update:
stmt = stmt.with_for_update()
result = await db.execute(stmt)
result = await execute_with_lock_timeout(db, stmt)
else:
result = await db.execute(stmt)
return {task.id: task for task in result.scalars().all()}
@@ -314,7 +331,8 @@ async def assert_project_has_no_active_chat_tasks(
config: ModuleGenerationFlowConfig,
detail_message: str = "当前存在生成中任务,请等待生成完成或失败后再操作",
) -> dict[str, ChatGenerationTask]:
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.project_id == project.id,
@@ -353,7 +371,8 @@ async def soft_delete_steps_from_index(
- 已完成任务只做软删任务与 generated_resources释放容量统计失败任务只软删任务
"""
deleted_at = deleted_at or utc_now()
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.project_id == project.id,
@@ -49,7 +49,7 @@ def _mask_string(value: str) -> str:
def _is_sensitive_key(key: str) -> bool:
lower = str(key).replace("-", "_").lower()
return any(pattern in lower for pattern in SENSITIVE_KEY_PATTERNS)
return lower == "sign" or any(pattern in lower for pattern in SENSITIVE_KEY_PATTERNS)
def _sanitize_url(value: str) -> str:
@@ -8,6 +8,7 @@ import logging
import os
import threading
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Optional, Union
@@ -24,6 +25,40 @@ logger = logging.getLogger("video_gen")
_redis_clients: Dict[tuple[int, int, int], Any] = {}
class RedisExecutionLockError(RuntimeError):
"""Redis execution-lock infrastructure error.
Execution locks are fail-closed: callers must stop the current Celery task
and retry later instead of falling back to an unlocked database path.
"""
class RedisExecutionLockUnavailable(RedisExecutionLockError):
"""Redis is unavailable, so execution ownership cannot be established."""
class RedisExecutionLockLost(RedisExecutionLockError):
"""The current worker no longer owns the execution lock."""
_RELEASE_LOCK_SCRIPT = """
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('del', KEYS[1])
else
return 0
end
"""
_RENEW_LOCK_SCRIPT = """
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('pexpire', KEYS[1], ARGV[2])
else
return 0
end
"""
def utc_now() -> datetime:
return datetime.now(timezone.utc)
@@ -73,8 +108,9 @@ async def get_registry_redis() -> Optional[Any]:
Redis 客户端会触发 got Future attached to a different loop
因此这里按 pid + thread_id + event_loop_id 缓存客户端确保同一个客户端
只在创建它的事件循环里使用Redis 不可用时返回 None调用方降级为
DB fallback不能影响生成主链路
只在创建它的事件循环里使用普通 active 注册表在 Redis 不可用时返回
None执行锁封装会把 None 转为 RedisExecutionLockUnavailable严格中止
当前任务不允许无锁执行外部副作用
"""
redis_url = registry_redis_url()
if not _is_supported_redis_url(redis_url):
@@ -328,6 +364,212 @@ async def redis_acquire_lock(
return None
async def redis_acquire_execution_lock(
*,
lock_key: str,
ttl_seconds: int,
token: Optional[str] = None,
log_context: str = "execution_lock",
) -> Optional[str]:
"""Fail-closed execution lock.
Returns a token when acquired and ``None`` when another worker owns the
lock. Redis connection/command failures raise
:class:`RedisExecutionLockUnavailable`; callers must retry the Celery task
and must not execute the external side effect without a lock.
"""
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis execution lock unavailable: context={log_context}, key={lock_key}"
)
lock_token = token or uuid.uuid4().hex
ttl_ms = max(1000, int(ttl_seconds or 60) * 1000)
try:
acquired = await redis.set(lock_key, lock_token, nx=True, px=ttl_ms)
return lock_token if acquired else None
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.error(
"获取 Redis 执行锁失败。context=%s, lock_key=%s, error=%s",
log_context,
lock_key,
exc,
)
raise RedisExecutionLockUnavailable(
f"Redis execution lock acquire failed: {lock_key}: {exc}"
) from exc
async def redis_check_lock_owner(
*,
lock_key: str,
token: str,
log_context: str = "execution_lock",
) -> bool:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis execution lock unavailable: context={log_context}, key={lock_key}"
)
try:
value = await redis.get(lock_key)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.error(
"检查 Redis 执行锁失败。context=%s, lock_key=%s, error=%s",
log_context,
lock_key,
exc,
)
raise RedisExecutionLockUnavailable(
f"Redis execution lock check failed: {lock_key}: {exc}"
) from exc
return bool(value and str(value) == str(token))
async def redis_renew_lock(
*,
lock_key: str,
token: str,
ttl_seconds: int,
log_context: str = "execution_lock",
) -> bool:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis execution lock unavailable: context={log_context}, key={lock_key}"
)
ttl_ms = max(1000, int(ttl_seconds or 60) * 1000)
try:
renewed = await redis.eval(_RENEW_LOCK_SCRIPT, 1, lock_key, token, ttl_ms)
return bool(renewed)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.error(
"续期 Redis 执行锁失败。context=%s, lock_key=%s, error=%s",
log_context,
lock_key,
exc,
)
raise RedisExecutionLockUnavailable(
f"Redis execution lock renew failed: {lock_key}: {exc}"
) from exc
@dataclass(slots=True)
class RedisExecutionLockLease:
"""Owned Redis execution lock with compare-and-expire heartbeat."""
lock_key: str
token: str
ttl_seconds: int
log_context: str = "execution_lock"
renew_interval_seconds: int | None = None
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
@classmethod
async def acquire(
cls,
*,
lock_key: str,
ttl_seconds: int,
token: str | None = None,
log_context: str = "execution_lock",
renew_interval_seconds: int | None = None,
) -> "RedisExecutionLockLease | None":
acquired_token = await redis_acquire_execution_lock(
lock_key=lock_key,
ttl_seconds=ttl_seconds,
token=token,
log_context=log_context,
)
if not acquired_token:
return None
lease = cls(
lock_key=lock_key,
token=acquired_token,
ttl_seconds=max(1, int(ttl_seconds or 60)),
log_context=log_context,
renew_interval_seconds=renew_interval_seconds,
)
lease.start_heartbeat()
return lease
def start_heartbeat(self) -> None:
if self._heartbeat_task is not None:
return
self._heartbeat_task = asyncio.create_task(self._heartbeat())
async def _heartbeat(self) -> None:
interval = int(
self.renew_interval_seconds
or max(1, min(60, self.ttl_seconds // 3))
)
while not self._stop_event.is_set():
try:
await asyncio.wait_for(self._stop_event.wait(), timeout=interval)
return
except asyncio.TimeoutError:
pass
try:
renewed = await redis_renew_lock(
lock_key=self.lock_key,
token=self.token,
ttl_seconds=self.ttl_seconds,
log_context=self.log_context,
)
if not renewed:
self._lost_error = RedisExecutionLockLost(
f"Redis execution lock ownership lost: {self.lock_key}"
)
return
except RedisExecutionLockError as exc:
self._lost_error = exc
return
async def ensure_owned(self) -> None:
if self._lost_error is not None:
raise self._lost_error
owned = await redis_check_lock_owner(
lock_key=self.lock_key,
token=self.token,
log_context=self.log_context,
)
if not owned:
self._lost_error = RedisExecutionLockLost(
f"Redis execution lock ownership lost: {self.lock_key}"
)
raise self._lost_error
async def close(self) -> None:
self._stop_event.set()
heartbeat = self._heartbeat_task
if heartbeat is not None:
try:
await heartbeat
except Exception:
logger.debug(
"Redis execution lock heartbeat close failed. context=%s key=%s",
self.log_context,
self.lock_key,
exc_info=True,
)
try:
await redis_release_lock(
lock_key=self.lock_key,
token=self.token,
log_context=self.log_context,
)
except Exception:
logger.debug(
"Redis execution lock release failed. context=%s key=%s",
self.log_context,
self.lock_key,
exc_info=True,
)
async def redis_release_lock(
*,
lock_key: str,
@@ -338,16 +580,8 @@ async def redis_release_lock(
redis = await get_registry_redis()
if redis is None:
return False
script = """
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('del', KEYS[1])
else
return 0
end
"""
try:
released = await redis.eval(script, 1, lock_key, token)
released = await redis.eval(_RELEASE_LOCK_SCRIPT, 1, lock_key, token)
return bool(released)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.warning("释放 Redis 锁失败。context=%s, lock_key=%s, error=%s", log_context, lock_key, exc)
@@ -317,20 +317,29 @@ async def record_generation_record_generated_resource(
remote_url: str | None = None,
generated_at: datetime | None = None,
) -> ResourceAccountingResult:
snapshot = _parse_json(getattr(record, "engine_snapshot_json", None))
return await record_generated_resource(
db,
user_id=record.user_id,
resource_type=record.gen_type,
resource_url=resource_url,
remote_url=remote_url,
remote_url=remote_url or getattr(record, "remote_result_url", None),
storage_type="local" if storage_path else "remote",
storage_path=storage_path,
file_size_bytes=file_size_bytes,
source_model=SOURCE_MODEL_GENERATION_RECORD,
source_model_module="app.models.generation_record",
source_id=record.id,
engine_id=getattr(record, "engine_id", None),
engine_type=snapshot.get("engine_type") or record.gen_type,
provider=snapshot.get("provider"),
model_name=snapshot.get("model_name"),
generated_at=generated_at or record.generated_at or datetime.now(timezone.utc),
extra={"project_id": record.project_id},
extra={
"project_id": record.project_id,
"generation_attempt_no": int(getattr(record, "generation_attempt_no", 1) or 1),
"pipeline_stage": record.pipeline_stage,
},
)
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import json
from datetime import datetime
from typing import Any
@@ -42,6 +43,11 @@ from app.services.generation.ai.engine_service import (
)
from app.services.generation.billing_service import charge_module_prompt_usage
from app.services.generation.refund_service import mark_chat_generation_task_failed_and_refund_once
from app.services.generation.pipeline.db_lock_service import (
DatabaseRowLockBusy,
apply_short_lock_timeout,
execute_with_lock_timeout,
)
from app.services.generation.task_factory_service import create_chat_generation_task_for_module
from app.services.hot_opening_video_prompt_service import (
build_final_video_prompt,
@@ -338,6 +344,18 @@ async def _soft_delete_steps_from_index(
refund_unfinished: bool = False,
release_stats: dict[str, int] | None = None,
) -> None:
processing_result = await db.execute(
select(ModuleGenerationStep.id).where(
ModuleGenerationStep.project_id == project.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.deleted_at.is_(None),
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.status == ModuleStepStatusEnum.PROCESSING.value,
ModuleGenerationStep.step_index >= start_index,
).limit(1)
)
if processing_result.scalar_one_or_none() is not None:
raise HTTPException(status_code=409, detail="当前步骤正在处理中,请等待完成后再操作")
await _base_soft_delete_steps_from_index(
db,
project=project,
@@ -693,6 +711,69 @@ async def update_shot_replicate_video_prompt_schema(
)
async def _reload_prompt_context_for_update(
db: AsyncSession,
*,
project_id: str,
step_id: str,
step_code: str,
) -> tuple[ModuleGenerationProject | None, ModuleGenerationStep | None]:
last_error: DatabaseRowLockBusy | None = None
for retry_index in range(3):
try:
project_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id == project_id,
ModuleGenerationProject.module == MODULE,
ModuleGenerationProject.deleted_at.is_(None),
)
.with_for_update()
.execution_options(populate_existing=True)
.limit(1)
)
project = project_result.scalar_one_or_none()
if project is None:
return None, None
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id,
ModuleGenerationStep.project_id == project_id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.step_code == step_code,
ModuleGenerationStep.deleted_at.is_(None),
ModuleGenerationStep.is_current == True,
)
.with_for_update()
.execution_options(populate_existing=True)
.limit(1)
)
return project, step_result.scalar_one_or_none()
except DatabaseRowLockBusy as exc:
last_error = exc
await db.rollback()
if retry_index < 2:
await asyncio.sleep(1 + retry_index)
raise last_error or DatabaseRowLockBusy()
def _prompt_context_matches(
step: ModuleGenerationStep | None,
*,
expected_version: int,
expected_input_json: str,
) -> bool:
if step is None or step.status != ModuleStepStatusEnum.PROCESSING.value:
return False
if int(step.version or 1) != int(expected_version):
return False
current_input = json.dumps(step.input_json, ensure_ascii=False, sort_keys=True, default=str)
return current_input == expected_input_json
async def submit_image_prompt_optimize(
db: AsyncSession,
*,
@@ -732,6 +813,7 @@ async def submit_image_prompt_optimize(
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
.where(ModuleGenerationProject.id == project_id, ModuleGenerationProject.module == MODULE, ModuleGenerationProject.deleted_at.is_(None))
@@ -749,6 +831,7 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
return None
if step_id:
await apply_short_lock_timeout(db)
result = await db.execute(
select(ModuleGenerationStep)
.where(
@@ -795,24 +878,45 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
{"type": "video", "url": material.get("material_video_url"), "name": "参考素材视频"},
{"type": "image", "url": material.get("material_image_url"), "name": "新产品图片"},
]
project_id_value = str(project.id)
step_id_value = str(step.id)
user_id_value = str(project.user_id)
module_value = str(project.module)
expected_step_version = int(step.version or 1)
expected_input_json = json.dumps(step.input_json, ensure_ascii=False, sort_keys=True, default=str)
await db.commit()
try:
request_log = {"original_prompt": prompt_text, "references": references, "gen_type": "image"}
log_module_prompt_event(
event_type="module_prompt_request",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
module=project.module,
project_id=project_id_value,
step_id=step_id_value,
user_id=user_id_value,
module=module_value,
prompt_type=ModulePromptTypeEnum.IMAGE_PROMPT.value,
request=request_log,
)
optimized, token_usage = await optimize_prompt(
db,
original_prompt=prompt_text,
user_id=project.user_id,
user_id=user_id_value,
references=references,
gen_type="image",
)
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
billing = await charge_module_prompt_usage(
db,
user_id=project.user_id,
@@ -857,7 +961,25 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
token_usage=usage,
)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.IMAGE_PROMPT_SUCCESS.value, message="图片 AI 提词生成成功")
await db.commit()
except DatabaseRowLockBusy:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
step.status = ModuleStepStatusEnum.FAILED.value
step.error_message = str(exc)
step.completed_at = _now()
@@ -875,6 +997,7 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
)
_log_project_error(project=project, step=step, event_type="IMAGE_PROMPT_FAILED", message=project.error_message, exc=exc)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.IMAGE_PROMPT_FAILED.value, message=project.error_message)
await db.commit()
return step
@@ -1053,6 +1176,7 @@ async def submit_video_prompt_optimize(
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
.where(ModuleGenerationProject.id == project_id, ModuleGenerationProject.module == MODULE, ModuleGenerationProject.deleted_at.is_(None))
@@ -1072,6 +1196,7 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
return None
if step_id:
await apply_short_lock_timeout(db)
result = await db.execute(
select(ModuleGenerationStep)
.where(
@@ -1111,8 +1236,17 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step.error_message = "缺少新项目图片结果,不能生成视频提词"
project.status = ModuleProjectStatusEnum.FAILED.value
project.error_message = step.error_message
await db.commit()
return step
project_id_value = str(project.id)
step_id_value = str(step.id)
user_id_value = str(project.user_id)
module_value = str(project.module)
expected_step_version = int(step.version or 1)
expected_input_json = json.dumps(step.input_json, ensure_ascii=False, sort_keys=True, default=str)
await db.commit()
try:
request_log = {
"source_project_name": material.get("source_project_name") or "",
@@ -1125,10 +1259,10 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
}
log_module_prompt_event(
event_type="module_prompt_request",
project_id=project.id,
step_id=step.id,
user_id=project.user_id,
module=project.module,
project_id=project_id_value,
step_id=step_id_value,
user_id=user_id_value,
module=module_value,
prompt_type=ModulePromptTypeEnum.VIDEO_PROMPT.value,
request=request_log,
)
@@ -1136,7 +1270,7 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
request_log["schema_config_source"] = schema_config_snapshot.get("source")
prompt_schema, final_prompt, token_usage = await optimize_shot_replicate_video_prompt(
db,
user_id=project.user_id,
user_id=user_id_value,
source_project_name=request_log["source_project_name"],
target_project_name=request_log["target_project_name"],
core_content_point=request_log["core_content_point"],
@@ -1145,11 +1279,24 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
video_config=video_config,
target_platform=target_platform,
schema_config_snapshot=schema_config_snapshot,
module=project.module,
project_id=project.id,
step_id=step.id,
trace_id=f"shot-video-prompt:{step.id}",
module=module_value,
project_id=project_id_value,
step_id=step_id_value,
trace_id=f"shot-video-prompt:{step_id_value}",
)
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
billing = await charge_module_prompt_usage(
db,
user_id=project.user_id,
@@ -1197,7 +1344,25 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
token_usage=usage,
)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.VIDEO_PROMPT_SUCCESS.value, message="视频 AI 提词生成成功")
await db.commit()
except DatabaseRowLockBusy:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
step_id=step_id_value,
step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
)
if not _prompt_context_matches(
step,
expected_version=expected_step_version,
expected_input_json=expected_input_json,
):
await db.rollback()
return None
step.status = ModuleStepStatusEnum.FAILED.value
step.error_message = str(exc)
step.completed_at = _now()
@@ -1215,6 +1380,7 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
)
_log_project_error(project=project, step=step, event_type="VIDEO_PROMPT_FAILED", message=project.error_message, exc=exc)
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.VIDEO_PROMPT_FAILED.value, message=project.error_message)
await db.commit()
return step
@@ -1331,9 +1497,37 @@ async def generate_video_from_prompt(
async def handle_chat_generation_task_completed(db: AsyncSession, task: ChatGenerationTask) -> None:
if not task or task.generation_mode != GENERATION_MODE:
return
result = await db.execute(
meta_result = await db.execute(
select(ModuleGenerationStep.id, ModuleGenerationStep.project_id).where(
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.deleted_at.is_(None),
).limit(1)
)
meta = meta_result.first()
if not meta:
return
step_id_value, project_id_value = str(meta.id), str(meta.project_id)
project_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id == project_id_value,
ModuleGenerationProject.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
project = project_result.scalar_one_or_none()
if not project:
return
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id_value,
ModuleGenerationStep.project_id == project_id_value,
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
@@ -1342,18 +1536,9 @@ async def handle_chat_generation_task_completed(db: AsyncSession, task: ChatGene
.with_for_update()
.limit(1)
)
step = result.scalar_one_or_none()
step = step_result.scalar_one_or_none()
if not step:
return
project_result = await db.execute(
select(ModuleGenerationProject)
.where(ModuleGenerationProject.id == step.project_id, ModuleGenerationProject.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
project = project_result.scalar_one_or_none()
if not project:
return
if step.step_code == ShotReplicateStepCodeEnum.IMAGE_GENERATE.value:
step.status = ModuleStepStatusEnum.COMPLETED.value
@@ -1394,19 +1579,48 @@ async def handle_chat_generation_task_completed(db: AsyncSession, task: ChatGene
async def handle_chat_generation_task_failed(db: AsyncSession, task: ChatGenerationTask) -> None:
if not task or task.generation_mode != GENERATION_MODE:
return
result = await db.execute(
select(ModuleGenerationStep)
.where(ModuleGenerationStep.chat_task_id == task.id, ModuleGenerationStep.module == MODULE, ModuleGenerationStep.is_current == True, ModuleGenerationStep.deleted_at.is_(None))
meta_result = await db.execute(
select(ModuleGenerationStep.id, ModuleGenerationStep.project_id).where(
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.deleted_at.is_(None),
).limit(1)
)
meta = meta_result.first()
if not meta:
return
step_id_value, project_id_value = str(meta.id), str(meta.project_id)
project_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject)
.where(
ModuleGenerationProject.id == project_id_value,
ModuleGenerationProject.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
step = result.scalar_one_or_none()
if not step:
return
project_result = await db.execute(select(ModuleGenerationProject).where(ModuleGenerationProject.id == step.project_id).with_for_update().limit(1))
project = project_result.scalar_one_or_none()
if not project:
return
step_result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id_value,
ModuleGenerationStep.project_id == project_id_value,
ModuleGenerationStep.chat_task_id == task.id,
ModuleGenerationStep.module == MODULE,
ModuleGenerationStep.is_current == True,
ModuleGenerationStep.deleted_at.is_(None),
)
.with_for_update()
.limit(1)
)
step = step_result.scalar_one_or_none()
if not step:
return
step.status = ModuleStepStatusEnum.FAILED.value
step.error_message = task.error_message
step.completed_at = _now()
@@ -1439,7 +1653,8 @@ async def mark_shot_replicate_step_dispatch_failed(
error_message: str,
) -> None:
project = await _get_project_for_user(db, project_id=project_id, user=current_user, for_update=True)
result = await db.execute(
result = await execute_with_lock_timeout(
db,
select(ModuleGenerationStep)
.where(
ModuleGenerationStep.id == step_id,
@@ -414,7 +414,7 @@ def filter_and_normalize_breakdown(result: dict[str, Any], *, mode: AnalysisMode
async def _select_model_config(db: AsyncSession) -> ModelConfig | None:
result = await db.execute(
select(ModelConfig)
.where(ModelConfig.is_active == True)
.where(ModelConfig.is_active == True, ModelConfig.deleted_at.is_(None))
.order_by(ModelConfig.priority.desc())
.limit(1)
)
+14 -3
View File
@@ -77,7 +77,7 @@ async def get_active_engine(db: AsyncSession) -> VideoEngine:
"""Get the active video engine with highest priority."""
result = await db.execute(
select(VideoEngine)
.where(VideoEngine.is_active == True)
.where(VideoEngine.is_active == True, VideoEngine.deleted_at.is_(None))
.order_by(VideoEngine.priority.desc())
.limit(1)
)
@@ -169,7 +169,7 @@ async def submit_video_task(
raise
finally:
await client.close()
return task_id
@@ -221,7 +221,12 @@ async def poll_task_status(engine: VideoEngine, task_id: str) -> dict:
}
async def download_video(video_url: str, dest_path: str) -> str:
async def download_video(
video_url: str,
dest_path: str,
*,
execution_guard=None,
) -> str:
"""Download video to local storage."""
import os
@@ -231,6 +236,12 @@ async def download_video(video_url: str, dest_path: str) -> str:
async with client.stream("GET", video_url) as response:
response.raise_for_status()
with open(dest_path, "wb") as f:
chunk_no = 0
async for chunk in response.aiter_bytes(chunk_size=8192):
f.write(chunk)
chunk_no += 1
if execution_guard is not None and chunk_no % 32 == 0:
await execution_guard()
if execution_guard is not None:
await execution_guard()
return dest_path
-433
View File
@@ -1,433 +0,0 @@
import asyncio
import json
import logging
import os
from datetime import datetime, timezone
from urllib.parse import urlparse
from sqlalchemy import or_, select
from app.config import settings
from app.enums.generation_status import GenerationRecordPipelineStage
from app.models.base import async_session
from app.models.generation_record import GenerationRecord
from app.services.generation.refund_service import mark_generation_record_failed_and_refund_once
from app.services.image_gen import download_image, get_active_image_engine
from app.services.media_token_usage_snapshot_service import sync_generation_record_media_token_snapshot
from app.services.provider_limit import provider_limit
from app.services.resource_accounting_service import (
record_generation_record_generated_resource,
safe_file_size,
)
from app.services.video_cover_service import create_video_cover_for_local_video
from app.services.video_gen import _log_video_response, download_video, get_active_engine, poll_task_status
from app.services.video_upscale.media_service import build_part_mp4_path, probe_video, safe_remove
logger = logging.getLogger("videogen")
POLL_INTERVAL = 30
MAX_POLLS = 60
_PROVIDER_RECOVERABLE_STAGES = {
None,
"",
GenerationRecordPipelineStage.CREATING_PROVIDER_TASK.value,
GenerationRecordPipelineStage.WAITING_REMOTE.value,
GenerationRecordPipelineStage.POLLING.value,
GenerationRecordPipelineStage.RESULT_READY.value,
GenerationRecordPipelineStage.DOWNLOADING.value,
}
def _is_provider_stage(record: GenerationRecord) -> bool:
return (record.pipeline_stage or "") in _PROVIDER_RECOVERABLE_STAGES
def _source_date_dir(record: GenerationRecord) -> str:
created = record.created_at
if created is None:
created = datetime.now(timezone.utc)
return created.strftime("%Y/%m/%d")
def _normalize_image_extension(output_format: str | None, remote_url: str | None = None) -> str:
value = str(output_format or "").strip().lower()
if value in {"jpg", "jpeg"}:
return "jpg"
if value == "png":
return "png"
if value == "webp":
return "webp"
if remote_url:
try:
path = urlparse(remote_url).path or ""
except Exception:
path = str(remote_url)
suffix = os.path.splitext(path)[1].lower().lstrip(".")
if suffix in {"jpg", "jpeg"}:
return "jpg"
if suffix == "png":
return "png"
if suffix == "webp":
return "webp"
return "jpg"
async def _download_generation_record_upscale_source(
record: GenerationRecord,
remote_url: str,
) -> tuple[str, int]:
date_dir = _source_date_dir(record)
dest_dir = os.path.join(settings.STORAGE_LOCAL_PATH, "_upscale_source", date_dir)
os.makedirs(dest_dir, exist_ok=True)
final_path = os.path.join(dest_dir, f"{record.id}.source.mp4")
if os.path.isfile(final_path) and os.path.getsize(final_path) > 0:
await probe_video(final_path)
return final_path, safe_file_size(final_path)
part_path = build_part_mp4_path(final_path)
try:
async with provider_limit("result_download", settings.RESULT_DOWNLOAD_MAX_CONCURRENCY):
await download_video(remote_url, part_path)
if not os.path.isfile(part_path) or os.path.getsize(part_path) <= 0:
raise RuntimeError("超分源视频下载完成但临时文件为空")
await probe_video(part_path)
os.replace(part_path, final_path)
return final_path, safe_file_size(final_path)
except Exception:
safe_remove(part_path)
raise
async def handle_generation_record_video_succeeded(
db,
record: GenerationRecord,
*,
remote_url: str,
provider_response: dict | None,
video_tokens: int = 0,
) -> bool:
"""处理 GenerationRecord 原视频生成成功。
返回 True 表示已进入超分队列False 表示按原流程直接完成
"""
record.video_tokens_used = int(video_tokens or 0)
await sync_generation_record_media_token_snapshot(db, record, provider_response=provider_response or {})
if bool(record.video_upscale_enabled_snapshot) and record.video_upscale_snapshot_json:
record.pipeline_stage = GenerationRecordPipelineStage.DOWNLOADING.value
await db.flush()
source_path, source_size = await _download_generation_record_upscale_source(record, remote_url)
from app.services.video_upscale.task_service import enqueue_upscale_task, prepare_video_upscale_task
upscale = await prepare_video_upscale_task(
db,
generation_record=record,
source_local_path=source_path,
source_file_size_bytes=source_size,
source_remote_url=remote_url,
)
# enqueue_upscale_task 会先提交数据库,再投递 Celery;投递失败由 gen_recovery 补投。
await enqueue_upscale_task(db, upscale=upscale, reason="generation_record_source_ready")
logger.info("GenerationRecord 已进入视频超分队列: record_id=%s upscale_task_id=%s", record.id, upscale.id)
return True
storage_path = None
file_size_bytes = 0
if settings.STORAGE_TYPE == "local" and remote_url:
try:
date_dir = _source_date_dir(record)
dest_dir = os.path.join(settings.STORAGE_LOCAL_PATH, date_dir)
os.makedirs(dest_dir, exist_ok=True)
dest = os.path.join(dest_dir, f"{record.id}.mp4")
await download_video(remote_url, dest)
record.video_url = f"/generate/videos/{date_dir}/{record.id}.mp4"
cover_url, _cover_storage_path = create_video_cover_for_local_video(
record_id=record.id,
video_path=dest,
date_dir=date_dir,
log_prefix=f"GenerationRecord视频封面生成 record_id={record.id}",
)
record.video_cover_url = cover_url
storage_path = dest
file_size_bytes = safe_file_size(dest)
except Exception as exc:
logger.warning("GenerationRecord 最终视频本地保存失败,回退远程地址: record_id=%s error=%s", record.id, exc)
record.video_url = remote_url
else:
record.video_url = remote_url
record.status = "completed"
record.pipeline_stage = GenerationRecordPipelineStage.DONE.value
record.generated_at = datetime.now(timezone.utc)
record.error_message = None
if record.video_url:
await record_generation_record_generated_resource(
db,
record,
resource_url=record.video_url,
storage_path=storage_path,
file_size_bytes=file_size_bytes,
remote_url=remote_url,
generated_at=record.generated_at,
)
await db.commit()
return False
class TaskQueue:
def __init__(self):
self.queue: asyncio.Queue[str] = asyncio.Queue()
self.running = False
self._active: dict[str, int] = {}
async def enqueue(self, record_id: str):
await self.queue.put(record_id)
async def recover(self):
async with async_session() as db:
result = await db.execute(
select(GenerationRecord).where(
GenerationRecord.status == "generating",
GenerationRecord.seedance_task_id.isnot(None),
GenerationRecord.deleted_at.is_(None),
or_(
GenerationRecord.pipeline_stage.is_(None),
GenerationRecord.pipeline_stage.in_(
[stage for stage in _PROVIDER_RECOVERABLE_STAGES if stage]
),
),
)
)
records = result.scalars().all()
for record in records:
await self.queue.put(record.id)
logger.info("Recovered task: %s (seedance: %s stage=%s)", record.id, record.seedance_task_id, record.pipeline_stage)
async def run(self):
self.running = True
logger.info("Video queue started")
while self.running:
try:
record_id = await asyncio.wait_for(self.queue.get(), timeout=5.0)
except asyncio.TimeoutError:
continue
try:
await self._process(record_id)
except Exception as exc:
logger.exception("Error processing %s: %s", record_id, exc)
finally:
self.queue.task_done()
logger.info("Video queue stopped")
async def _process(self, record_id: str):
async with async_session() as db:
result = await db.execute(
select(GenerationRecord).where(
GenerationRecord.id == record_id,
GenerationRecord.deleted_at.is_(None),
).with_for_update().limit(1)
)
record = result.scalar_one_or_none()
if not record or record.status != "generating":
return
if record.gen_type == "video":
if not _is_provider_stage(record):
return
await self._process_video(db, record)
else:
await self._process_image(db, record)
async def _process_video(self, db, record: GenerationRecord):
record_id = record.id
if not record.seedance_task_id:
record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(db, record=record, error_message="缺少外部任务ID")
await db.commit()
return
record.pipeline_stage = GenerationRecordPipelineStage.POLLING.value
try:
engine = await get_active_engine(db)
poll_result = await poll_task_status(engine, record.seedance_task_id)
except Exception as exc:
logger.error("Poll error for %s: %s", record_id, exc)
count = self._active.get(record_id, 0) + 1
self._active[record_id] = count
if count >= MAX_POLLS:
record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(db, record=record, error_message=f"轮询超时: {exc}")
await db.commit()
self._active.pop(record_id, None)
else:
await db.commit()
await asyncio.sleep(POLL_INTERVAL)
await self.queue.put(record_id)
return
status = poll_result["status"]
try:
resp_data = json.loads(poll_result.get("response_data", "{}"))
except (json.JSONDecodeError, TypeError):
resp_data = {}
_log_video_response(record_id, resp_data, poll_result.get("error"))
if status == "succeeded":
file_url = str(poll_result.get("video_url") or "").strip()
if not file_url:
record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(db, record=record, error_message="供应商成功但未返回视频地址")
await db.commit()
return
record.pipeline_stage = GenerationRecordPipelineStage.RESULT_READY.value
await db.flush()
try:
await handle_generation_record_video_succeeded(
db,
record,
remote_url=file_url,
provider_response=resp_data,
video_tokens=poll_result.get("video_tokens", 0),
)
except Exception as exc:
await db.rollback()
result = await db.execute(
select(GenerationRecord).where(GenerationRecord.id == record_id).with_for_update().limit(1)
)
failed_record = result.scalar_one_or_none()
if failed_record:
failed_record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(
db,
record=failed_record,
error_message=f"视频结果下载失败: {exc}",
)
await db.commit()
logger.exception("GenerationRecord 视频成功结果处理失败: %s", record_id)
self._active.pop(record_id, None)
return
if status == "failed":
record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(
db,
record=record,
error_message=poll_result.get("error", "视频生成失败"),
)
self._active.pop(record_id, None)
await db.commit()
logger.info("Video task failed: %s", record_id)
return
count = self._active.get(record_id, 0) + 1
self._active[record_id] = count
if count >= MAX_POLLS:
record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(db, record=record, error_message="视频生成超时")
self._active.pop(record_id, None)
await db.commit()
logger.info("Video task timed out: %s", record_id)
else:
await db.commit()
await asyncio.sleep(POLL_INTERVAL)
await self.queue.put(record_id)
async def _process_image(self, db, record: GenerationRecord):
"""Process image generation task - calls API directly."""
record_id = record.id
from app.services.image_gen import submit_image_task
try:
engine = await get_active_image_engine(db)
poll_result = await asyncio.to_thread(
submit_image_task,
db,
engine,
record,
include_media_references=False,
)
if not isinstance(poll_result, dict):
raise RuntimeError("图片供应商返回结构异常")
items = poll_result.get("items") or []
if not isinstance(items, list):
raise RuntimeError("图片供应商返回结果列表异常")
if not items:
raise RuntimeError("图片供应商未返回图片结果")
if len(items) != 1:
raise RuntimeError(f"图片供应商单图返回数量异常,期望 1,实际 {len(items)}")
item = items[0] or {}
if not isinstance(item, dict):
raise RuntimeError("图片供应商返回单项结果结构异常")
item_error = item.get("error_message") or item.get("error_code")
if item_error:
raise RuntimeError(str(item_error))
remote_url = str(item.get("remote_result_url") or "").strip()
if not remote_url:
raise RuntimeError("图片供应商成功响应但没有图片地址")
storage_path = None
file_size_bytes = 0
if settings.STORAGE_TYPE == "local":
try:
date_dir = _source_date_dir(record)
dest_dir = os.path.join(settings.STORAGE_IMAGE_LOCAL_PATH, date_dir)
os.makedirs(dest_dir, exist_ok=True)
extension = _normalize_image_extension(item.get("output_format"), remote_url)
dest = os.path.join(dest_dir, f"{record_id}.{extension}")
await download_image(remote_url, dest)
record.image_url = f"/generate/images/{date_dir}/{record_id}.{extension}"
storage_path = dest
file_size_bytes = safe_file_size(dest)
except Exception as exc:
logger.warning("GenerationRecord 图片本地保存失败,回退远程地址: record_id=%s error=%s", record_id, exc)
record.image_url = remote_url
else:
record.image_url = remote_url
record.image_tokens_used = int(poll_result.get("image_tokens", 0) or 0)
provider_response = poll_result.get("response_data") or {}
await sync_generation_record_media_token_snapshot(
db,
record,
provider_response=provider_response if isinstance(provider_response, dict) else {},
)
record.status = "completed"
record.pipeline_stage = GenerationRecordPipelineStage.DONE.value
record.generated_at = datetime.now(timezone.utc)
record.error_message = None
if record.image_url:
await record_generation_record_generated_resource(
db,
record,
resource_url=record.image_url,
storage_path=storage_path,
file_size_bytes=file_size_bytes,
remote_url=remote_url,
generated_at=record.generated_at,
)
await db.commit()
logger.info("Image task completed: %s", record_id)
except Exception as exc:
record.pipeline_stage = GenerationRecordPipelineStage.FAILED.value
await mark_generation_record_failed_and_refund_once(
db,
record=record,
error_message=(getattr(exc, "safe_message", None) or str(exc) or "图片生成失败"),
)
await db.commit()
logger.error("Image task failed: %s, error: %s", record_id, exc, exc_info=True)
def stop(self):
"""Signal the queue to stop."""
self.running = False
task_queue = TaskQueue()
@@ -2,12 +2,16 @@ from __future__ import annotations
import asyncio
import os
import subprocess
from pathlib import Path
from collections.abc import Awaitable, Callable
from app.config import settings
from app.services.video_cover_service import get_ffmpeg_bin
from app.services.video_upscale.media_service import build_part_mp4_path, is_valid_file, probe_video, safe_remove
from app.services.video_upscale.media_service import (
build_part_mp4_path,
is_valid_file,
probe_video,
safe_remove,
)
class LocalVideoUpscaleError(RuntimeError):
@@ -28,17 +32,15 @@ def _build_filter(target_width: int, target_height: int) -> str:
)
def _run_ffmpeg_sync(
def _build_command(
*,
source_path: str,
part_path: str,
target_width: int,
target_height: int,
timeout_seconds: int,
) -> None:
ffmpeg_bin = get_ffmpeg_bin() # 明确复用 config.py 的 FFMPEG_BIN。
cmd = [
ffmpeg_bin,
) -> list[str]:
return [
get_ffmpeg_bin(),
"-hide_banner",
"-nostdin",
"-y",
@@ -68,23 +70,69 @@ def _run_ffmpeg_sync(
"192k",
part_path,
]
async def _terminate_process(process: asyncio.subprocess.Process) -> None:
if process.returncode is not None:
return
process.terminate()
try:
result = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
timeout=max(30, int(timeout_seconds)),
shell=False,
await asyncio.wait_for(process.wait(), timeout=5)
except asyncio.TimeoutError:
process.kill()
await process.wait()
async def _run_ffmpeg(
*,
source_path: str,
part_path: str,
target_width: int,
target_height: int,
timeout_seconds: int,
execution_guard: Callable[[], Awaitable[None]] | None,
) -> None:
try:
process = await asyncio.create_subprocess_exec(
*_build_command(
source_path=source_path,
part_path=part_path,
target_width=target_width,
target_height=target_height,
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
except subprocess.TimeoutExpired as exc:
raise LocalVideoUpscaleError(f"本地 FFmpeg 超分超时: {timeout_seconds}") from exc
except OSError as exc:
raise LocalVideoUpscaleError(f"本地 FFmpeg 无法启动: {exc}") from exc
if result.returncode != 0:
raise LocalVideoUpscaleError(f"本地 FFmpeg 超分失败: {(result.stderr or '').strip()[-4000:]}")
communicate_task = asyncio.create_task(process.communicate())
deadline = asyncio.get_running_loop().time() + max(30, int(timeout_seconds))
try:
while not communicate_task.done():
if execution_guard is not None:
await execution_guard()
remaining = deadline - asyncio.get_running_loop().time()
if remaining <= 0:
raise LocalVideoUpscaleError(f"本地 FFmpeg 超分超时: {timeout_seconds}")
try:
await asyncio.wait_for(asyncio.shield(communicate_task), timeout=min(2.0, remaining))
except asyncio.TimeoutError:
continue
stdout, stderr = await communicate_task
except BaseException:
await _terminate_process(process)
if not communicate_task.done():
communicate_task.cancel()
try:
await communicate_task
except BaseException:
pass
raise
if process.returncode != 0:
error_text = (stderr or b"").decode("utf-8", errors="replace").strip()
raise LocalVideoUpscaleError(f"本地 FFmpeg 超分失败: {error_text[-4000:]}")
async def execute_local_ffmpeg_crop(
@@ -94,6 +142,7 @@ async def execute_local_ffmpeg_crop(
target_width: int,
target_height: int,
timeout_seconds: int | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> str:
if not is_valid_file(source_path):
raise LocalVideoUpscaleError(f"超分源视频不存在或为空: {source_path}")
@@ -106,13 +155,13 @@ async def execute_local_ffmpeg_crop(
part_path = build_part_mp4_path(final_path)
safe_remove(part_path)
try:
await asyncio.to_thread(
_run_ffmpeg_sync,
await _run_ffmpeg(
source_path=source_path,
part_path=part_path,
target_width=target_width,
target_height=target_height,
timeout_seconds=int(timeout_seconds or settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS),
execution_guard=execution_guard,
)
if not is_valid_file(part_path):
raise LocalVideoUpscaleError("本地 FFmpeg 输出文件为空")
@@ -121,6 +170,8 @@ async def execute_local_ffmpeg_crop(
raise LocalVideoUpscaleError(
f"本地 FFmpeg 输出尺寸不正确: {info.width}x{info.height},预期 {target_width}x{target_height}"
)
if execution_guard is not None:
await execution_guard()
os.replace(part_path, final_path)
return final_path
except Exception:
@@ -9,6 +9,7 @@ import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from collections.abc import Awaitable, Callable
from typing import Any
from urllib.parse import parse_qsl, urlsplit
@@ -180,7 +181,13 @@ def build_local_source_signed_url(source_local_path: str, expire_seconds: int) -
)
async def download_video_to_path(url: str, final_path: str, timeout_seconds: int) -> str:
async def download_video_to_path(
url: str,
final_path: str,
timeout_seconds: int,
*,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> str:
if is_valid_file(final_path):
try:
await probe_video(final_path)
@@ -196,10 +203,14 @@ async def download_video_to_path(url: str, final_path: str, timeout_seconds: int
response.raise_for_status()
with open(part_path, "wb") as file_obj:
async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
if execution_guard is not None:
await execution_guard()
file_obj.write(chunk)
if not is_valid_file(part_path):
raise RuntimeError("视频下载完成但临时文件为空")
await probe_video(part_path)
if execution_guard is not None:
await execution_guard()
os.replace(part_path, final_path)
return final_path
except Exception:
@@ -10,8 +10,7 @@ from app.enums.generation_task import ChatGenerationPipelineStage, ChatGeneratio
from app.models.chat_generation_task import ChatGenerationTask
from app.models.generation_record import GenerationRecord
from app.models.video_upscale_task import VideoUpscaleTask
from app.services.generation.ai.task_group_service import aggregate_parent_for_child
from app.services.generation.module_hook_service import notify_chat_generation_task_finished
from app.services.generation.pipeline.db_lock_service import apply_short_lock_timeout
VideoUpscaleOwner: TypeAlias = ChatGenerationTask | GenerationRecord
@@ -73,6 +72,7 @@ async def load_upscale_owner(
else:
return None
if for_update:
await apply_short_lock_timeout(db)
query = query.with_for_update()
result = await db.execute(query.limit(1))
return result.scalar_one_or_none()
@@ -88,8 +88,6 @@ async def mark_owner_upscale_failed(
owner.status = ChatGenerationTaskStatus.FAILED.value
owner.pipeline_stage = ChatGenerationPipelineStage.UPSCALE_FAILED.value
owner.error_message = error_message
await notify_chat_generation_task_finished(db, owner)
await aggregate_parent_for_child(db, owner)
return
owner.status = GenerationStatus.failed.value
@@ -5,6 +5,7 @@ import math
import os
import uuid
from datetime import datetime, timedelta, timezone
from collections.abc import Awaitable, Callable
from typing import Any
from urllib.parse import urlsplit, urlunsplit
@@ -30,8 +31,9 @@ from app.enums.video_upscale import (
from app.models.chat_generation_task import ChatGenerationTask
from app.models.generation_record import GenerationRecord
from app.models.video_upscale_task import VideoUpscaleTask
from app.services.generation.ai.task_group_service import aggregate_parent_for_child
from app.services.generation.module_hook_service import notify_chat_generation_task_finished
from app.services.generation.pipeline.db_lock_service import apply_short_lock_timeout
from app.services.generation.pipeline.lifecycle_service import notify_owner_finished
from app.services.redis_registry_service import RedisExecutionLockError, RedisExecutionLockLost
from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot,
sync_generation_record_media_token_snapshot,
@@ -73,6 +75,14 @@ from app.services.video_upscale.volc_service import (
from app.utils.id_gen import generate_id
ExecutionGuard = Callable[[], Awaitable[None]]
async def _ensure_guard(execution_guard: ExecutionGuard | None) -> None:
if execution_guard is not None:
await execution_guard()
def _now() -> datetime:
return datetime.now(timezone.utc)
@@ -185,6 +195,7 @@ async def _load_pair(
) -> tuple[VideoUpscaleTask | None, VideoUpscaleOwner | None]:
query = select(VideoUpscaleTask).where(VideoUpscaleTask.id == upscale_task_id)
if for_update:
await apply_short_lock_timeout(db)
query = query.with_for_update()
result = await db.execute(query.limit(1))
upscale = result.scalar_one_or_none()
@@ -217,6 +228,7 @@ async def prepare_video_upscale_task(
source_local_path: str,
source_file_size_bytes: int,
source_remote_url: str | None = None,
source_info: Any | None = None,
) -> VideoUpscaleTask:
owner: VideoUpscaleOwner | None = task or generation_record
if owner is None:
@@ -224,7 +236,9 @@ async def prepare_video_upscale_task(
if owner.gen_type != GenerationType.VIDEO.value or not bool(owner.video_upscale_enabled_snapshot):
raise RuntimeError("当前任务未启用视频超分快照")
snapshot = _snapshot(owner)
source_info = await probe_video(source_local_path)
# ffprobe should run before the owner row is locked by the caller.
# Keep this fallback for non-generation callers that do not pass probe data.
source_info = source_info or await probe_video(source_local_path)
remote_url = str(source_remote_url or getattr(owner, "remote_result_url", None) or "").strip() or None
signed_at, expires_at = parse_tos_signed_url_expiry(remote_url)
@@ -233,6 +247,7 @@ async def prepare_video_upscale_task(
if isinstance(owner, ChatGenerationTask)
else VideoUpscaleTask.generation_record_id == owner.id
)
await apply_short_lock_timeout(db)
result = await db.execute(
select(VideoUpscaleTask).where(owner_filter).with_for_update().limit(1)
)
@@ -333,6 +348,7 @@ async def _claim(
stage: str,
chat_stage: str,
increment_attempt: bool,
execution_token: str,
lease_seconds: int | None = None,
) -> tuple[VideoUpscaleTask, VideoUpscaleOwner, dict[str, Any]] | None:
upscale, task = await _load_pair(db, upscale_task_id, for_update=True)
@@ -349,7 +365,7 @@ async def _claim(
snapshot = _snapshot(task)
upscale.status = VideoUpscaleTaskStatus.PROCESSING.value
upscale.stage = stage
upscale.lease_token = uuid.uuid4().hex
upscale.lease_token = execution_token
upscale.lease_until = _lease_until(lease_seconds)
upscale.started_at = upscale.started_at or _now()
upscale.next_retry_at = None
@@ -360,6 +376,21 @@ async def _claim(
return upscale, task, snapshot
async def _load_owned_pair(
db: AsyncSession,
*,
upscale_task_id: str,
execution_token: str,
for_update: bool = True,
) -> tuple[VideoUpscaleTask, VideoUpscaleOwner]:
upscale, task = await _load_pair(db, upscale_task_id, for_update=for_update)
if not upscale or not task:
raise RedisExecutionLockLost(f"超分任务或所属任务不存在: {upscale_task_id}")
if str(upscale.lease_token or "") != str(execution_token):
raise RedisExecutionLockLost(f"超分数据库执行租约已失效: {upscale_task_id}")
return upscale, task
async def _final_fail(
db: AsyncSession,
*,
@@ -381,6 +412,7 @@ async def _final_fail(
upscale.lease_token = None
await mark_owner_upscale_failed(db, task, error_message=error_message)
await db.commit()
await notify_owner_finished(db, task)
log_video_upscale_event(
event_type="upscale_retry_exhausted",
event_status="failed",
@@ -464,8 +496,13 @@ async def _queue_finalize(
upscale_task_id: str,
final_path: str,
reason: str,
execution_token: str,
execution_guard: ExecutionGuard | None = None,
) -> None:
upscale, task = await _load_pair(db, upscale_task_id, for_update=True)
await _ensure_guard(execution_guard)
upscale, task = await _load_owned_pair(
db, upscale_task_id=upscale_task_id, execution_token=execution_token, for_update=True
)
if not upscale or not task:
return
if not is_valid_file(final_path):
@@ -488,6 +525,7 @@ async def _queue_finalize(
task=task,
upscale_task=upscale,
action="finalize",
countdown=max(1, int(settings.VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS or 2)),
)
log_video_upscale_event(
event_type="upscale_finalize_enqueued",
@@ -498,8 +536,18 @@ async def _queue_finalize(
)
async def _finalize_success(db: AsyncSession, *, upscale_task_id: str, final_path: str) -> None:
upscale, task = await _load_pair(db, upscale_task_id, for_update=False)
async def _finalize_success(
db: AsyncSession,
*,
upscale_task_id: str,
final_path: str,
execution_token: str,
execution_guard: ExecutionGuard | None = None,
) -> None:
await _ensure_guard(execution_guard)
upscale, task = await _load_owned_pair(
db, upscale_task_id=upscale_task_id, execution_token=execution_token, for_update=False
)
if not upscale or not task:
return
snapshot = _snapshot(task)
@@ -550,7 +598,10 @@ async def _finalize_success(db: AsyncSession, *, upscale_task_id: str, final_pat
if not cover_url or not cover_path:
raise RuntimeError("超分最终视频封面生成失败")
upscale, task = await _load_pair(db, upscale_task_id, for_update=True)
await _ensure_guard(execution_guard)
upscale, task = await _load_owned_pair(
db, upscale_task_id=upscale_task_id, execution_token=execution_token, for_update=True
)
if not upscale or not task:
return
if owner_is_completed(task) and task.video_url:
@@ -569,7 +620,7 @@ async def _finalize_success(db: AsyncSession, *, upscale_task_id: str, final_pat
task.generated_at = now
task.error_message = None
if isinstance(task, ChatGenerationTask):
task.retry_count = 0
task.retry_count = int(task.manual_retry_count or 0)
upscale.status = VideoUpscaleTaskStatus.COMPLETED.value
upscale.stage = VideoUpscaleStage.COMPLETED.value
@@ -595,8 +646,6 @@ async def _finalize_success(db: AsyncSession, *, upscale_task_id: str, final_pat
generated_at=now,
)
await sync_chat_generation_task_media_token_snapshot(db, task)
await notify_chat_generation_task_finished(db, task)
await aggregate_parent_for_child(db, task)
else:
await record_generation_record_generated_resource(
db,
@@ -614,9 +663,12 @@ async def _finalize_success(db: AsyncSession, *, upscale_task_id: str, final_pat
if not delete_source_after_success:
upscale.source_delete_error = VIDEO_UPSCALE_SOURCE_RETAINED_MARKER
await db.commit()
if isinstance(task, ChatGenerationTask):
await notify_owner_finished(db, task)
if delete_source_after_success and source_path and os.path.abspath(source_path) != os.path.abspath(final_path):
removed = safe_remove(source_path)
cleanup_error = None if removed else f"源视频删除失败: {source_path}"
await apply_short_lock_timeout(db)
cleanup_result = await db.execute(
select(VideoUpscaleTask).where(VideoUpscaleTask.id == upscale_task_id).with_for_update().limit(1)
)
@@ -668,13 +720,16 @@ async def _finalize_success(db: AsyncSession, *, upscale_task_id: str, final_pat
)
async def run_local_upscale(db: AsyncSession, upscale_task_id: str) -> None:
async def run_local_upscale(
db: AsyncSession, upscale_task_id: str, *, execution_token: str, execution_guard: ExecutionGuard | None = None
) -> None:
claimed = await _claim(
db,
upscale_task_id=upscale_task_id,
stage=VideoUpscaleStage.LOCAL_PROCESSING.value,
chat_stage=ChatGenerationPipelineStage.UPSCALE_PROCESSING.value,
increment_attempt=True,
execution_token=execution_token,
lease_seconds=int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 300,
)
if not claimed:
@@ -690,15 +745,21 @@ async def run_local_upscale(db: AsyncSession, upscale_task_id: str) -> None:
target_width=int(snapshot.get("target_width") or upscale.target_width),
target_height=int(snapshot.get("target_height") or upscale.target_height),
timeout_seconds=int(processor.get("timeout_seconds") or settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS),
execution_guard=execution_guard,
)
await _ensure_guard(execution_guard)
log_video_upscale_event(
event_type="upscale_local_success",
task=task,
upscale_task=upscale,
detail={"final_local_path": final_path},
)
except RedisExecutionLockError:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
await _ensure_guard(execution_guard)
await _schedule_retry(
db,
upscale_task_id=upscale_task_id,
@@ -714,9 +775,15 @@ async def run_local_upscale(db: AsyncSession, upscale_task_id: str) -> None:
upscale_task_id=upscale_task_id,
final_path=final_path,
reason="local_upscale_completed",
execution_token=execution_token,
execution_guard=execution_guard,
)
except RedisExecutionLockError:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
await _ensure_guard(execution_guard)
await _schedule_retry(
db,
upscale_task_id=upscale_task_id,
@@ -770,13 +837,17 @@ async def _select_remote_input(db: AsyncSession, upscale: VideoUpscaleTask, proc
return signed_url, VideoUpscaleInputSourceType.LOCAL_SIGNED.value
async def run_remote_submit(db: AsyncSession, upscale_task_id: str, *, count_attempt: bool = True) -> None:
async def run_remote_submit(
db: AsyncSession, upscale_task_id: str, *, count_attempt: bool = True, execution_token: str,
execution_guard: ExecutionGuard | None = None,
) -> None:
claimed = await _claim(
db,
upscale_task_id=upscale_task_id,
stage=VideoUpscaleStage.REMOTE_SUBMITTING.value,
chat_stage=ChatGenerationPipelineStage.UPSCALE_PROCESSING.value,
increment_attempt=count_attempt,
execution_token=execution_token,
lease_seconds=300,
)
if not claimed:
@@ -831,9 +902,10 @@ async def run_remote_submit(db: AsyncSession, upscale_task_id: str, *, count_att
processor=processor,
client_token=f"{upscale.id}-{int(upscale.attempt_count or 0)}-{int(upscale.input_source_fallback_count or 0)}",
)
upscale, task = await _load_pair(db, upscale_task_id, for_update=True)
if not upscale or not task:
return
await _ensure_guard(execution_guard)
upscale, task = await _load_owned_pair(
db, upscale_task_id=upscale_task_id, execution_token=execution_token, for_update=True
)
upscale.provider_task_id = submit_result.task_id
upscale.provider_submitted_at = _now()
upscale.provider_request_json = json.dumps(_sanitize_provider_payload(submit_result.request_payload), ensure_ascii=False, default=str)
@@ -863,6 +935,9 @@ async def run_remote_submit(db: AsyncSession, upscale_task_id: str, *, count_att
remote_request_id=submit_result.request_id,
detail={"provider_task_id": submit_result.task_id, "input_source_type": source_type},
)
except RedisExecutionLockError:
await db.rollback()
raise
except VolcMediaKitError as exc:
log_video_upscale_event(
event_type="upscale_provider_submit_failed",
@@ -875,6 +950,7 @@ async def run_remote_submit(db: AsyncSession, upscale_task_id: str, *, count_att
error=str(exc),
)
await db.rollback()
await _ensure_guard(execution_guard)
await _persist_provider_error_payload(
db,
upscale_task_id=upscale_task_id,
@@ -897,6 +973,7 @@ async def run_remote_submit(db: AsyncSession, upscale_task_id: str, *, count_att
error=str(exc),
)
await db.rollback()
await _ensure_guard(execution_guard)
await _schedule_retry(
db,
upscale_task_id=upscale_task_id,
@@ -905,13 +982,16 @@ async def run_remote_submit(db: AsyncSession, upscale_task_id: str, *, count_att
)
async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
async def run_remote_poll(
db: AsyncSession, upscale_task_id: str, *, execution_token: str, execution_guard: ExecutionGuard | None = None
) -> None:
claimed = await _claim(
db,
upscale_task_id=upscale_task_id,
stage=VideoUpscaleStage.REMOTE_POLLING.value,
chat_stage=ChatGenerationPipelineStage.UPSCALE_POLLING.value,
increment_attempt=False,
execution_token=execution_token,
lease_seconds=300,
)
if not claimed:
@@ -939,9 +1019,10 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
provider_task_id,
request_timeout_seconds=int(processor.get("request_timeout_seconds") or 30),
)
upscale, task = await _load_pair(db, upscale_task_id, for_update=True)
if not upscale or not task:
return
await _ensure_guard(execution_guard)
upscale, task = await _load_owned_pair(
db, upscale_task_id=upscale_task_id, execution_token=execution_token, for_update=True
)
upscale.provider_response_json = json.dumps(query_result.response_payload, ensure_ascii=False, default=str)
upscale.lease_until = None
upscale.lease_token = None
@@ -1001,6 +1082,7 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
task=task,
upscale_task=upscale,
action="submit_local_source_fallback",
countdown=max(1, int(settings.VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS or 2)),
)
log_video_upscale_event(
event_type="upscale_source_fallback_local",
@@ -1071,6 +1153,7 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
task=task,
upscale_task=upscale,
action="download_remote_result",
countdown=max(1, int(settings.VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS or 2)),
)
log_video_upscale_event(
event_type="upscale_provider_poll_success",
@@ -1079,6 +1162,9 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
remote_request_id=query_result.request_id,
detail={"provider_output_url_expires_at": upscale.provider_output_url_expires_at},
)
except RedisExecutionLockError:
await db.rollback()
raise
except VolcMediaKitError as exc:
log_video_upscale_event(
event_type="upscale_provider_poll_failed",
@@ -1091,6 +1177,7 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
error=str(exc),
)
await db.rollback()
await _ensure_guard(execution_guard)
await _persist_provider_error_payload(
db,
upscale_task_id=upscale_task_id,
@@ -1113,6 +1200,7 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
error=str(exc),
)
await db.rollback()
await _ensure_guard(execution_guard)
await _schedule_retry(
db,
upscale_task_id=upscale_task_id,
@@ -1121,13 +1209,16 @@ async def run_remote_poll(db: AsyncSession, upscale_task_id: str) -> None:
)
async def run_finalize_upscale(db: AsyncSession, upscale_task_id: str) -> None:
async def run_finalize_upscale(
db: AsyncSession, upscale_task_id: str, *, execution_token: str, execution_guard: ExecutionGuard | None = None
) -> None:
claimed = await _claim(
db,
upscale_task_id=upscale_task_id,
stage=VideoUpscaleStage.FINALIZING.value,
chat_stage=ChatGenerationPipelineStage.UPSCALE_FINALIZING.value,
increment_attempt=False,
execution_token=execution_token,
lease_seconds=max(300, int(settings.VIDEO_COVER_TIMEOUT_SECONDS or 15) + 300),
)
if not claimed:
@@ -1143,13 +1234,19 @@ async def run_finalize_upscale(db: AsyncSession, upscale_task_id: str) -> None:
detail={"final_local_path": final_path},
)
try:
await _finalize_success(db, upscale_task_id=upscale_task_id, final_path=final_path)
await _finalize_success(
db, upscale_task_id=upscale_task_id, final_path=final_path,
execution_token=execution_token, execution_guard=execution_guard,
)
log_video_upscale_event(
event_type="upscale_finalize_success",
task=task,
upscale_task=upscale,
detail={"final_local_path": final_path},
)
except RedisExecutionLockError:
await db.rollback()
raise
except Exception as exc:
log_video_upscale_event(
event_type="upscale_finalize_failed",
@@ -1161,6 +1258,7 @@ async def run_finalize_upscale(db: AsyncSession, upscale_task_id: str) -> None:
error=str(exc),
)
await db.rollback()
await _ensure_guard(execution_guard)
await _schedule_retry(
db,
upscale_task_id=upscale_task_id,
@@ -1170,13 +1268,16 @@ async def run_finalize_upscale(db: AsyncSession, upscale_task_id: str) -> None:
)
async def run_remote_result_download(db: AsyncSession, upscale_task_id: str) -> None:
async def run_remote_result_download(
db: AsyncSession, upscale_task_id: str, *, execution_token: str, execution_guard: ExecutionGuard | None = None
) -> None:
claimed = await _claim(
db,
upscale_task_id=upscale_task_id,
stage=VideoUpscaleStage.RESULT_DOWNLOADING.value,
chat_stage=ChatGenerationPipelineStage.UPSCALE_DOWNLOADING.value,
increment_attempt=False,
execution_token=execution_token,
lease_seconds=int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 120,
)
if not claimed:
@@ -1198,6 +1299,7 @@ async def run_remote_result_download(db: AsyncSession, upscale_task_id: str) ->
task=task,
upscale_task=upscale,
action="renew_remote_result",
countdown=max(1, int(settings.VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS or 2)),
)
log_video_upscale_event(
event_type="upscale_provider_result_renew_query",
@@ -1228,13 +1330,18 @@ async def run_remote_result_download(db: AsyncSession, upscale_task_id: str) ->
output_url,
final_path,
int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600),
execution_guard=execution_guard,
)
await _ensure_guard(execution_guard)
log_video_upscale_event(
event_type="upscale_output_download_success",
task=task,
upscale_task=upscale,
detail={"final_local_path": final_path, "file_size_bytes": safe_file_size(final_path)},
)
except RedisExecutionLockError:
await db.rollback()
raise
except Exception as exc:
log_video_upscale_event(
event_type="upscale_output_download_failed",
@@ -1246,6 +1353,7 @@ async def run_remote_result_download(db: AsyncSession, upscale_task_id: str) ->
error=str(exc),
)
await db.rollback()
await _ensure_guard(execution_guard)
expires_at = _aware(upscale.provider_output_url_expires_at)
action = "submit" if expires_at and expires_at <= _now() else "download"
await _schedule_retry(
@@ -1263,9 +1371,15 @@ async def run_remote_result_download(db: AsyncSession, upscale_task_id: str) ->
upscale_task_id=upscale_task_id,
final_path=final_path,
reason="remote_result_download_completed",
execution_token=execution_token,
execution_guard=execution_guard,
)
except RedisExecutionLockError:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
await _ensure_guard(execution_guard)
await _schedule_retry(
db,
upscale_task_id=upscale_task_id,
@@ -1278,6 +1392,7 @@ async def run_remote_result_download(db: AsyncSession, upscale_task_id: str) ->
async def recover_video_upscale_tasks_once(db: AsyncSession) -> dict[str, Any]:
now = _now()
batch_size = max(1, int(settings.VIDEO_UPSCALE_RECOVERY_BATCH_SIZE or 50))
await apply_short_lock_timeout(db)
result = await db.execute(
select(VideoUpscaleTask)
.where(
@@ -1339,6 +1454,7 @@ async def recover_video_upscale_tasks_once(db: AsyncSession) -> dict[str, Any]:
continue
source_path = str(cleanup_upscale.source_local_path or "")
removed = safe_remove(source_path)
await apply_short_lock_timeout(db)
cleanup_result = await db.execute(
select(VideoUpscaleTask).where(VideoUpscaleTask.id == item.id).with_for_update().limit(1)
)