root
f5d7cfadb4
2、图片返回的 URL 是本地路径 3、视频生成中媒体文件重复下载 4、幂等性检查无数据库唯一约束 5、幂等键冲突返回 409 改为返回已有任务信息 6、虚拟素材库配额校验 TOCTOU 7、项目级联删除与独立素材删除任务并发冲突
566 lines
20 KiB
Python
566 lines
20 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Awaitable, Callable
|
||
|
||
from fastapi import HTTPException
|
||
from sqlalchemy import case, func, select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.enums.private_portrait import (
|
||
PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
|
||
PrivatePortraitAssetStatus,
|
||
PrivatePortraitAssetType,
|
||
PrivatePortraitEventSource,
|
||
PrivatePortraitEventStatus,
|
||
PrivatePortraitEventType,
|
||
PrivatePortraitProjectStatus,
|
||
PrivatePortraitRemoteDeleteStatus,
|
||
)
|
||
from app.models.virtual_portrait_v3 import VpV3Asset, VpV3Project
|
||
from app.schemas.virtual_portrait_v3.project import (
|
||
VpV3ProjectCreate,
|
||
VpV3ProjectListOut,
|
||
VpV3ProjectOut,
|
||
VpV3ProjectUpdate,
|
||
)
|
||
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||
from app.services.private_portrait.ark_client import ArkPrivateAssetClient
|
||
from app.services.virtual_portrait_v3.quota_service import (
|
||
_bytes_to_mb,
|
||
_refresh_quota_used,
|
||
_slug,
|
||
check_project_quota,
|
||
get_quota,
|
||
remote_group_name,
|
||
remote_project_name,
|
||
)
|
||
from app.utils.id_gen import generate_id
|
||
|
||
logger = logging.getLogger("videogen")
|
||
DOMAIN = "virtual_portrait_v3"
|
||
|
||
# 北京时间（UTC+8）统一基准：V3 业务所有时间写入 / 时间比较唯一参考
|
||
_BJ_TZ = timezone(timedelta(hours=8))
|
||
|
||
|
||
def _bj_now() -> datetime:
|
||
"""返回当前北京时间（UTC+8）naive datetime（去掉 tzinfo，与 DB naive 存储一致）。"""
|
||
return datetime.now(_BJ_TZ).replace(tzinfo=None)
|
||
|
||
|
||
def _json(data) -> str | None:
|
||
if data is None:
|
||
return None
|
||
return json.dumps(data, ensure_ascii=False, default=str)
|
||
|
||
|
||
def project_to_out(p: VpV3Project) -> VpV3ProjectOut:
|
||
return VpV3ProjectOut(
|
||
project_id=p.remote_group_id,
|
||
name=p.name,
|
||
description=p.description,
|
||
status=p.status,
|
||
asset_count=int(p.asset_count or 0),
|
||
active_asset_count=int(p.active_asset_count or 0),
|
||
image_asset_count=int(p.image_asset_count or 0),
|
||
video_asset_count=int(p.video_asset_count or 0),
|
||
storage_mb_used=float(p.storage_mb_used or 0),
|
||
remote_delete_status=p.remote_delete_status or PrivatePortraitRemoteDeleteStatus.NONE.value,
|
||
error_message=p.error_message,
|
||
created_at=p.created_at,
|
||
updated_at=p.updated_at,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Project CRUD
|
||
# ---------------------------------------------------------------------------
|
||
|
||
async def create_project(
|
||
db: AsyncSession,
|
||
*,
|
||
api_key_id: str,
|
||
payload: VpV3ProjectCreate,
|
||
) -> VpV3Project:
|
||
"""创建虚拟素材项目（同步调用 Ark CreateAssetGroup）。
|
||
|
||
1. 配额校验
|
||
2. 本地落库 status=creating_remote_group
|
||
3. 调 Ark CreateAssetGroup 拿 remote_group_id
|
||
4. 本地更新为 active，返回
|
||
"""
|
||
await check_project_quota(db, api_key_id=api_key_id, delta=1)
|
||
|
||
# slug = _slug(payload.name)
|
||
proj = VpV3Project(
|
||
id=generate_id(),
|
||
api_key_id=api_key_id,
|
||
name=payload.name.strip()[:128],
|
||
name_slug=payload.name.strip()[:128],
|
||
description=payload.description,
|
||
remote_project_name=remote_project_name(),
|
||
remote_group_id="",
|
||
status=PrivatePortraitProjectStatus.CREATING_REMOTE_GROUP.value,
|
||
asset_count=0,
|
||
active_asset_count=0,
|
||
image_asset_count=0,
|
||
video_asset_count=0,
|
||
storage_mb_used=0,
|
||
)
|
||
db.add(proj)
|
||
await db.flush()
|
||
await db.refresh(proj)
|
||
|
||
group_name = remote_group_name(api_key_id=api_key_id, project_slug=proj.name_slug,id=proj.id)
|
||
|
||
log_operation_event(
|
||
domain=DOMAIN,
|
||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_START.value,
|
||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||
source=PrivatePortraitEventSource.API.value,
|
||
api_key_id=api_key_id,
|
||
project_id=proj.id,
|
||
detail={"remote_group_name": group_name, "remote_project_name": proj.remote_project_name},
|
||
)
|
||
try:
|
||
resp = await ArkPrivateAssetClient().create_asset_group(
|
||
project_name=proj.remote_project_name,
|
||
name=group_name,
|
||
description=payload.description,
|
||
group_type=PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
|
||
)
|
||
remote_group_id = resp.get("Id") or resp.get("GroupId") or resp.get("groupId")
|
||
if not remote_group_id:
|
||
raise RuntimeError("CreateAssetGroup 未返回素材组 ID")
|
||
proj.remote_group_id = str(remote_group_id)
|
||
proj.remote_group_name = group_name
|
||
proj.status = PrivatePortraitProjectStatus.ACTIVE.value
|
||
proj.raw_response_json = _json(resp)
|
||
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||
log_operation_event(
|
||
domain=DOMAIN,
|
||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_SUCCESS.value,
|
||
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||
source=PrivatePortraitEventSource.API.value,
|
||
api_key_id=api_key_id,
|
||
project_id=proj.id,
|
||
detail={"remote_group_id": remote_group_id, "group_name": group_name},
|
||
)
|
||
return proj
|
||
except Exception as exc: # noqa: BLE001
|
||
proj.status = PrivatePortraitProjectStatus.CREATE_GROUP_FAILED.value
|
||
proj.error_message = str(exc)
|
||
log_operation_error(
|
||
domain=DOMAIN,
|
||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_FAILED.value,
|
||
source=PrivatePortraitEventSource.API.value,
|
||
api_key_id=api_key_id,
|
||
project_id=proj.id,
|
||
exc=exc,
|
||
)
|
||
raise HTTPException(status_code=502, detail=f"创建虚拟素材项目失败：{exc}") from exc
|
||
|
||
|
||
async def list_projects(
|
||
db: AsyncSession,
|
||
*,
|
||
api_key_id: str,
|
||
page: int,
|
||
page_size: int,
|
||
keyword: str | None = None,
|
||
status: str | None = None,
|
||
) -> tuple[list[VpV3Project], int]:
|
||
"""按 API Key 分页查询项目列表。"""
|
||
conds = [VpV3Project.api_key_id == api_key_id, VpV3Project.deleted_at.is_(None)]
|
||
if keyword:
|
||
conds.append(VpV3Project.name.ilike(f"%{keyword}%"))
|
||
if status:
|
||
conds.append(VpV3Project.status == status)
|
||
count_result = await db.execute(
|
||
select(func.count(VpV3Project.id)).where(*conds)
|
||
)
|
||
total = int(count_result.scalar() or 0)
|
||
q = (
|
||
select(VpV3Project)
|
||
.where(*conds)
|
||
.order_by(VpV3Project.created_at.desc())
|
||
.limit(page_size)
|
||
.offset((page - 1) * page_size)
|
||
)
|
||
items = list((await db.execute(q)).scalars().all())
|
||
return items, total
|
||
|
||
|
||
async def get_project(db: AsyncSession, *, api_key_id: str, project_id: str) -> VpV3Project:
|
||
"""获取项目详情（权限校验）。"""
|
||
row = (await db.execute(
|
||
select(VpV3Project).where(
|
||
VpV3Project.remote_group_id == project_id,
|
||
VpV3Project.api_key_id == api_key_id,
|
||
VpV3Project.deleted_at.is_(None),
|
||
).limit(1)
|
||
)).scalar_one_or_none()
|
||
if not row:
|
||
raise HTTPException(status_code=404, detail="虚拟素材项目不存在")
|
||
return row
|
||
|
||
|
||
async def update_project(
|
||
db: AsyncSession,
|
||
*,
|
||
api_key_id: str,
|
||
project_id: str,
|
||
payload: VpV3ProjectUpdate,
|
||
) -> VpV3Project:
|
||
"""更新项目展示信息（名称/描述，不会重新创建远端 Group）。"""
|
||
proj = await get_project(db, api_key_id=api_key_id, project_id=project_id)
|
||
changed = False
|
||
if payload.name is not None and payload.name != proj.name:
|
||
proj.name = payload.name.strip()[:128]
|
||
proj.name_slug = _slug(payload.name)
|
||
changed = True
|
||
if payload.description is not None and payload.description != proj.description:
|
||
proj.description = payload.description
|
||
changed = True
|
||
if changed:
|
||
await db.flush()
|
||
return proj
|
||
|
||
|
||
async def soft_delete_project(
|
||
db: AsyncSession,
|
||
*,
|
||
api_key_id: str,
|
||
project_id: str,
|
||
) -> VpV3Project:
|
||
"""软删项目和其下所有素材（本地先删，等 commit 后再投递异步远端删除任务）。
|
||
|
||
会把 quota used 重新刷新一次。
|
||
"""
|
||
proj = await get_project(db, api_key_id=api_key_id, project_id=project_id)
|
||
now = _bj_now()
|
||
proj.deleted_at = now
|
||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.PENDING.value
|
||
proj.status = PrivatePortraitProjectStatus.DELETING.value
|
||
# 级联软删其下所有素材
|
||
await db.execute(
|
||
VpV3Asset.__table__.update() # type: ignore[attr-defined]
|
||
.where(
|
||
VpV3Asset.project_id == proj.id,
|
||
VpV3Asset.deleted_at.is_(None),
|
||
)
|
||
.values(
|
||
deleted_at=now,
|
||
remote_delete_status=PrivatePortraitRemoteDeleteStatus.PENDING.value,
|
||
)
|
||
)
|
||
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||
return proj
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 项目计数刷新（增删素材后调用，用于项目列表快速显示）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
async def refresh_project_counters(db: AsyncSession, project_ids: list[str]) -> None:
|
||
"""按真实数据刷新项目 asset 计数和 storage。"""
|
||
if not project_ids:
|
||
return
|
||
for pid in project_ids:
|
||
row = (await db.execute(
|
||
select(
|
||
func.count(VpV3Asset.id),
|
||
func.sum(case((VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value, 1), else_=0)),
|
||
func.sum(case((VpV3Asset.asset_type == PrivatePortraitAssetType.IMAGE.value, 1), else_=0)),
|
||
func.sum(case((VpV3Asset.asset_type == PrivatePortraitAssetType.VIDEO.value, 1), else_=0)),
|
||
func.sum(case(
|
||
(VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value,
|
||
case((VpV3Asset.asset_type == PrivatePortraitAssetType.IMAGE.value, 1), else_=0)),
|
||
else_=0
|
||
)),
|
||
func.sum(case(
|
||
(VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value,
|
||
case((VpV3Asset.asset_type == PrivatePortraitAssetType.VIDEO.value, 1), else_=0)),
|
||
else_=0
|
||
)),
|
||
func.coalesce(func.sum(VpV3Asset.file_size_bytes), 0),
|
||
).where(
|
||
VpV3Asset.project_id == pid,
|
||
VpV3Asset.deleted_at.is_(None),
|
||
)
|
||
)).one()
|
||
(total, active, img_cnt, vid_cnt, active_img, active_vid, storage_bytes) = row
|
||
proj = (await db.execute(
|
||
select(VpV3Project).where(VpV3Project.id == pid).limit(1)
|
||
)).scalar_one_or_none()
|
||
if proj:
|
||
proj.asset_count = int(total or 0)
|
||
proj.active_asset_count = int(active or 0)
|
||
proj.image_asset_count = int(img_cnt or 0)
|
||
proj.video_asset_count = int(vid_cnt or 0)
|
||
proj.active_image_asset_count = int(active_img or 0)
|
||
proj.active_video_asset_count = int(active_vid or 0)
|
||
proj.storage_mb_used = float(_bytes_to_mb(storage_bytes))
|
||
|
||
|
||
# V3 专属的项目远端删除服务
|
||
V3_DOMAIN = "virtual_portrait_v3"
|
||
|
||
|
||
async def _load_v3_project_delete_snapshot(db: AsyncSession, *, project_id: str) -> dict | None:
|
||
"""加载 V3 项目删除快照。"""
|
||
proj = (
|
||
await db.execute(
|
||
select(VpV3Project).where(VpV3Project.id == project_id).limit(1)
|
||
)
|
||
).scalar_one_or_none()
|
||
if not proj:
|
||
return None
|
||
return {
|
||
"owner_id": str(proj.id),
|
||
"owner_type": "project",
|
||
"api_key_id": str(proj.api_key_id),
|
||
"remote_id": str(proj.remote_group_id) if proj.remote_group_id else None,
|
||
"remote_project_name": str(proj.remote_project_name or ""),
|
||
"remote_delete_status": str(proj.remote_delete_status or ""),
|
||
}
|
||
|
||
|
||
async def _apply_v3_project_delete_result(
|
||
db: AsyncSession,
|
||
*,
|
||
project_id: str,
|
||
remote_id: str | None,
|
||
succeeded: bool,
|
||
skipped: bool = False,
|
||
error: BaseException | None = None,
|
||
) -> None:
|
||
"""应用 V3 项目远端删除结果到数据库。"""
|
||
proj = (
|
||
await db.execute(
|
||
select(VpV3Project)
|
||
.where(VpV3Project.id == project_id)
|
||
.with_for_update()
|
||
.limit(1)
|
||
)
|
||
).scalar_one_or_none()
|
||
if not proj:
|
||
return
|
||
if proj.remote_delete_status in {
|
||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||
}:
|
||
return
|
||
if remote_id and str(proj.remote_group_id or "") != remote_id:
|
||
raise RuntimeError("V3 项目远程 Group 已变化，旧删除结果已丢弃")
|
||
now = _bj_now()
|
||
if skipped:
|
||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
||
proj.remote_delete_error = None
|
||
elif succeeded:
|
||
proj.status = PrivatePortraitProjectStatus.DELETED.value
|
||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||
proj.remote_deleted_at = now
|
||
proj.remote_delete_error = None
|
||
else:
|
||
proj.status = PrivatePortraitProjectStatus.DELETED.value
|
||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||
proj.remote_delete_error = str(error or "远程删除失败")
|
||
await db.flush()
|
||
# 刷新配额
|
||
quota = await get_quota(db, api_key_id=proj.api_key_id, refresh=False)
|
||
await _refresh_quota_used(db, quota)
|
||
|
||
|
||
async def delete_v3_project_remote(
|
||
db: AsyncSession,
|
||
*,
|
||
project_id: str,
|
||
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||
) -> None:
|
||
"""V3 项目远端删除（异步 Celery 任务调用）。
|
||
|
||
会先级联删除项目下所有素材的远端资源，再删除项目的远端 Group。
|
||
"""
|
||
# 先删除项目下所有素材的远端资源
|
||
# 使用 with_for_update(skip_locked=True) 避免与独立素材删除任务冲突
|
||
assets = (
|
||
await db.execute(
|
||
select(VpV3Asset).where(
|
||
VpV3Asset.project_id == project_id,
|
||
VpV3Asset.deleted_at.is_not(None),
|
||
VpV3Asset.remote_delete_status == PrivatePortraitRemoteDeleteStatus.PENDING.value,
|
||
)
|
||
.with_for_update(skip_locked=True)
|
||
)
|
||
).scalars().all()
|
||
|
||
for asset in assets:
|
||
# 二次确认：如果独立素材删除任务已处理完该素材，跳过
|
||
if asset.remote_delete_status not in (
|
||
PrivatePortraitRemoteDeleteStatus.PENDING.value,
|
||
PrivatePortraitRemoteDeleteStatus.FAILED.value,
|
||
):
|
||
continue
|
||
if asset.remote_asset_id:
|
||
try:
|
||
await ArkPrivateAssetClient(for_celery=True).delete_asset(
|
||
project_name=asset.remote_project_name,
|
||
asset_id=asset.remote_asset_id,
|
||
)
|
||
await _apply_v3_asset_delete_result_for_project(
|
||
db,
|
||
asset_id=asset.id,
|
||
succeeded=True,
|
||
)
|
||
except Exception as exc:
|
||
if "not found" in str(exc).lower() or "404" in str(exc):
|
||
await _apply_v3_asset_delete_result_for_project(
|
||
db,
|
||
asset_id=asset.id,
|
||
succeeded=True,
|
||
)
|
||
else:
|
||
await _apply_v3_asset_delete_result_for_project(
|
||
db,
|
||
asset_id=asset.id,
|
||
succeeded=False,
|
||
error=exc,
|
||
)
|
||
|
||
# 再删除项目的远端 Group
|
||
snapshot = await _load_v3_project_delete_snapshot(db, project_id=project_id)
|
||
if snapshot is None:
|
||
log_operation_event(
|
||
domain=V3_DOMAIN,
|
||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
|
||
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||
source=PrivatePortraitEventSource.CELERY.value,
|
||
project_id=project_id,
|
||
message="远程删除跳过：本地项目不存在",
|
||
)
|
||
await db.rollback()
|
||
return
|
||
|
||
if snapshot["remote_delete_status"] in {
|
||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||
}:
|
||
await db.rollback()
|
||
return
|
||
|
||
remote_id = snapshot["remote_id"]
|
||
if not remote_id:
|
||
await _apply_v3_project_delete_result(
|
||
db,
|
||
project_id=project_id,
|
||
remote_id=None,
|
||
succeeded=False,
|
||
skipped=True,
|
||
)
|
||
log_operation_event(
|
||
domain=V3_DOMAIN,
|
||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
|
||
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||
source=PrivatePortraitEventSource.CELERY.value,
|
||
project_id=project_id,
|
||
message="远程删除跳过：项目没有 remote_group_id",
|
||
)
|
||
return
|
||
|
||
log_operation_event(
|
||
domain=V3_DOMAIN,
|
||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
|
||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||
source=PrivatePortraitEventSource.CELERY.value,
|
||
project_id=project_id,
|
||
detail={
|
||
"remote_group_id": remote_id,
|
||
"remote_project_name": snapshot["remote_project_name"],
|
||
},
|
||
)
|
||
await db.rollback()
|
||
|
||
remote_error: BaseException | None = None
|
||
succeeded = False
|
||
try:
|
||
await ArkPrivateAssetClient(for_celery=True).delete_asset_group(
|
||
project_name=snapshot["remote_project_name"],
|
||
group_id=remote_id,
|
||
)
|
||
succeeded = True
|
||
except Exception as exc:
|
||
remote_error = exc
|
||
# 404 视为幂等成功
|
||
if "not found" in str(exc).lower() or "404" in str(exc):
|
||
succeeded = True
|
||
|
||
if execution_guard is not None:
|
||
await execution_guard()
|
||
|
||
await _apply_v3_project_delete_result(
|
||
db,
|
||
project_id=project_id,
|
||
remote_id=remote_id,
|
||
succeeded=succeeded,
|
||
error=remote_error,
|
||
)
|
||
|
||
if succeeded:
|
||
log_operation_event(
|
||
domain=V3_DOMAIN,
|
||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
|
||
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||
source=PrivatePortraitEventSource.CELERY.value,
|
||
project_id=project_id,
|
||
message="远程 Group 不存在，按幂等删除成功处理" if remote_error is not None else None,
|
||
detail={
|
||
"remote_group_id": remote_id,
|
||
"remote_project_name": snapshot["remote_project_name"],
|
||
},
|
||
)
|
||
else:
|
||
assert remote_error is not None
|
||
log_operation_error(
|
||
domain=V3_DOMAIN,
|
||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
|
||
source=PrivatePortraitEventSource.CELERY.value,
|
||
project_id=project_id,
|
||
exc=remote_error,
|
||
)
|
||
|
||
|
||
async def _apply_v3_asset_delete_result_for_project(
|
||
db: AsyncSession,
|
||
*,
|
||
asset_id: str,
|
||
succeeded: bool,
|
||
error: BaseException | None = None,
|
||
) -> None:
|
||
"""项目删除时级联应用素材删除结果。"""
|
||
asset = (
|
||
await db.execute(
|
||
select(VpV3Asset).where(VpV3Asset.id == asset_id).with_for_update().limit(1)
|
||
)
|
||
).scalar_one_or_none()
|
||
if not asset:
|
||
return
|
||
if asset.remote_delete_status in {
|
||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||
}:
|
||
return
|
||
if succeeded:
|
||
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
|
||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||
asset.remote_deleted_at = _bj_now()
|
||
asset.remote_delete_error = None
|
||
else:
|
||
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
|
||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||
asset.remote_delete_error = str(error or "远程删除失败")
|
||
await db.flush()
|