1、增加调用 AI 视频生成能力和虚拟素材库管理的对外api
2、增加后台apikkey管理 3、增加apikey单独的模型定价 4、增加apikey调用情况 5、完善所有数据的注释增加
This commit is contained in:
@@ -0,0 +1,59 @@
|
||||
from app.services.api_v3.auth_service import ApiKeyContext, get_api_key_dependency
|
||||
from app.services.api_v3.key_service import (
|
||||
create_api_key,
|
||||
list_api_keys,
|
||||
get_api_key,
|
||||
update_api_key,
|
||||
delete_api_key,
|
||||
reset_quota_if_needed,
|
||||
)
|
||||
from app.services.api_v3.quota_service import check_quota, can_start_video_task, get_active_video_tasks_count, get_queued_video_tasks
|
||||
from app.services.api_v3.usage_log_service import record_usage, get_usage_summary, list_usage_logs
|
||||
from app.services.api_v3.generation_service import submit_video_generation, generate_image_sync
|
||||
from app.services.api_v3.task_service import create_video_task, create_image_task, get_task, map_task_to_status_response
|
||||
from app.services.api_v3.upscale_service import (
|
||||
get_or_create_upscale_config,
|
||||
save_upscale_config,
|
||||
build_api_upscale_snapshot,
|
||||
prepare_api_upscale_task,
|
||||
)
|
||||
from app.services.api_v3.engine_service import resolve_video_engine, resolve_image_engine, build_engine_snapshot
|
||||
from app.services.api_v3.pricing_service import (
|
||||
calc_api_video_price,
|
||||
calc_api_image_price,
|
||||
get_priced_models,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ApiKeyContext",
|
||||
"get_api_key_dependency",
|
||||
"create_api_key",
|
||||
"list_api_keys",
|
||||
"get_api_key",
|
||||
"update_api_key",
|
||||
"delete_api_key",
|
||||
"reset_quota_if_needed",
|
||||
"check_quota",
|
||||
"can_start_video_task",
|
||||
"get_active_video_tasks_count",
|
||||
"get_queued_video_tasks",
|
||||
"record_usage",
|
||||
"get_usage_summary",
|
||||
"list_usage_logs",
|
||||
"submit_video_generation",
|
||||
"generate_image_sync",
|
||||
"create_video_task",
|
||||
"create_image_task",
|
||||
"get_task",
|
||||
"map_task_to_status_response",
|
||||
"get_or_create_upscale_config",
|
||||
"save_upscale_config",
|
||||
"build_api_upscale_snapshot",
|
||||
"prepare_api_upscale_task",
|
||||
"resolve_video_engine",
|
||||
"resolve_image_engine",
|
||||
"build_engine_snapshot",
|
||||
"calc_api_video_price",
|
||||
"calc_api_image_price",
|
||||
"get_priced_models",
|
||||
]
|
||||
@@ -0,0 +1,99 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.dependencies import get_db
|
||||
from app.models.api.api_key import ApiKey
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
security = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
class ApiKeyContext:
|
||||
"""API Key 验证上下文,携带解析后的可调用模型列表。"""
|
||||
|
||||
def __init__(self, api_key: ApiKey, callable_models: list[dict]):
|
||||
self.api_key = api_key
|
||||
self.api_key_id = api_key.id
|
||||
self.callable_models = callable_models
|
||||
|
||||
|
||||
async def get_api_key_dependency(
|
||||
credentials: HTTPAuthorizationCredentials | None = Depends(security),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> ApiKeyContext:
|
||||
"""FastAPI Dependency: 验证 API Key 并返回上下文。
|
||||
|
||||
验证流程:
|
||||
1. 提取 Bearer <REDACTED>
|
||||
2. SHA-256 哈希后查询数据库
|
||||
3. 检查 is_active、deleted_at
|
||||
4. 检查有效期 (valid_from, valid_until)
|
||||
5. 检查配额 (quota_limit, quota_used)
|
||||
6. 重置过期周期的配额
|
||||
"""
|
||||
if not credentials:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="缺少 Authorization 头,请提供 Bearer <REDACTED>",
|
||||
)
|
||||
|
||||
token_hash = hashlib.sha256(credentials.credentials.encode()).hexdigest()
|
||||
|
||||
result = await db.execute(
|
||||
select(ApiKey).where(
|
||||
ApiKey.api_key_hash == token_hash,
|
||||
ApiKey.is_active == True,
|
||||
ApiKey.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
key = result.scalar_one_or_none()
|
||||
|
||||
if not key:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="无效的 API Key",
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# 检查有效期
|
||||
if key.valid_from and now < key.valid_from:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="API Key 尚未生效",
|
||||
)
|
||||
if key.valid_until and now >= key.valid_until:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="API Key 已过期",
|
||||
)
|
||||
|
||||
# 配额周期重置
|
||||
from app.services.api_v3.key_service import reset_quota_if_needed
|
||||
key = await reset_quota_if_needed(db, key)
|
||||
|
||||
# 检查配额
|
||||
if key.quota_limit is not None and key.quota_used >= key.quota_limit:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=f"API Key 配额已用尽 (已用 {key.quota_used:.2f} / 限额 {key.quota_limit:.2f})",
|
||||
)
|
||||
|
||||
# 解析可调用模型
|
||||
try:
|
||||
callable_models = json.loads(key.callable_models) if key.callable_models else []
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
callable_models = []
|
||||
|
||||
# 更新最后使用时间
|
||||
key.last_used_at = now
|
||||
|
||||
return ApiKeyContext(api_key=key, callable_models=callable_models)
|
||||
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.image_engine import ImageEngine
|
||||
from app.models.video_engine import VideoEngine
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
async def resolve_video_engine(
|
||||
db: AsyncSession,
|
||||
engine_id: str,
|
||||
callable_models: list[dict],
|
||||
) -> VideoEngine:
|
||||
"""根据 engine_id 解析视频引擎,并验证是否在 api-key 的可调用列表中。"""
|
||||
# 验证授权
|
||||
allowed = {m["engine_id"] for m in callable_models if m.get("engine_type") == "video"}
|
||||
if engine_id not in allowed:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"该 API Key 无权使用引擎 {engine_id}",
|
||||
)
|
||||
|
||||
result = await db.execute(
|
||||
select(VideoEngine).where(
|
||||
VideoEngine.id == engine_id,
|
||||
VideoEngine.is_active == True,
|
||||
VideoEngine.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
engine = result.scalar_one_or_none()
|
||||
if not engine:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"视频引擎 {engine_id} 不存在或未启用",
|
||||
)
|
||||
return engine
|
||||
|
||||
|
||||
async def resolve_image_engine(
|
||||
db: AsyncSession,
|
||||
engine_id: str,
|
||||
callable_models: list[dict],
|
||||
) -> ImageEngine:
|
||||
"""根据 engine_id 解析图片引擎,并验证是否在 api-key 的可调用列表中。"""
|
||||
allowed = {m["engine_id"] for m in callable_models if m.get("engine_type") == "image"}
|
||||
if engine_id not in allowed:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"该 API Key 无权使用引擎 {engine_id}",
|
||||
)
|
||||
|
||||
result = await db.execute(
|
||||
select(ImageEngine).where(
|
||||
ImageEngine.id == engine_id,
|
||||
ImageEngine.is_active == True,
|
||||
ImageEngine.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
engine = result.scalar_one_or_none()
|
||||
if not engine:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"图片引擎 {engine_id} 不存在或未启用",
|
||||
)
|
||||
return engine
|
||||
|
||||
|
||||
def build_engine_snapshot(engine: VideoEngine | ImageEngine) -> dict:
|
||||
"""构建引擎配置快照。"""
|
||||
snapshot = {
|
||||
"id": str(engine.id),
|
||||
"name": str(engine.name),
|
||||
"provider": str(getattr(engine, "provider", "")),
|
||||
"api_base": str(engine.api_base),
|
||||
"model_name": str(engine.model_name),
|
||||
}
|
||||
# 可选字段
|
||||
for field in [
|
||||
"supported_ratios", "supported_resolutions", "supported_durations",
|
||||
"default_size", "multi_generation_enabled", "max_generation_count",
|
||||
]:
|
||||
val = getattr(engine, field, None)
|
||||
if val is not None:
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
val = json.loads(val)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
snapshot[field] = val
|
||||
return snapshot
|
||||
|
||||
|
||||
async def resolve_engine_by_model_name(
|
||||
db: AsyncSession,
|
||||
model_name: str,
|
||||
callable_models: list[dict],
|
||||
engine_type: str,
|
||||
) -> tuple[str, VideoEngine | ImageEngine]:
|
||||
"""根据模型名称查找对应的引擎。
|
||||
|
||||
Returns:
|
||||
(engine_id, engine 对象)
|
||||
"""
|
||||
# 在 callable_models 中查找
|
||||
target = None
|
||||
for m in callable_models:
|
||||
if m.get("model_name") == model_name and m.get("engine_type") == engine_type:
|
||||
target = m
|
||||
break
|
||||
|
||||
if not target:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"该 API Key 无权使用模型 {model_name}",
|
||||
)
|
||||
|
||||
engine_id = target["engine_id"]
|
||||
if engine_type == "video":
|
||||
engine = await resolve_video_engine(db, engine_id, callable_models)
|
||||
else:
|
||||
engine = await resolve_image_engine(db, engine_id, callable_models)
|
||||
|
||||
return engine_id, engine
|
||||
@@ -0,0 +1,148 @@
|
||||
"""API v3 文件下载服务。
|
||||
|
||||
下载用户提供的图片/视频/音频到本地存储。
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
def _get_date_str() -> str:
|
||||
"""获取当前日期字符串。"""
|
||||
return datetime.now().strftime("%Y%m%d")
|
||||
|
||||
|
||||
def _get_uploads_dir() -> str:
|
||||
"""获取上传文件存储目录。"""
|
||||
upload_dir = os.path.join(os.path.dirname(settings.STORAGE_LOCAL_PATH), "uploads", "api")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
return upload_dir
|
||||
|
||||
|
||||
async def download_file_from_url(url: str, sub_dir: str = "") -> str:
|
||||
"""从 URL 下载文件到本地。
|
||||
|
||||
Args:
|
||||
url: 文件 URL
|
||||
sub_dir: 子目录(如 images/videos/audios)
|
||||
|
||||
Returns:
|
||||
相对路径: /uploads/api/{sub_dir}/{date}/{filename}
|
||||
"""
|
||||
upload_dir = _get_uploads_dir()
|
||||
date_str = _get_date_str()
|
||||
|
||||
# 创建目标目录
|
||||
dest_dir = os.path.join(upload_dir, sub_dir, date_str)
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
|
||||
# 从 URL 提取扩展名
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if not ext or len(ext) > 10:
|
||||
ext = ".bin" # 默认扩展名
|
||||
|
||||
# 生成唯一文件名
|
||||
filename = f"{uuid.uuid4().hex}{ext}"
|
||||
dest_path = os.path.join(dest_dir, filename)
|
||||
|
||||
# 下载文件
|
||||
async with httpx.AsyncClient(timeout=300, follow_redirects=True) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
response.raise_for_status()
|
||||
with open(dest_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
# 返回相对路径
|
||||
rel_path = f"/uploads/api/{sub_dir}/{date_str}/{filename}"
|
||||
logger.info("Downloaded file: %s -> %s", url[:80], rel_path)
|
||||
return rel_path
|
||||
|
||||
|
||||
def save_base64_file(data: str, sub_dir: str = "") -> str:
|
||||
"""保存 Base64 编码的文件到本地。
|
||||
|
||||
Args:
|
||||
data: Base64 编码的数据(可包含 data:...;base64, 前缀)
|
||||
sub_dir: 子目录
|
||||
|
||||
Returns:
|
||||
相对路径: /uploads/api/{sub_dir}/{date}/{filename}
|
||||
"""
|
||||
upload_dir = _get_uploads_dir()
|
||||
date_str = _get_date_str()
|
||||
|
||||
# 创建目标目录
|
||||
dest_dir = os.path.join(upload_dir, sub_dir, date_str)
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
|
||||
# 解析 Base64 数据
|
||||
if "," in data:
|
||||
header, b64_data = data.split(",", 1)
|
||||
# 从 header 提取 MIME 类型
|
||||
mime_match = re.search(r"data:([^;]+)", header)
|
||||
mime_type = mime_match.group(1) if mime_match else "application/octet-stream"
|
||||
# 根据 MIME 类型确定扩展名
|
||||
ext_map = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
"video/mp4": ".mp4",
|
||||
"video/webm": ".webm",
|
||||
"audio/mpeg": ".mp3",
|
||||
"audio/wav": ".wav",
|
||||
"audio/ogg": ".ogg",
|
||||
}
|
||||
ext = ext_map.get(mime_type, ".bin")
|
||||
else:
|
||||
b64_data = data
|
||||
ext = ".bin"
|
||||
|
||||
# 解码并保存
|
||||
try:
|
||||
file_data = base64.b64decode(b64_data)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Base64 解码失败: {exc}")
|
||||
|
||||
filename = f"{uuid.uuid4().hex}{ext}"
|
||||
dest_path = os.path.join(dest_dir, filename)
|
||||
|
||||
with open(dest_path, "wb") as f:
|
||||
f.write(file_data)
|
||||
|
||||
rel_path = f"/uploads/api/{sub_dir}/{date_str}/{filename}"
|
||||
logger.info("Saved base64 file: %s (%d bytes)", rel_path, len(file_data))
|
||||
return rel_path
|
||||
|
||||
|
||||
async def process_media_url(url: str, media_type: str) -> str:
|
||||
"""处理媒体 URL:下载到本地或保存 Base64。
|
||||
|
||||
Args:
|
||||
url: URL 或 Base64 数据
|
||||
media_type: image / video / audio
|
||||
|
||||
Returns:
|
||||
相对路径: /uploads/api/{type}/{date}/{filename}
|
||||
"""
|
||||
sub_dir = {"image": "images", "video": "videos", "audio": "audios"}.get(media_type, "files")
|
||||
|
||||
# 判断是 Base64 还是 URL
|
||||
if url.startswith("data:"):
|
||||
return save_base64_file(url, sub_dir)
|
||||
else:
|
||||
return await download_file_from_url(url, sub_dir)
|
||||
@@ -0,0 +1,494 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_key import ApiKey
|
||||
from app.schemas.api_v3.image import ApiImageGenerateRequest, ApiImageGenerateResponse, ApiImageGenerateDataItem
|
||||
from app.schemas.api_v3.video import ApiVideoCreateRequest, ApiVideoCreateResponse
|
||||
from app.services.api_v3 import task_service, engine_service, upscale_service
|
||||
from app.services.api_v3.quota_service import can_start_video_task
|
||||
from app.services.api_v3.pricing_service import calc_api_video_price, calc_api_image_price, PricingNotConfiguredError
|
||||
from app.services.api_v3.logging_service import log_model_request
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
async def submit_video_generation(
|
||||
db: AsyncSession,
|
||||
key: ApiKey,
|
||||
callable_models: list[dict],
|
||||
req: ApiVideoCreateRequest,
|
||||
) -> ApiVideoCreateResponse:
|
||||
"""提交视频生成任务(异步)。
|
||||
|
||||
流程:
|
||||
1. 检查并发视频任务数
|
||||
2. 解析引擎
|
||||
3. 构建超分快照
|
||||
4. 创建任务记录
|
||||
5. 入队 Celery 任务
|
||||
6. 返回 task_id
|
||||
"""
|
||||
# 1. 检查是否可以立即启动(并发限制)
|
||||
can_start = await can_start_video_task(key, db)
|
||||
|
||||
# 2. 解析引擎
|
||||
engine_id, engine = await engine_service.resolve_engine_by_model_name(
|
||||
db, req.model, callable_models, "video"
|
||||
)
|
||||
engine_snapshot = engine_service.build_engine_snapshot(engine)
|
||||
|
||||
# 3. 构建超分快照
|
||||
provider_resolution, upscale_enabled, upscale_snapshot_json = await upscale_service.build_api_upscale_snapshot(
|
||||
db, key.id, req.resolution or "480p", req.ratio
|
||||
)
|
||||
|
||||
# 如果超分要求不同的生成分辨率,使用超分的
|
||||
final_provider_resolution = provider_resolution or req.resolution or "480p"
|
||||
|
||||
def _get_max_supported_duration(engine) -> int | None:
|
||||
"""从引擎 supported_durations 获取最大时长。"""
|
||||
try:
|
||||
durations = json.loads(engine.supported_durations) if engine.supported_durations else []
|
||||
return max(durations) if durations else engine.max_duration
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return engine.max_duration
|
||||
|
||||
# 3.5 验证传入的媒体文件是否符合引擎配置要求
|
||||
from app.services.api_v3.file_service import process_media_url
|
||||
from app.services.video_upscale.media_service import probe_video
|
||||
from fastapi import HTTPException, status # noqa: F401
|
||||
|
||||
input_image_count = 0
|
||||
input_video_count = 0
|
||||
input_audio_count = 0
|
||||
input_video_duration = 0.0
|
||||
local_media_refs = [] # 存储本地路径
|
||||
|
||||
# 统计各类媒体数量
|
||||
for p in req.content:
|
||||
ptype = p.type
|
||||
if ptype == "image_url":
|
||||
input_image_count += 1
|
||||
elif ptype == "video_url":
|
||||
input_video_count += 1
|
||||
elif ptype == "audio_url":
|
||||
input_audio_count += 1
|
||||
|
||||
# 视频引擎校验(本函数仅处理视频生成)
|
||||
# 校验图片数量限制
|
||||
if input_image_count > (engine.max_image_count or 0):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"该引擎最多支持 {engine.max_image_count} 张参考图片,当前传入 {input_image_count} 张",
|
||||
)
|
||||
# 校验视频数量限制
|
||||
if input_video_count > (engine.max_video_count or 0):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"该引擎最多支持 {engine.max_video_count} 个参考视频,当前传入 {input_video_count} 个",
|
||||
)
|
||||
# 校验音频数量限制
|
||||
if input_audio_count > (engine.max_audio_count or 0):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"该引擎最多支持 {engine.max_audio_count} 个参考音频,当前传入 {input_audio_count} 个",
|
||||
)
|
||||
|
||||
for p in req.content:
|
||||
ptype = p.type
|
||||
if ptype == "text":
|
||||
local_media_refs.append(p.model_dump(exclude_none=True))
|
||||
continue
|
||||
|
||||
original_url = ""
|
||||
if ptype == "image_url" and p.image_url:
|
||||
original_url = p.image_url.get("url", "")
|
||||
elif ptype == "video_url" and p.video_url:
|
||||
original_url = p.video_url.get("url", "")
|
||||
elif ptype == "audio_url" and p.audio_url:
|
||||
original_url = p.audio_url.get("url", "")
|
||||
|
||||
# 下载文件到本地
|
||||
try:
|
||||
local_path = await process_media_url(original_url, ptype.replace("_url", ""))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to download media %s: %s", original_url[:80], exc)
|
||||
local_path = original_url
|
||||
|
||||
# 如果是视频/音频,探测实际时长并校验
|
||||
if ptype in ("video_url", "audio_url") and local_path:
|
||||
try:
|
||||
media_info = await probe_video(local_path)
|
||||
if media_info and media_info.duration_seconds:
|
||||
duration = media_info.duration_seconds
|
||||
# 校验最低时长(2秒)
|
||||
if duration < 2.0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"上传的{ptype.replace('_url', '')}时长不能低于2秒,当前时长: {duration:.1f}秒",
|
||||
)
|
||||
# 校验最高时长(根据引擎 supported_durations 最大值)
|
||||
max_duration = _get_max_supported_duration(engine)
|
||||
if max_duration and duration > max_duration:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"上传的{ptype.replace('_url', '')}时长不能超过{max_duration}秒,当前时长: {duration:.1f}秒",
|
||||
)
|
||||
if ptype == "video_url":
|
||||
input_video_duration += duration
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to probe media duration: %s", exc)
|
||||
|
||||
local_media_refs.append({
|
||||
"type": ptype,
|
||||
ptype: {"url": local_path},
|
||||
"role": p.role,
|
||||
})
|
||||
|
||||
# 3.6 计算价格(基于实际探测的视频时长)
|
||||
try:
|
||||
estimated_price = await calc_api_video_price(
|
||||
db,
|
||||
duration=req.duration or 5,
|
||||
resolution=req.resolution or "480p",
|
||||
engine_id=engine_id,
|
||||
input_video_duration=input_video_duration,
|
||||
input_image_count=input_image_count,
|
||||
)
|
||||
except PricingNotConfiguredError as exc:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
)
|
||||
# 预检配额
|
||||
if key.quota_limit is not None and key.quota_used + estimated_price > key.quota_limit:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=f"配额不足 (需要 {estimated_price:.2f} 元, 剩余 {key.quota_limit - key.quota_used:.2f} 元)",
|
||||
)
|
||||
# 预扣配额
|
||||
if estimated_price > 0:
|
||||
key.quota_used = round((key.quota_used or 0.0) + estimated_price, 2)
|
||||
|
||||
# 4. 创建任务记录(幂等性已在路由层检查)
|
||||
content_dicts = [p.model_dump(exclude_none=True) for p in req.content]
|
||||
task = await task_service.create_video_task(
|
||||
db=db,
|
||||
api_key_id=key.id,
|
||||
model_name=req.model,
|
||||
engine_id=engine_id,
|
||||
engine_snapshot=engine_snapshot,
|
||||
content=content_dicts,
|
||||
ratio=req.ratio,
|
||||
duration=req.duration,
|
||||
resolution=req.resolution,
|
||||
provider_generation_resolution=final_provider_resolution,
|
||||
upscale_enabled=upscale_enabled,
|
||||
upscale_snapshot_json=upscale_snapshot_json,
|
||||
idempotency_key=req.idempotency_key,
|
||||
local_media_refs=local_media_refs,
|
||||
)
|
||||
task.credits_cost = estimated_price # 记录预扣金额
|
||||
|
||||
# 根据并发限制决定立即执行还是排队
|
||||
if can_start:
|
||||
# 立即执行
|
||||
task.status = "pending"
|
||||
task.pipeline_stage = "queued"
|
||||
else:
|
||||
# 排队等待
|
||||
task.status = "queued"
|
||||
task.pipeline_stage = "waiting_concurrency"
|
||||
|
||||
await db.commit()
|
||||
|
||||
# 提交时即记录使用日志(配额已预扣)
|
||||
try:
|
||||
from app.services.api_v3.usage_log_service import record_usage
|
||||
quota_before = key.quota_used - estimated_price # 扣减前的余额
|
||||
quota_after = key.quota_used # 扣减后的余额
|
||||
price_detail = {
|
||||
"base_price": getattr(locals(), "base_price", 0),
|
||||
"per_second_price": getattr(locals(), "per_second_price", 0),
|
||||
"duration": req.duration,
|
||||
"resolution": req.resolution,
|
||||
"ratio": req.ratio,
|
||||
"total": estimated_price,
|
||||
}
|
||||
await record_usage(
|
||||
db=db,
|
||||
api_key_id=key.id,
|
||||
request_type="video_create",
|
||||
model_name=req.model,
|
||||
gen_type="video",
|
||||
status="success",
|
||||
task_id=task.id,
|
||||
credits_cost=estimated_price,
|
||||
price_action="deduct",
|
||||
resolution=req.resolution,
|
||||
duration=req.duration,
|
||||
quota_before=quota_before,
|
||||
quota_after=quota_after,
|
||||
price_detail_json=json.dumps(price_detail, ensure_ascii=False),
|
||||
)
|
||||
except Exception as log_exc:
|
||||
logger.error("Failed to record usage on submit: %s", log_exc)
|
||||
|
||||
# 记录模型调用日志
|
||||
log_model_request(
|
||||
engine_id=engine_id,
|
||||
model_name=req.model,
|
||||
task_id=task.id,
|
||||
params={
|
||||
"ratio": req.ratio,
|
||||
"duration": req.duration,
|
||||
"resolution": req.resolution,
|
||||
"generate_audio": req.generate_audio,
|
||||
"watermark": req.watermark,
|
||||
"content_count": len(req.content),
|
||||
"queued": not can_start,
|
||||
},
|
||||
)
|
||||
|
||||
# 5. 只有立即执行的才入队 Celery
|
||||
if can_start:
|
||||
from app.tasks.api_generation_tasks import api_create_generation_task
|
||||
api_create_generation_task.apply_async(
|
||||
args=[task.id],
|
||||
queue="gen_api_create",
|
||||
)
|
||||
|
||||
status_str = "queued" if can_start else "pending_queue"
|
||||
logger.info("API video task created: task_id=%s model=%s key=%s price=%.2f status=%s", task.id, req.model, key.id, estimated_price, status_str)
|
||||
|
||||
return ApiVideoCreateResponse(id=task.id)
|
||||
|
||||
|
||||
async def generate_image_sync(
|
||||
db: AsyncSession,
|
||||
key: ApiKey,
|
||||
callable_models: list[dict],
|
||||
req: "ApiImageGenerateRequest",
|
||||
start_time: float,
|
||||
) -> ApiImageGenerateResponse:
|
||||
"""同步生成图片。
|
||||
|
||||
流程:
|
||||
1. 解析引擎
|
||||
2. 创建任务记录
|
||||
3. 调用 Volcano Ark SDK(同步)
|
||||
4. 下载图片
|
||||
5. 更新任务状态
|
||||
6. 记录使用日志
|
||||
7. 返回结果
|
||||
"""
|
||||
from app.services.api_v3.usage_log_service import record_usage
|
||||
|
||||
# 1. 解析引擎
|
||||
engine_id, engine = await engine_service.resolve_engine_by_model_name(
|
||||
db, req.model, callable_models, "image"
|
||||
)
|
||||
engine_snapshot = engine_service.build_engine_snapshot(engine)
|
||||
|
||||
# 2. 创建任务记录
|
||||
task = await task_service.create_image_task(
|
||||
db=db,
|
||||
api_key_id=key.id,
|
||||
model_name=req.model,
|
||||
engine_id=engine_id,
|
||||
engine_snapshot=engine_snapshot,
|
||||
prompt=req.prompt,
|
||||
size=req.size,
|
||||
)
|
||||
|
||||
# 2.5 验证传入的媒体文件是否符合引擎配置要求
|
||||
# 校验参考图片数量限制
|
||||
input_image_count = len(req.image) if req.image else 0
|
||||
if input_image_count > (engine.max_reference_image_count or 0):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"该引擎最多支持 {engine.max_reference_image_count} 张参考图片,当前传入 {input_image_count} 张",
|
||||
)
|
||||
# 校验组图数量限制
|
||||
generation_count = req.generation_count or 1
|
||||
if generation_count > (engine.multi_image_max_images or 1):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"该引擎最多支持生成 {engine.multi_image_max_images} 张图片,当前请求 {generation_count} 张",
|
||||
)
|
||||
try:
|
||||
estimated_price = await calc_api_image_price(
|
||||
db,
|
||||
image_size=req.size or "2K",
|
||||
engine_id=engine_id,
|
||||
input_image_count=input_image_count,
|
||||
)
|
||||
except PricingNotConfiguredError as exc:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
)
|
||||
# 预检配额
|
||||
if key.quota_limit is not None and key.quota_used + estimated_price > key.quota_limit:
|
||||
from fastapi import HTTPException, status
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=f"配额不足 (需要 {estimated_price:.2f} 元, 剩余 {key.quota_limit - key.quota_used:.2f} 元)",
|
||||
)
|
||||
# 预扣配额
|
||||
if estimated_price > 0:
|
||||
key.quota_used = round((key.quota_used or 0.0) + estimated_price, 2)
|
||||
task.credits_cost = estimated_price
|
||||
await db.commit()
|
||||
|
||||
# 提交时即记录使用日志(配额已预扣)
|
||||
try:
|
||||
quota_before = key.quota_used - estimated_price
|
||||
quota_after = key.quota_used
|
||||
price_detail = {
|
||||
"base_price": getattr(locals(), "base_price", 0),
|
||||
"size": req.size,
|
||||
"generation_count": req.generation_count or 1,
|
||||
"total": estimated_price,
|
||||
}
|
||||
await record_usage(
|
||||
db=db,
|
||||
api_key_id=key.id,
|
||||
request_type="image_generate",
|
||||
model_name=req.model,
|
||||
gen_type="image",
|
||||
status="success",
|
||||
task_id=task.id,
|
||||
credits_cost=estimated_price,
|
||||
price_action="deduct",
|
||||
resolution=req.size,
|
||||
quota_before=quota_before,
|
||||
quota_after=quota_after,
|
||||
price_detail_json=json.dumps(price_detail, ensure_ascii=False),
|
||||
)
|
||||
except Exception as log_exc:
|
||||
logger.error("Failed to record usage on image submit: %s", log_exc)
|
||||
|
||||
try:
|
||||
# 3. 调用 Volcano Ark SDK(同步函数,在线程中执行)
|
||||
from app.services.image_gen import submit_image_task, download_image
|
||||
from app.config import settings
|
||||
import os
|
||||
|
||||
# 构建 media_references,下载图片到本地
|
||||
from app.services.api_v3.file_service import process_media_url
|
||||
|
||||
image_refs = []
|
||||
if req.image:
|
||||
for url in req.image:
|
||||
try:
|
||||
local_path = await process_media_url(url, "image")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to download image %s: %s", url[:80], exc)
|
||||
local_path = url
|
||||
image_refs.append({"type": "image", "url": local_path})
|
||||
|
||||
# 临时设置 media_references
|
||||
task.media_references = json.dumps(image_refs, ensure_ascii=False) if image_refs else None
|
||||
task.image_size = req.size or "2K"
|
||||
await db.flush()
|
||||
|
||||
# 在线程中执行同步 SDK 调用
|
||||
result = await asyncio.to_thread(
|
||||
submit_image_task,
|
||||
db,
|
||||
engine,
|
||||
task,
|
||||
True, # include_media_references
|
||||
req.generation_count or 1,
|
||||
)
|
||||
|
||||
# 4. 下载图片
|
||||
items = result.get("items", [])
|
||||
downloaded_items: list[ApiImageGenerateDataItem] = []
|
||||
|
||||
for item in items:
|
||||
url = item.get("remote_result_url")
|
||||
if url:
|
||||
# 下载到本地
|
||||
date_dir = datetime.now().strftime("%Y%m%d")
|
||||
dest_dir = f"./storage/generate/api/images/{date_dir}"
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
dest_path = os.path.join(dest_dir, f"{task.id}_{item.get('generation_index', 1)}.png")
|
||||
try:
|
||||
await download_image(url, dest_path)
|
||||
except Exception as dl_err:
|
||||
logger.warning("Image download failed: %s", dl_err)
|
||||
|
||||
downloaded_items.append(ApiImageGenerateDataItem(
|
||||
url=dest_path, # 使用本地路径
|
||||
size=item.get("size"),
|
||||
output_format=item.get("output_format"),
|
||||
))
|
||||
elif item.get("error_message"):
|
||||
downloaded_items.append(ApiImageGenerateDataItem(
|
||||
url=None,
|
||||
))
|
||||
|
||||
# 5. 使用预扣金额(不再重复扣减)
|
||||
task.credits_cost = estimated_price
|
||||
|
||||
# 6. 更新任务状态
|
||||
task.status = "completed"
|
||||
task.pipeline_stage = "done"
|
||||
task.generated_at = datetime.now(timezone.utc)
|
||||
if downloaded_items and downloaded_items[0].url:
|
||||
task.image_url = downloaded_items[0].url
|
||||
await db.commit()
|
||||
|
||||
# 提交时已记录使用日志,成功时无需重复记录
|
||||
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||
|
||||
return ApiImageGenerateResponse(
|
||||
created=result.get("created", int(time.time())),
|
||||
data=downloaded_items,
|
||||
model=result.get("model", req.model),
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
# 失败:退回预扣配额
|
||||
if estimated_price > 0:
|
||||
key.quota_used = round(max(0, (key.quota_used or 0.0) - estimated_price), 2)
|
||||
|
||||
task.status = "failed"
|
||||
task.error_message = str(exc)
|
||||
task.credits_cost = 0 # 实际消耗为0(已退回)
|
||||
await db.commit()
|
||||
|
||||
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||
quota_after_refund = key.quota_used # 退回后的余额
|
||||
await record_usage(
|
||||
db=db,
|
||||
api_key_id=key.id,
|
||||
request_type="image_generate",
|
||||
model_name=req.model,
|
||||
gen_type="image",
|
||||
status="failed",
|
||||
task_id=task.id,
|
||||
credits_cost=estimated_price,
|
||||
refund_amount=estimated_price,
|
||||
request_duration_ms=duration_ms,
|
||||
error_message=str(exc),
|
||||
error_code="generation_failed",
|
||||
price_action="refund",
|
||||
resolution=req.size,
|
||||
generation_count=req.generation_count or 1,
|
||||
quota_before=quota_after_refund,
|
||||
quota_after=quota_after_refund + estimated_price,
|
||||
)
|
||||
await db.commit()
|
||||
raise
|
||||
@@ -0,0 +1,151 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_key import ApiKey
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
API_KEY_PREFIX = "vk_"
|
||||
|
||||
|
||||
async def create_api_key(
|
||||
db: AsyncSession,
|
||||
company_name: str,
|
||||
callable_models: list[dict] | None = None,
|
||||
quota_limit: float | None = None,
|
||||
quota_cycle: str | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
valid_until: datetime | None = None,
|
||||
max_concurrent_video_tasks: int | None = None,
|
||||
description: str | None = None,
|
||||
) -> tuple[ApiKey, str]:
|
||||
"""创建新的 API Key。
|
||||
|
||||
Returns:
|
||||
(ApiKey 对象, 明文 API Key) — 明文仅返回这一次。
|
||||
"""
|
||||
# 生成密钥: vk_ + 32字节随机hex
|
||||
raw_key = API_KEY_PREFIX + secrets.token_hex(24) # vk_ + 48位hex = 51字符
|
||||
key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
|
||||
key_prefix = raw_key[:8] # 前8位用于展示: vk_xxxxx
|
||||
|
||||
api_key = ApiKey(
|
||||
id=generate_id(),
|
||||
company_name=company_name,
|
||||
api_key_hash=key_hash,
|
||||
api_key_prefix=key_prefix,
|
||||
description=description,
|
||||
callable_models=json.dumps(callable_models or [], ensure_ascii=False),
|
||||
quota_limit=quota_limit,
|
||||
quota_cycle=quota_cycle,
|
||||
quota_used=0.0,
|
||||
valid_from=valid_from,
|
||||
valid_until=valid_until,
|
||||
max_concurrent_video_tasks=max_concurrent_video_tasks,
|
||||
is_active=True,
|
||||
)
|
||||
api_key.set_plaintext_key(raw_key) # 加密存储完整 Key
|
||||
db.add(api_key)
|
||||
await db.flush()
|
||||
|
||||
logger.info("API Key created: id=%s company=%s prefix=%s", api_key.id, company_name, key_prefix)
|
||||
return api_key, raw_key
|
||||
|
||||
|
||||
async def list_api_keys(
|
||||
db: AsyncSession,
|
||||
skip: int = 0,
|
||||
limit: int = 50,
|
||||
company_name: str | None = None,
|
||||
is_active: bool | None = None,
|
||||
) -> tuple[int, list[ApiKey]]:
|
||||
"""列出 API Key(分页+筛选)。"""
|
||||
from sqlalchemy import func
|
||||
|
||||
query = select(ApiKey).where(ApiKey.deleted_at.is_(None))
|
||||
count_query = select(func.count(ApiKey.id)).where(ApiKey.deleted_at.is_(None))
|
||||
|
||||
if company_name:
|
||||
query = query.where(ApiKey.company_name.ilike(f"%{company_name}%"))
|
||||
count_query = count_query.where(ApiKey.company_name.ilike(f"%{company_name}%"))
|
||||
if is_active is not None:
|
||||
query = query.where(ApiKey.is_active == is_active)
|
||||
count_query = count_query.where(ApiKey.is_active == is_active)
|
||||
|
||||
total_result = await db.execute(count_query)
|
||||
total = total_result.scalar_one()
|
||||
|
||||
query = query.order_by(ApiKey.created_at.desc()).offset(skip).limit(limit)
|
||||
result = await db.execute(query)
|
||||
keys = list(result.scalars().all())
|
||||
|
||||
return total, keys
|
||||
|
||||
|
||||
async def get_api_key(db: AsyncSession, key_id: str) -> ApiKey | None:
|
||||
"""获取单个 API Key 详情。"""
|
||||
result = await db.execute(
|
||||
select(ApiKey).where(ApiKey.id == key_id, ApiKey.deleted_at.is_(None)).limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def update_api_key(db: AsyncSession, key: ApiKey, **kwargs) -> ApiKey:
|
||||
"""更新 API Key 配置。"""
|
||||
updatable_fields = {
|
||||
"company_name", "description", "callable_models",
|
||||
"quota_limit", "quota_cycle", "valid_from", "valid_until",
|
||||
"max_concurrent_video_tasks", "is_active",
|
||||
}
|
||||
for field, value in kwargs.items():
|
||||
if field in updatable_fields and value is not None:
|
||||
if field == "callable_models" and isinstance(value, list):
|
||||
value = json.dumps(value, ensure_ascii=False)
|
||||
setattr(key, field, value)
|
||||
|
||||
await db.flush()
|
||||
return key
|
||||
|
||||
|
||||
async def delete_api_key(db: AsyncSession, key: ApiKey) -> None:
|
||||
"""软删除 API Key。"""
|
||||
key.deleted_at = datetime.now(timezone.utc)
|
||||
key.is_active = False
|
||||
await db.flush()
|
||||
|
||||
|
||||
async def reset_quota_if_needed(db: AsyncSession, key: ApiKey) -> ApiKey:
|
||||
"""检查并重置过期周期的配额。
|
||||
|
||||
- daily: 如果上次重置不是今天,重置 quota_used=0
|
||||
- monthly: 如果上次重置不是本月,重置 quota_used=0
|
||||
"""
|
||||
if key.quota_limit is None or key.quota_cycle is None:
|
||||
return key
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# 使用 quota_used 的 updated_at 作为周期判断依据
|
||||
last_reset = key.updated_at or key.created_at
|
||||
if last_reset is None:
|
||||
return key
|
||||
|
||||
should_reset = False
|
||||
if key.quota_cycle == "daily":
|
||||
should_reset = last_reset.date() < now.date()
|
||||
elif key.quota_cycle == "monthly":
|
||||
should_reset = (last_reset.year, last_reset.month) < (now.year, now.month)
|
||||
|
||||
if should_reset and key.quota_used > 0:
|
||||
key.quota_used = 0.0
|
||||
await db.flush()
|
||||
logger.info("Quota reset for API Key %s (cycle=%s)", key.id, key.quota_cycle)
|
||||
|
||||
return key
|
||||
@@ -0,0 +1,187 @@
|
||||
"""外部 API v3 日志服务。
|
||||
|
||||
按天分类存储在 log/api/ 目录下:
|
||||
- log/api/requests/YYYY-MM-DD.log — 所有外部请求和响应
|
||||
- log/api/models/YYYY-MM-DD.log — 模型调用(Volcano Ark SDK)
|
||||
- log/api/upscale/YYYY-MM-DD.log — 超分轮询
|
||||
- log/api/errors/YYYY-MM-DD.log — 错误日志
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# === 日志目录 ===
|
||||
# video-gen-api/log/api/
|
||||
_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
# 上溯3级: services/api_v3 -> services -> app -> video-gen-api (即项目根目录)
|
||||
_BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_THIS_DIR)))
|
||||
BASE_LOG_DIR = os.path.join(_BASE_DIR, "log", "api")
|
||||
os.makedirs(BASE_LOG_DIR, exist_ok=True)
|
||||
|
||||
# 子目录
|
||||
REQUESTS_LOG_DIR = os.path.join(BASE_LOG_DIR, "requests")
|
||||
MODELS_LOG_DIR = os.path.join(BASE_LOG_DIR, "models")
|
||||
UPSCALE_LOG_DIR = os.path.join(BASE_LOG_DIR, "upscale")
|
||||
ERRORS_LOG_DIR = os.path.join(BASE_LOG_DIR, "errors")
|
||||
|
||||
for d in [REQUESTS_LOG_DIR, MODELS_LOG_DIR, UPSCALE_LOG_DIR, ERRORS_LOG_DIR]:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
|
||||
def _get_date_str() -> str:
|
||||
"""获取当前日期字符串。"""
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
class _DailyFileHandler(logging.Handler):
|
||||
"""按天写入的日志处理器。"""
|
||||
|
||||
def __init__(self, log_dir: str):
|
||||
super().__init__()
|
||||
self.log_dir = log_dir
|
||||
self._current_date = None
|
||||
self._file_handler = None
|
||||
self._open_file()
|
||||
|
||||
def _open_file(self):
|
||||
"""打开当天的日志文件。"""
|
||||
date_str = _get_date_str()
|
||||
if date_str == self._current_date and self._file_handler:
|
||||
return
|
||||
|
||||
if self._file_handler:
|
||||
self._file_handler.close()
|
||||
|
||||
self._current_date = date_str
|
||||
filepath = os.path.join(self.log_dir, f"{date_str}.log")
|
||||
self._file_handler = logging.FileHandler(filepath, encoding="utf-8")
|
||||
self._file_handler.setFormatter(
|
||||
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
|
||||
)
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
self._open_file()
|
||||
self._file_handler.emit(record)
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
def close(self):
|
||||
if self._file_handler:
|
||||
self._file_handler.close()
|
||||
super().close()
|
||||
|
||||
|
||||
def _create_logger(name: str, log_dir: str) -> logging.Logger:
|
||||
"""创建按天写入的 Logger。"""
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
# 避免重复添加 handler
|
||||
if not logger.handlers:
|
||||
handler = _DailyFileHandler(log_dir)
|
||||
logger.addHandler(handler)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
# === Logger 实例 ===
|
||||
requests_logger = _create_logger("api_v3.requests", REQUESTS_LOG_DIR)
|
||||
models_logger = _create_logger("api_v3.models", MODELS_LOG_DIR)
|
||||
upscale_logger = _create_logger("api_v3.upscale", UPSCALE_LOG_DIR)
|
||||
errors_logger = _create_logger("api_v3.errors", ERRORS_LOG_DIR)
|
||||
|
||||
|
||||
def _safe_json(obj) -> str:
|
||||
"""安全地序列化为 JSON。"""
|
||||
try:
|
||||
return json.dumps(obj, ensure_ascii=False, default=str)
|
||||
except Exception:
|
||||
return str(obj)
|
||||
|
||||
|
||||
# === 请求/响应日志 ===
|
||||
|
||||
def log_request(method: str, path: str, api_key_id: str, body: dict | None = None):
|
||||
"""记录外部请求。"""
|
||||
requests_logger.info(
|
||||
f"REQUEST | {method} {path} | key={api_key_id} | body={_safe_json(body)}"
|
||||
)
|
||||
|
||||
|
||||
def log_response(method: str, path: str, api_key_id: str, status_code: int, body=None, duration_ms: int = 0):
|
||||
"""记录外部响应。"""
|
||||
requests_logger.info(
|
||||
f"RESPONSE | {method} {path} | key={api_key_id} | status={status_code} | duration={duration_ms}ms | body={_safe_json(body)}"
|
||||
)
|
||||
|
||||
|
||||
def log_request_error(method: str, path: str, api_key_id: str, error: str, status_code: int = 500):
|
||||
"""记录请求错误。"""
|
||||
errors_logger.error(
|
||||
f"REQUEST_ERROR | {method} {path} | key={api_key_id} | status={status_code} | error={error}"
|
||||
)
|
||||
|
||||
|
||||
# === 模型调用日志 ===
|
||||
|
||||
def log_model_request(engine_id: str, model_name: str, task_id: str, params: dict):
|
||||
"""记录模型调用请求。"""
|
||||
models_logger.info(
|
||||
f"MODEL_REQUEST | engine={engine_id} | model={model_name} | task={task_id} | params={_safe_json(params)}"
|
||||
)
|
||||
|
||||
|
||||
def log_model_response(engine_id: str, model_name: str, task_id: str, success: bool, result: dict | None = None, error: str | None = None):
|
||||
"""记录模型调用响应。"""
|
||||
if success:
|
||||
models_logger.info(
|
||||
f"MODEL_RESPONSE | engine={engine_id} | model={model_name} | task={task_id} | success | result={_safe_json(result)}"
|
||||
)
|
||||
else:
|
||||
models_logger.error(
|
||||
f"MODEL_RESPONSE | engine={engine_id} | model={model_name} | task={task_id} | failed | error={error}"
|
||||
)
|
||||
errors_logger.error(
|
||||
f"MODEL_ERROR | engine={engine_id} | model={model_name} | task={task_id} | error={error}"
|
||||
)
|
||||
|
||||
|
||||
# === 超分轮询日志 ===
|
||||
|
||||
def log_upscale_poll_start(task_id: str, api_task_id: str):
|
||||
"""记录超分轮询开始。"""
|
||||
upscale_logger.info(f"UPSCALE_POLL_START | task={task_id} | api_task={api_task_id}")
|
||||
|
||||
|
||||
def log_upscale_poll(task_id: str, api_task_id: str, status: str, attempt: int, result: dict | None = None):
|
||||
"""记录超分轮询状态。"""
|
||||
upscale_logger.info(
|
||||
f"UPSCALE_POLL | task={task_id} | api_task={api_task_id} | status={status} | attempt={attempt} | result={_safe_json(result)}"
|
||||
)
|
||||
|
||||
|
||||
def log_upscale_poll_end(task_id: str, api_task_id: str, success: bool, final_status: str, total_attempts: int):
|
||||
"""记录超分轮询结束。"""
|
||||
if success:
|
||||
upscale_logger.info(
|
||||
f"UPSCALE_POLL_END | task={task_id} | api_task={api_task_id} | success | status={final_status} | attempts={total_attempts}"
|
||||
)
|
||||
else:
|
||||
upscale_logger.error(
|
||||
f"UPSCALE_POLL_END | task={task_id} | api_task={api_task_id} | failed | status={final_status} | attempts={total_attempts}"
|
||||
)
|
||||
errors_logger.error(
|
||||
f"UPSCALE_ERROR | task={task_id} | api_task={api_task_id} | status={final_status} | attempts={total_attempts}"
|
||||
)
|
||||
|
||||
|
||||
# === 通用错误日志 ===
|
||||
|
||||
def log_error(category: str, message: str, details: dict | None = None):
|
||||
"""记录通用错误。"""
|
||||
errors_logger.error(
|
||||
f"{category} | {message} | details={_safe_json(details)}"
|
||||
)
|
||||
@@ -0,0 +1,173 @@
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_model_pricing import ApiModelPricing
|
||||
from app.models.image_engine import ImageEngine
|
||||
from app.models.video_engine import VideoEngine
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
class PricingNotConfiguredError(Exception):
|
||||
"""模型+分辨率组合未配置价格。"""
|
||||
|
||||
def __init__(self, *, model_name: str, resolution: str):
|
||||
self.model_name = model_name
|
||||
self.resolution = resolution
|
||||
super().__init__(
|
||||
f"模型或引擎 '{self.model_name}' 在分辨率 '{self.resolution}' 下未配置,无法生成"
|
||||
)
|
||||
|
||||
|
||||
async def resolve_engine_display_name(db: AsyncSession, engine_id: str) -> str:
|
||||
"""根据引擎 ID 解析展示名称(找不到时原样返回 ID)。"""
|
||||
if not engine_id:
|
||||
return engine_id or "unknown"
|
||||
result = await db.execute(
|
||||
select(VideoEngine.name).where(VideoEngine.id == engine_id, VideoEngine.deleted_at.is_(None)).limit(1)
|
||||
)
|
||||
name = result.scalar_one_or_none()
|
||||
if name:
|
||||
return name
|
||||
result = await db.execute(
|
||||
select(ImageEngine.name).where(ImageEngine.id == engine_id, ImageEngine.deleted_at.is_(None)).limit(1)
|
||||
)
|
||||
name = result.scalar_one_or_none()
|
||||
return name or engine_id
|
||||
|
||||
|
||||
async def _get_api_pricing(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
gen_type: str,
|
||||
resolution: str,
|
||||
engine_id: str | None = None,
|
||||
) -> ApiModelPricing | None:
|
||||
"""按引擎精确规则优先获取定价;找不到时回退到同类型同分辨率。
|
||||
|
||||
查询优先级:
|
||||
1. gen_type + engine_id + resolution 精确规则
|
||||
2. gen_type + resolution 下 base_price 最高规则
|
||||
"""
|
||||
gen_type = (gen_type or "").lower().strip()
|
||||
resolution = (resolution or "").strip()
|
||||
engine_id = (engine_id or "").strip() or None
|
||||
|
||||
if engine_id:
|
||||
result = await db.execute(
|
||||
select(ApiModelPricing)
|
||||
.where(ApiModelPricing.gen_type == gen_type)
|
||||
.where(ApiModelPricing.model_config_id == engine_id)
|
||||
.where(ApiModelPricing.resolution == resolution)
|
||||
.order_by(ApiModelPricing.base_price.desc(), ApiModelPricing.per_second_price.desc())
|
||||
.limit(1)
|
||||
)
|
||||
pricing = result.scalar_one_or_none()
|
||||
if pricing:
|
||||
return pricing
|
||||
|
||||
result = await db.execute(
|
||||
select(ApiModelPricing)
|
||||
.where(ApiModelPricing.gen_type == gen_type)
|
||||
.where(ApiModelPricing.resolution == resolution)
|
||||
.order_by(ApiModelPricing.base_price.desc(), ApiModelPricing.per_second_price.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def calc_api_video_price(
|
||||
db: AsyncSession,
|
||||
duration: int,
|
||||
resolution: str,
|
||||
engine_id: str | None = None,
|
||||
input_video_duration: float = 0,
|
||||
input_image_count: int = 0,
|
||||
) -> float:
|
||||
"""计算 API 视频生成价格(元)。
|
||||
|
||||
未配置价格时抛出 PricingNotConfiguredError。
|
||||
|
||||
公式(与 credit_ratios 一致):
|
||||
base_cost = (base_price + per_second_price × duration) × price_ratio
|
||||
if 传入视频: += (input_video_base_price + input_video_per_second_price × input_video_duration) × input_video_ratio
|
||||
if 传入图片: += (input_image_base_price + input_image_per_image_price × input_image_count) × input_image_ratio
|
||||
"""
|
||||
if not engine_id:
|
||||
result = await db.execute(
|
||||
select(VideoEngine.id)
|
||||
.where(VideoEngine.is_active == True, VideoEngine.deleted_at.is_(None))
|
||||
.order_by(VideoEngine.priority.desc())
|
||||
.limit(1)
|
||||
)
|
||||
engine_id = result.scalar_one_or_none()
|
||||
|
||||
pricing = await _get_api_pricing(db, gen_type="video", resolution=resolution, engine_id=engine_id)
|
||||
|
||||
if not pricing:
|
||||
raise PricingNotConfiguredError(
|
||||
model_name=await resolve_engine_display_name(db, engine_id),
|
||||
resolution=resolution,
|
||||
)
|
||||
|
||||
# 基础价格
|
||||
base_cost = (pricing.base_price + pricing.per_second_price * duration) * pricing.price_ratio
|
||||
# 传入视频附加费(每秒 × 倍率)
|
||||
if input_video_duration > 0:
|
||||
base_cost += (pricing.input_video_base_price + pricing.input_video_per_second_price * input_video_duration) * pricing.input_video_ratio
|
||||
# 传入图片附加费(每张 × 倍率)
|
||||
if input_image_count > 0:
|
||||
base_cost += (pricing.input_image_base_price + pricing.input_image_per_image_price * input_image_count) * pricing.input_image_ratio
|
||||
return round(base_cost, 2)
|
||||
|
||||
|
||||
async def calc_api_image_price(
|
||||
db: AsyncSession,
|
||||
image_size: str,
|
||||
engine_id: str | None = None,
|
||||
input_image_count: int = 0,
|
||||
) -> float:
|
||||
"""计算 API 图片生成价格(元)。
|
||||
|
||||
未配置价格时抛出 PricingNotConfiguredError。
|
||||
|
||||
公式(与 credit_ratios 一致):
|
||||
base_cost = base_price × price_ratio
|
||||
if 传入图片: += (input_image_base_price + input_image_per_image_price × input_image_count) × input_image_ratio
|
||||
"""
|
||||
if not engine_id:
|
||||
result = await db.execute(
|
||||
select(ImageEngine.id)
|
||||
.where(ImageEngine.is_active == True, ImageEngine.deleted_at.is_(None))
|
||||
.order_by(ImageEngine.priority.desc())
|
||||
.limit(1)
|
||||
)
|
||||
engine_id = result.scalar_one_or_none()
|
||||
|
||||
pricing = await _get_api_pricing(db, gen_type="image", resolution=image_size, engine_id=engine_id)
|
||||
|
||||
if not pricing:
|
||||
raise PricingNotConfiguredError(
|
||||
model_name=await resolve_engine_display_name(db, engine_id),
|
||||
resolution=image_size,
|
||||
)
|
||||
|
||||
# 基础价格
|
||||
base_cost = pricing.base_price * pricing.price_ratio
|
||||
# 传入图片附加费(每张 × 倍率)
|
||||
if input_image_count > 0:
|
||||
base_cost += (pricing.input_image_base_price + pricing.input_image_per_image_price * input_image_count) * pricing.input_image_ratio
|
||||
return round(base_cost, 2)
|
||||
|
||||
|
||||
async def get_priced_models(db: AsyncSession) -> set[str]:
|
||||
"""获取所有已配置价格的引擎 ID 集合。
|
||||
|
||||
用于过滤 /api/v3/models 接口,仅返回已定价的模型。
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(ApiModelPricing.model_config_id).distinct()
|
||||
)
|
||||
return {row[0] for row in result.all()}
|
||||
@@ -0,0 +1,68 @@
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_generation_task import ApiGenerationTask
|
||||
from app.models.api.api_key import ApiKey
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
async def check_quota(key: ApiKey) -> bool:
|
||||
"""检查 API Key 配额是否充足。
|
||||
|
||||
Returns:
|
||||
True = 配额充足或无限额, False = 已超限。
|
||||
"""
|
||||
if key.quota_limit is None:
|
||||
return True
|
||||
return key.quota_used < key.quota_limit
|
||||
|
||||
|
||||
async def get_active_video_tasks_count(api_key_id: str, db: AsyncSession) -> int:
|
||||
"""统计 API Key 当前活跃的视频任务数。
|
||||
|
||||
活跃 = status IN ('pending', 'generating', 'processing') AND gen_type='video'
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(func.count(ApiGenerationTask.id)).where(
|
||||
ApiGenerationTask.api_key_id == api_key_id,
|
||||
ApiGenerationTask.gen_type == "video",
|
||||
ApiGenerationTask.status.in_(["pending", "generating", "processing"]),
|
||||
ApiGenerationTask.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return result.scalar_one() or 0
|
||||
|
||||
|
||||
async def can_start_video_task(key: ApiKey, db: AsyncSession) -> bool:
|
||||
"""检查是否可以立即启动新的视频任务。
|
||||
|
||||
Returns:
|
||||
True = 可以立即启动, False = 需要排队。
|
||||
"""
|
||||
if key.max_concurrent_video_tasks is None:
|
||||
return True # 无限制
|
||||
current = await get_active_video_tasks_count(key.id, db)
|
||||
return current < key.max_concurrent_video_tasks
|
||||
|
||||
|
||||
async def get_queued_video_tasks(key: ApiKey, db: AsyncSession, limit: int = 10) -> list[ApiGenerationTask]:
|
||||
"""获取排队的视频任务列表(按创建时间排序)。"""
|
||||
result = await db.execute(
|
||||
select(ApiGenerationTask).where(
|
||||
ApiGenerationTask.api_key_id == key.id,
|
||||
ApiGenerationTask.gen_type == "video",
|
||||
ApiGenerationTask.status == "queued",
|
||||
ApiGenerationTask.deleted_at.is_(None),
|
||||
).order_by(ApiGenerationTask.created_at.asc()).limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def increment_quota(db: AsyncSession, key: ApiKey, credits_cost: float) -> None:
|
||||
"""原子性增加配额使用量。"""
|
||||
key.quota_used = round((key.quota_used or 0.0) + credits_cost, 2)
|
||||
await db.flush()
|
||||
@@ -0,0 +1,215 @@
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_generation_task import ApiGenerationTask
|
||||
from app.schemas.api_v3.video import ApiVideoStatusResponse
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
async def create_video_task(
|
||||
db: AsyncSession,
|
||||
api_key_id: str,
|
||||
model_name: str,
|
||||
engine_id: str,
|
||||
engine_snapshot: dict,
|
||||
content: list[dict],
|
||||
ratio: str | None,
|
||||
duration: int | None,
|
||||
resolution: str | None,
|
||||
provider_generation_resolution: str | None,
|
||||
upscale_enabled: bool,
|
||||
upscale_snapshot_json: str | None,
|
||||
idempotency_key: str | None = None,
|
||||
local_media_refs: list[dict] | None = None,
|
||||
) -> ApiGenerationTask:
|
||||
"""创建视频生成任务记录。"""
|
||||
# 提取文本提示词
|
||||
text_parts = [p.get("text", "") for p in content if p.get("type") == "text"]
|
||||
original_prompt = " ".join(text_parts) if text_parts else content[0].get("text", "") if content else ""
|
||||
|
||||
# 构建 media_references(扁平格式,便于外部读取)
|
||||
# 构建 local_media_json(嵌套格式,与 Volcano SDK 兼容)
|
||||
from app.services.api_v3.file_service import process_media_url
|
||||
|
||||
media_refs = [] # 扁平格式: {"type": "image", "url": "...", "role": "..."}
|
||||
local_media_refs = [] # 本地下载路径(嵌套格式)
|
||||
|
||||
for p in content:
|
||||
ptype = p.get("type", "")
|
||||
if ptype == "text":
|
||||
continue
|
||||
|
||||
# 提取原始 URL(从嵌套格式中提取)
|
||||
original_url = ""
|
||||
media_type = ptype.replace("_url", "") # image_url -> image
|
||||
if ptype == "image_url" and p.get("image_url"):
|
||||
original_url = p["image_url"].get("url", "")
|
||||
elif ptype == "video_url" and p.get("video_url"):
|
||||
original_url = p["video_url"].get("url", "")
|
||||
elif ptype == "audio_url" and p.get("audio_url"):
|
||||
original_url = p["audio_url"].get("url", "")
|
||||
|
||||
# 存储扁平格式到 media_references
|
||||
media_refs.append({
|
||||
"type": media_type,
|
||||
"url": original_url,
|
||||
"role": p.get("role"),
|
||||
})
|
||||
|
||||
# 下载文件到本地
|
||||
try:
|
||||
local_path = await process_media_url(original_url, media_type)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to download media %s: %s", original_url[:80], exc)
|
||||
local_path = original_url
|
||||
|
||||
# 本地路径使用嵌套格式(与 Volcano SDK 兼容)
|
||||
local_media_refs.append({
|
||||
"type": ptype,
|
||||
ptype: {"url": local_path},
|
||||
"role": p.get("role"),
|
||||
})
|
||||
|
||||
media_references_json = json.dumps(media_refs, ensure_ascii=False) if media_refs else None
|
||||
local_media_json = json.dumps(local_media_refs, ensure_ascii=False) if local_media_refs else None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
deadline = now + timedelta(hours=24)
|
||||
|
||||
task = ApiGenerationTask(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
external_idempotency_key=idempotency_key,
|
||||
original_prompt=original_prompt,
|
||||
gen_type="video",
|
||||
model_name=model_name,
|
||||
duration=duration,
|
||||
aspect_ratio=ratio,
|
||||
resolution=resolution,
|
||||
provider_generation_resolution=provider_generation_resolution,
|
||||
generation_count=1,
|
||||
engine_id=engine_id,
|
||||
media_references=media_references_json,
|
||||
local_media_json=local_media_json,
|
||||
engine_snapshot_json=json.dumps(engine_snapshot, ensure_ascii=False),
|
||||
status="pending",
|
||||
pipeline_stage="queued",
|
||||
deadline_at=deadline,
|
||||
video_upscale_enabled_snapshot=upscale_enabled,
|
||||
video_upscale_snapshot_json=upscale_snapshot_json,
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
return task
|
||||
|
||||
|
||||
async def create_image_task(
|
||||
db: AsyncSession,
|
||||
api_key_id: str,
|
||||
model_name: str,
|
||||
engine_id: str,
|
||||
engine_snapshot: dict,
|
||||
prompt: str,
|
||||
size: str | None,
|
||||
idempotency_key: str | None = None,
|
||||
) -> ApiGenerationTask:
|
||||
"""创建图片生成任务记录。"""
|
||||
task = ApiGenerationTask(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
external_idempotency_key=idempotency_key,
|
||||
original_prompt=prompt,
|
||||
gen_type="image",
|
||||
image_size=size,
|
||||
generation_count=1,
|
||||
engine_id=engine_id,
|
||||
engine_snapshot_json=json.dumps(engine_snapshot, ensure_ascii=False),
|
||||
status="processing",
|
||||
pipeline_stage="creating_provider_task",
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
return task
|
||||
|
||||
|
||||
async def get_task(db: AsyncSession, task_id: str, api_key_id: str) -> ApiGenerationTask | None:
|
||||
"""获取任务(带所有权验证)。"""
|
||||
result = await db.execute(
|
||||
select(ApiGenerationTask).where(
|
||||
ApiGenerationTask.id == task_id,
|
||||
ApiGenerationTask.api_key_id == api_key_id,
|
||||
ApiGenerationTask.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def find_by_idempotency_key(db: AsyncSession, api_key_id: str, idempotency_key: str) -> ApiGenerationTask | None:
|
||||
"""根据幂等键查找已存在的任务。"""
|
||||
result = await db.execute(
|
||||
select(ApiGenerationTask).where(
|
||||
ApiGenerationTask.api_key_id == api_key_id,
|
||||
ApiGenerationTask.external_idempotency_key == idempotency_key,
|
||||
ApiGenerationTask.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def map_task_to_status_response(task: ApiGenerationTask) -> ApiVideoStatusResponse:
|
||||
"""将任务对象映射为状态查询响应。"""
|
||||
from app.config import settings
|
||||
# 返回完整 URL(包含 BASE_URL)
|
||||
video_url = _make_full_url(task.video_url)
|
||||
video_cover_url = _make_full_url(task.video_cover_url)
|
||||
return ApiVideoStatusResponse(
|
||||
task_id=task.id,
|
||||
status=_map_status(task.status),
|
||||
video_url=video_url,
|
||||
video_cover_url=video_cover_url,
|
||||
duration=task.duration,
|
||||
ratio=task.aspect_ratio,
|
||||
resolution=task.resolution,
|
||||
error=task.error_message,
|
||||
created_at=task.created_at,
|
||||
completed_at=task.generated_at,
|
||||
)
|
||||
|
||||
|
||||
def _make_full_url(path: str | None) -> str | None:
|
||||
"""将本地路径转换为完整 URL。"""
|
||||
if not path:
|
||||
return None
|
||||
from app.config import settings
|
||||
# 如果已经是完整 URL,直接返回
|
||||
if path.startswith(("http://", "https://")):
|
||||
return path
|
||||
# 处理 ./storage/generate/... 格式 → /generate/...
|
||||
if path.startswith("./storage"):
|
||||
url_path = path[len("./storage"):]
|
||||
elif path.startswith("/"):
|
||||
url_path = path
|
||||
else:
|
||||
url_path = f"/{path}"
|
||||
# 拼接 BASE_URL
|
||||
base = settings.BASE_URL.rstrip("/")
|
||||
return f"{base}{url_path}"
|
||||
|
||||
|
||||
def _map_status(status: str) -> str:
|
||||
"""将内部状态映射为 API 状态。"""
|
||||
status_map = {
|
||||
"pending": "queued",
|
||||
"queued": "pending_queue",
|
||||
"generating": "generating",
|
||||
"processing": "generating",
|
||||
"completed": "completed",
|
||||
"failed": "failed",
|
||||
}
|
||||
return status_map.get(status, status)
|
||||
@@ -0,0 +1,212 @@
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_generation_task import ApiGenerationTask
|
||||
from app.models.api.api_key_upscale_config import ApiKeyUpscaleConfig
|
||||
from app.models.api.api_upscale_link import ApiUpscaleLink
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
async def get_or_create_upscale_config(
|
||||
db: AsyncSession,
|
||||
api_key_id: str,
|
||||
) -> ApiKeyUpscaleConfig:
|
||||
"""获取或创建 API Key 的超分配置。"""
|
||||
result = await db.execute(
|
||||
select(ApiKeyUpscaleConfig).where(
|
||||
ApiKeyUpscaleConfig.api_key_id == api_key_id
|
||||
).limit(1)
|
||||
)
|
||||
config = result.scalar_one_or_none()
|
||||
|
||||
if not config:
|
||||
config = ApiKeyUpscaleConfig(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
enabled=False,
|
||||
delete_source_after_success=True,
|
||||
rules_json="[]",
|
||||
)
|
||||
db.add(config)
|
||||
await db.flush()
|
||||
|
||||
return config
|
||||
|
||||
|
||||
async def save_upscale_config(
|
||||
db: AsyncSession,
|
||||
api_key_id: str,
|
||||
enabled: bool,
|
||||
delete_source_after_success: bool,
|
||||
rules: list[dict],
|
||||
) -> ApiKeyUpscaleConfig:
|
||||
"""保存 API Key 的超分配置。"""
|
||||
config = await get_or_create_upscale_config(db, api_key_id)
|
||||
config.enabled = enabled
|
||||
config.delete_source_after_success = delete_source_after_success
|
||||
config.rules_json = json.dumps(rules, ensure_ascii=False)
|
||||
await db.flush()
|
||||
return config
|
||||
|
||||
|
||||
async def build_api_upscale_snapshot(
|
||||
db: AsyncSession,
|
||||
api_key_id: str,
|
||||
target_resolution: str,
|
||||
aspect_ratio: str | None = None,
|
||||
) -> tuple[str | None, bool, str | None]:
|
||||
"""构建 API 超分快照。
|
||||
|
||||
读取 api_key_upscale_configs(而非 system_configs),
|
||||
匹配目标分辨率对应的超分规则。
|
||||
|
||||
Returns:
|
||||
(provider_generation_resolution, enabled, snapshot_json)
|
||||
"""
|
||||
config = await get_or_create_upscale_config(db, api_key_id)
|
||||
|
||||
if not config.enabled:
|
||||
return None, False, None
|
||||
|
||||
try:
|
||||
rules = json.loads(config.rules_json) if config.rules_json else []
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None, False, None
|
||||
|
||||
# 匹配规则
|
||||
matched_rule = None
|
||||
for rule in rules:
|
||||
if rule.get("enabled") and rule.get("target_resolution") == target_resolution:
|
||||
matched_rule = rule
|
||||
break
|
||||
|
||||
if not matched_rule:
|
||||
return None, False, None
|
||||
|
||||
snapshot = {
|
||||
"enabled": True,
|
||||
"delete_source_after_success": config.delete_source_after_success,
|
||||
"rule": matched_rule,
|
||||
"matched_at": datetime.now(timezone.utc).isoformat(),
|
||||
# 兼容现有超分流水线的 processor 字段
|
||||
"processor": {
|
||||
"max_attempts": 3,
|
||||
"processor_key": matched_rule.get("processor_key", "volc_large_model_v1"),
|
||||
},
|
||||
"target_resolution": target_resolution,
|
||||
"provider_generation_resolution": matched_rule.get("provider_generation_resolution", target_resolution),
|
||||
"aspect_ratio": aspect_ratio,
|
||||
}
|
||||
|
||||
provider_resolution = matched_rule.get("provider_generation_resolution", target_resolution)
|
||||
snapshot_json = json.dumps(snapshot, ensure_ascii=False)
|
||||
|
||||
return provider_resolution, True, snapshot_json
|
||||
|
||||
|
||||
async def prepare_api_upscale_task(
|
||||
db: AsyncSession,
|
||||
api_task: ApiGenerationTask,
|
||||
source_local_path: str,
|
||||
source_width: int = 0,
|
||||
source_height: int = 0,
|
||||
source_duration: float = 0.0,
|
||||
) -> "VideoUpscaleTask | None":
|
||||
"""为 API 任务创建超分子任务。
|
||||
|
||||
复用现有的 VideoUpscaleTask 表和 upscale 执行流水线。
|
||||
如果已存在超分任务则返回 None(避免重复创建)。
|
||||
"""
|
||||
from app.models.video_upscale_task import VideoUpscaleTask
|
||||
from sqlalchemy import select
|
||||
|
||||
# 检查是否已存在超分任务(避免重复创建)
|
||||
existing = await db.execute(
|
||||
select(VideoUpscaleTask).where(
|
||||
VideoUpscaleTask.api_generation_task_id == api_task.id
|
||||
).limit(1)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
logger.info("Upscale task already exists for API task %s, skipping", api_task.id)
|
||||
return None
|
||||
|
||||
# 解析快照获取处理器配置
|
||||
try:
|
||||
snapshot = json.loads(api_task.video_upscale_snapshot_json) if api_task.video_upscale_snapshot_json else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
snapshot = {}
|
||||
|
||||
rule = snapshot.get("rule", {})
|
||||
processor_key = rule.get("processor_key", "volc_large_model_v1")
|
||||
target_resolution = rule.get("target_resolution", api_task.resolution or "1080p")
|
||||
|
||||
# 计算目标尺寸
|
||||
target_width, target_height = _resolution_to_dimensions(target_resolution, api_task.aspect_ratio)
|
||||
|
||||
upscale_task = VideoUpscaleTask(
|
||||
id=generate_id(),
|
||||
chat_generation_task_id=None,
|
||||
generation_record_id=None,
|
||||
api_generation_task_id=api_task.id, # 关联 API v3 任务
|
||||
processor_key=processor_key,
|
||||
target_width=target_width,
|
||||
target_height=target_height,
|
||||
effective_target_width=target_width,
|
||||
effective_target_height=target_height,
|
||||
source_local_path=api_task.local_path or source_local_path, # 优先使用已下载的本地文件
|
||||
source_remote_url=api_task.remote_result_url, # 火山 MediaKit 需要远程 URL
|
||||
input_source_type="provider_remote",
|
||||
source_file_size_bytes=0,
|
||||
source_width=source_width,
|
||||
source_height=source_height,
|
||||
source_duration_seconds=source_duration,
|
||||
status="pending",
|
||||
stage="upscale_queued",
|
||||
)
|
||||
db.add(upscale_task)
|
||||
await db.flush()
|
||||
|
||||
# 创建关联记录
|
||||
link = ApiUpscaleLink(
|
||||
id=generate_id(),
|
||||
api_generation_task_id=api_task.id,
|
||||
video_upscale_task_id=upscale_task.id,
|
||||
)
|
||||
db.add(link)
|
||||
await db.flush()
|
||||
|
||||
logger.info(
|
||||
"API upscale task prepared: api_task=%s upscale_task=%s processor=%s",
|
||||
api_task.id, upscale_task.id, processor_key,
|
||||
)
|
||||
return upscale_task
|
||||
|
||||
|
||||
def _resolution_to_dimensions(resolution: str, aspect_ratio: str | None) -> tuple[int, int]:
|
||||
"""将分辨率名称转换为像素尺寸。"""
|
||||
# 标准分辨率映射
|
||||
resolution_map = {
|
||||
"480p": (852, 480),
|
||||
"720p": (1280, 720),
|
||||
"1080p": (1920, 1080),
|
||||
"2K": (2560, 1440),
|
||||
"4K": (3840, 2160),
|
||||
}
|
||||
|
||||
base = resolution_map.get(resolution, (1920, 1080))
|
||||
|
||||
# 根据宽高比调整
|
||||
if aspect_ratio == "9:16":
|
||||
return (base[1], base[0]) # 竖屏
|
||||
elif aspect_ratio == "1:1":
|
||||
return (base[0], base[0]) # 正方形
|
||||
elif aspect_ratio == "4:3":
|
||||
return (base[0], int(base[0] * 3 / 4))
|
||||
|
||||
return base
|
||||
@@ -0,0 +1,150 @@
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.api.api_usage_log import ApiUsageLog
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
|
||||
async def record_usage(
|
||||
db: AsyncSession,
|
||||
api_key_id: str,
|
||||
request_type: str,
|
||||
model_name: str,
|
||||
gen_type: str,
|
||||
status: str,
|
||||
task_id: str | None = None,
|
||||
credits_cost: float = 0.0,
|
||||
tokens_used: int = 0,
|
||||
request_duration_ms: int = 0,
|
||||
error_message: str | None = None,
|
||||
error_code: str | None = None,
|
||||
request_payload_json: str | None = None,
|
||||
price_action: str | None = None,
|
||||
resolution: str | None = None,
|
||||
duration: int | None = None,
|
||||
refund_amount: float | None = None,
|
||||
quota_before: float | None = None,
|
||||
quota_after: float | None = None,
|
||||
price_detail_json: str | None = None,
|
||||
) -> ApiUsageLog:
|
||||
"""记录一次 API 调用日志。"""
|
||||
# 确定 price_action
|
||||
if price_action:
|
||||
action = price_action
|
||||
elif status == "failed":
|
||||
action = "refund"
|
||||
else:
|
||||
action = "deduct"
|
||||
|
||||
log = ApiUsageLog(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
api_generation_task_id=task_id,
|
||||
price_action=action,
|
||||
request_type=request_type,
|
||||
model_name=model_name,
|
||||
gen_type=gen_type,
|
||||
resolution=resolution,
|
||||
duration=duration,
|
||||
credits_cost=credits_cost,
|
||||
refund_amount=refund_amount or 0.0,
|
||||
quota_before=quota_before,
|
||||
quota_after=quota_after,
|
||||
tokens_used=tokens_used,
|
||||
request_duration_ms=request_duration_ms,
|
||||
price_detail_json=price_detail_json,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
error_code=error_code,
|
||||
request_payload_json=request_payload_json,
|
||||
)
|
||||
db.add(log)
|
||||
await db.flush()
|
||||
return log
|
||||
|
||||
|
||||
async def list_usage_logs(
|
||||
db: AsyncSession,
|
||||
api_key_id: str | None = None,
|
||||
skip: int = 0,
|
||||
limit: int = 50,
|
||||
start_date: datetime | None = None,
|
||||
end_date: datetime | None = None,
|
||||
) -> tuple[int, list[ApiUsageLog]]:
|
||||
"""查询使用日志(分页+筛选)。"""
|
||||
query = select(ApiUsageLog)
|
||||
count_query = select(func.count(ApiUsageLog.id))
|
||||
|
||||
filters = []
|
||||
if api_key_id:
|
||||
filters.append(ApiUsageLog.api_key_id == api_key_id)
|
||||
if start_date:
|
||||
filters.append(ApiUsageLog.created_at >= start_date)
|
||||
if end_date:
|
||||
filters.append(ApiUsageLog.created_at <= end_date)
|
||||
|
||||
for f in filters:
|
||||
query = query.where(f)
|
||||
count_query = count_query.where(f)
|
||||
|
||||
total_result = await db.execute(count_query)
|
||||
total = total_result.scalar_one()
|
||||
|
||||
query = query.order_by(ApiUsageLog.created_at.desc()).offset(skip).limit(limit)
|
||||
result = await db.execute(query)
|
||||
logs = list(result.scalars().all())
|
||||
|
||||
return total, logs
|
||||
|
||||
|
||||
async def get_usage_summary(
|
||||
db: AsyncSession,
|
||||
api_key_id: str | None = None,
|
||||
days: int = 30,
|
||||
) -> dict:
|
||||
"""获取使用汇总统计。"""
|
||||
now = datetime.now(timezone.utc)
|
||||
start = now - timedelta(days=days)
|
||||
|
||||
query = select(
|
||||
func.count(ApiUsageLog.id).label("total_requests"),
|
||||
func.coalesce(func.sum(ApiUsageLog.credits_cost), 0).label("total_credits"),
|
||||
func.coalesce(func.sum(ApiUsageLog.tokens_used), 0).label("total_tokens"),
|
||||
func.coalesce(func.avg(ApiUsageLog.request_duration_ms), 0).label("avg_duration"),
|
||||
).where(ApiUsageLog.created_at >= start)
|
||||
|
||||
if api_key_id:
|
||||
query = query.where(ApiUsageLog.api_key_id == api_key_id)
|
||||
|
||||
result = await db.execute(query)
|
||||
row = result.one()
|
||||
|
||||
# 成功/失败统计
|
||||
success_query = select(func.count(ApiUsageLog.id)).where(
|
||||
ApiUsageLog.created_at >= start,
|
||||
ApiUsageLog.status == "success",
|
||||
)
|
||||
failed_query = select(func.count(ApiUsageLog.id)).where(
|
||||
ApiUsageLog.created_at >= start,
|
||||
ApiUsageLog.status == "failed",
|
||||
)
|
||||
if api_key_id:
|
||||
success_query = success_query.where(ApiUsageLog.api_key_id == api_key_id)
|
||||
failed_query = failed_query.where(ApiUsageLog.api_key_id == api_key_id)
|
||||
|
||||
success_result = await db.execute(success_query)
|
||||
failed_result = await db.execute(failed_query)
|
||||
|
||||
return {
|
||||
"total_requests": row.total_requests or 0,
|
||||
"total_credits_cost": float(row.total_credits or 0),
|
||||
"total_tokens_used": int(row.total_tokens or 0),
|
||||
"avg_duration_ms": int(row.avg_duration or 0),
|
||||
"success_count": success_result.scalar_one() or 0,
|
||||
"failed_count": failed_result.scalar_one() or 0,
|
||||
}
|
||||
@@ -59,15 +59,15 @@ AI_LOG_ENABLED: bool = True # Set True to enable logging, or use env var AI_LOG
|
||||
|
||||
|
||||
# ── Log output settings ────────────────────────────────────
|
||||
LOG_DIR = os.path.join(
|
||||
BASE_LOG_DIR = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
|
||||
"log", "AiModel",
|
||||
"log",
|
||||
)
|
||||
LOG_DIR = os.path.join(BASE_LOG_DIR, "AiModel")
|
||||
# 请求响应日志目录
|
||||
LOG_R_Q_DIR = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
|
||||
"log", "RequestResponse",
|
||||
)
|
||||
LOG_R_Q_DIR = os.path.join(BASE_LOG_DIR, "RequestResponse")
|
||||
# VP V3 虚拟素材库专用日志目录
|
||||
VP_V3_LOG_DIR = os.path.join(BASE_LOG_DIR, "virtual_portrait_v3")
|
||||
|
||||
LOG_FILENAME_FORMAT = "{date}.log" # e.g. 2026-05-12.log
|
||||
LOG_DATE_FORMAT = "%Y-%m-%d"
|
||||
|
||||
@@ -298,6 +298,7 @@ def _base_entry(
|
||||
step_id: str | None = None,
|
||||
remote_action: str | None = None,
|
||||
remote_request_id: str | None = None,
|
||||
api_key_id: str | None = None,
|
||||
message: str | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
@@ -321,6 +322,7 @@ def _base_entry(
|
||||
"step_id": step_id,
|
||||
"remote_action": remote_action,
|
||||
"remote_request_id": remote_request_id,
|
||||
"api_key_id": api_key_id,
|
||||
"message": message,
|
||||
"detail": detail or {},
|
||||
"error": error,
|
||||
@@ -345,6 +347,7 @@ def log_operation_event(
|
||||
step_id: str | None = None,
|
||||
remote_action: str | None = None,
|
||||
remote_request_id: str | None = None,
|
||||
api_key_id: str | None = None,
|
||||
message: str | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
@@ -369,6 +372,7 @@ def log_operation_event(
|
||||
step_id=step_id,
|
||||
remote_action=remote_action,
|
||||
remote_request_id=remote_request_id,
|
||||
api_key_id=api_key_id,
|
||||
message=message,
|
||||
detail=detail,
|
||||
error=error,
|
||||
@@ -390,6 +394,7 @@ def log_module_generation_event(
|
||||
step_id: str | None = None,
|
||||
remote_action: str | None = None,
|
||||
remote_request_id: str | None = None,
|
||||
api_key_id: str | None = None,
|
||||
message: str | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
@@ -413,6 +418,7 @@ def log_module_generation_event(
|
||||
step_id=step_id,
|
||||
remote_action=remote_action,
|
||||
remote_request_id=remote_request_id,
|
||||
api_key_id=api_key_id,
|
||||
message=message,
|
||||
detail=detail,
|
||||
error=error,
|
||||
|
||||
@@ -30,23 +30,36 @@ def owner_id(owner: VideoUpscaleOwner | None) -> str | None:
|
||||
def owner_is_generating(owner: VideoUpscaleOwner) -> bool:
|
||||
if isinstance(owner, ChatGenerationTask):
|
||||
return owner.status == ChatGenerationTaskStatus.GENERATING.value
|
||||
if hasattr(owner, "api_key_id"):
|
||||
# ApiGenerationTask
|
||||
return owner.status in ("generating", "processing", "pending")
|
||||
return owner.status == GenerationStatus.generating.value
|
||||
|
||||
|
||||
def owner_is_completed(owner: VideoUpscaleOwner) -> bool:
|
||||
if isinstance(owner, ChatGenerationTask):
|
||||
return owner.status == ChatGenerationTaskStatus.COMPLETED.value
|
||||
if hasattr(owner, "api_key_id"):
|
||||
# ApiGenerationTask
|
||||
return owner.status == "completed"
|
||||
return owner.status == GenerationStatus.completed.value
|
||||
|
||||
|
||||
def set_owner_stage(owner: VideoUpscaleOwner, stage: str) -> None:
|
||||
owner.pipeline_stage = stage
|
||||
# ApiGenerationTask 没有 pipeline_stage 字段,使用 stage 字段
|
||||
if hasattr(owner, "pipeline_stage"):
|
||||
owner.pipeline_stage = stage
|
||||
elif hasattr(owner, "stage"):
|
||||
owner.stage = stage
|
||||
|
||||
|
||||
def upscale_stage_value(owner: VideoUpscaleOwner, chat_stage: ChatGenerationPipelineStage | str) -> str:
|
||||
value = chat_stage.value if hasattr(chat_stage, "value") else str(chat_stage)
|
||||
if isinstance(owner, ChatGenerationTask):
|
||||
return value
|
||||
if hasattr(owner, "api_key_id"):
|
||||
# ApiGenerationTask - 直接返回 stage 值
|
||||
return value
|
||||
try:
|
||||
return GenerationRecordPipelineStage(value).value
|
||||
except ValueError:
|
||||
@@ -69,6 +82,13 @@ async def load_upscale_owner(
|
||||
GenerationRecord.id == upscale.generation_record_id,
|
||||
GenerationRecord.deleted_at.is_(None),
|
||||
)
|
||||
elif upscale.api_generation_task_id:
|
||||
# API v3 任务
|
||||
from app.models.api.api_generation_task import ApiGenerationTask
|
||||
query = select(ApiGenerationTask).where(
|
||||
ApiGenerationTask.id == upscale.api_generation_task_id,
|
||||
ApiGenerationTask.deleted_at.is_(None),
|
||||
)
|
||||
else:
|
||||
return None
|
||||
if for_update:
|
||||
|
||||
@@ -379,7 +379,8 @@ async def _claim(
|
||||
return None
|
||||
if upscale.status in {VideoUpscaleTaskStatus.COMPLETED.value, VideoUpscaleTaskStatus.FAILED.value}:
|
||||
return None
|
||||
if not owner_is_generating(task):
|
||||
# 对于 API v3 任务(有 api_key_id 属性),即使所有者已完成也允许超分继续
|
||||
if not hasattr(task, "api_key_id") and not owner_is_generating(task):
|
||||
return None
|
||||
lease_until = _aware(upscale.lease_until)
|
||||
if lease_until and lease_until > _now() and upscale.status == VideoUpscaleTaskStatus.PROCESSING.value:
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
from app.services.virtual_portrait_v3 import (
|
||||
quota_service,
|
||||
project_service,
|
||||
asset_service,
|
||||
upload_service,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"quota_service",
|
||||
"project_service",
|
||||
"asset_service",
|
||||
"upload_service",
|
||||
]
|
||||
@@ -0,0 +1,674 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.enums.private_portrait import (
|
||||
PrivatePortraitAssetStatus,
|
||||
PrivatePortraitAssetType,
|
||||
PrivatePortraitEventSource,
|
||||
PrivatePortraitEventStatus,
|
||||
PrivatePortraitEventType,
|
||||
PrivatePortraitProjectStatus,
|
||||
PrivatePortraitRemoteDeleteStatus,
|
||||
)
|
||||
from app.models.virtual_portrait_v3 import VpV3Asset, VpV3Project
|
||||
from app.schemas.virtual_portrait_v3.asset import (
|
||||
VpV3AssetCreate,
|
||||
VpV3AssetListOut,
|
||||
VpV3AssetOut,
|
||||
VpV3SelectableAssetListOut,
|
||||
VpV3SelectableAssetOut,
|
||||
)
|
||||
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||||
from app.services.private_portrait.ark_client import (
|
||||
ArkPrivateAssetClient,
|
||||
ArkPrivateAssetClientError,
|
||||
)
|
||||
from app.services.virtual_portrait_v3.project_service import (
|
||||
refresh_project_counters,
|
||||
)
|
||||
from app.services.virtual_portrait_v3.quota_service import (
|
||||
_bytes_to_mb,
|
||||
_refresh_quota_used,
|
||||
check_asset_quota,
|
||||
get_quota,
|
||||
remote_project_name,
|
||||
)
|
||||
from app.services.virtual_portrait_v3.upload_service import (
|
||||
delete_local_file_by_url,
|
||||
download_url_to_local,
|
||||
)
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
DOMAIN = "virtual_portrait_v3"
|
||||
|
||||
URL_RE_REMOTE_URL_EXPR = re.compile(r"^https?://", re.IGNORECASE)
|
||||
URL_LOCAL_UPLOAD_EXPR = re.compile(r"^/uploads/|^https?://[^/]+/uploads/", re.IGNORECASE)
|
||||
|
||||
# 北京时间(UTC+8)统一基准:V3 业务所有时间写入 / 时间比较唯一参考
|
||||
_BJ_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def _bj_now() -> datetime:
|
||||
"""返回当前北京时间(UTC+8)naive datetime(去掉 tzinfo,与 DB naive 存储一致)。"""
|
||||
return datetime.now(_BJ_TZ).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _json(data) -> str | None:
|
||||
if data is None:
|
||||
return None
|
||||
return json.dumps(data, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def asset_to_out(a: VpV3Asset) -> VpV3AssetOut:
|
||||
|
||||
return VpV3AssetOut(
|
||||
asset_id=a.id,
|
||||
project_id=a.project_id,
|
||||
name=a.name,
|
||||
asset_type=a.asset_type,
|
||||
status=a.status,
|
||||
source_url=a.source_url,
|
||||
preview_url=a.preview_url,
|
||||
remote_url=a.remote_url,
|
||||
remote_url_expired_at=a.remote_url_expired_at,
|
||||
video_duration=a.video_duration,
|
||||
video_cover_url=a.video_cover_url,
|
||||
file_size_bytes=a.file_size_bytes,
|
||||
mime_type=a.mime_type,
|
||||
moderation_json=a.moderation_json,
|
||||
error_message=a.error_message,
|
||||
remote_delete_status=a.remote_delete_status or PrivatePortraitRemoteDeleteStatus.NONE.value,
|
||||
created_at=a.created_at,
|
||||
updated_at=a.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def asset_to_selectable(a: VpV3Asset) -> VpV3SelectableAssetOut:
|
||||
return VpV3SelectableAssetOut(
|
||||
asset_id=a.id,
|
||||
project_id=a.project_id,
|
||||
name=a.name,
|
||||
asset_type=a.asset_type,
|
||||
status=a.status,
|
||||
source_url=a.source_url,
|
||||
preview_url=a.preview_url or a.remote_url or a.source_url,
|
||||
video_duration=a.video_duration,
|
||||
video_cover_url=a.video_cover_url,
|
||||
file_size_bytes=a.file_size_bytes,
|
||||
created_at=a.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _validate_source_url(url: str, asset_type: str) -> None:
|
||||
"""创建素材时的 source_url 现在只允许 http(s) 的外部 URL。
|
||||
旧的 /uploads/* 本地 URL 已不再推荐(直接让系统自己下载保存)。"""
|
||||
if not url or not url.strip():
|
||||
raise HTTPException(status_code=400, detail="source_url 不能为空")
|
||||
stripped = url.strip()
|
||||
if not stripped.lower().startswith("http://") and not stripped.lower().startswith("https://"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="source_url 必须是公网可访问的 http(s) URL;本服务会自动下载并保存到本地",
|
||||
)
|
||||
if len(stripped) > 2000:
|
||||
raise HTTPException(status_code=400, detail="source_url 过长(最多 2000 字符)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Asset CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def create_asset(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
project: VpV3Project,
|
||||
payload: VpV3AssetCreate,
|
||||
) -> VpV3Asset:
|
||||
"""在项目下创建素材:
|
||||
|
||||
**新流程(一步到位)**:
|
||||
1. project 状态校验
|
||||
2. source_url 格式校验
|
||||
3. 将 source_url 下载保存到本地 vp_v3 上传目录(占用磁盘,校验 MIME/大小/网络)
|
||||
- 失败:抛 HTTPException(400/413/415/502/500),不留临时文件
|
||||
4. 配额校验(素材数 + 存储 MB,用下载后的实际 file_size_bytes)
|
||||
- 失败:**立刻删除本地已下载的文件**,避免占用磁盘;再抛 403
|
||||
5. Video 时长校验(payload.video_duration 优先,否则用 ffprobe 探测到的值;>60s 报错)
|
||||
- 失败:删本地文件 → 抛 400
|
||||
6. 写 VpV3Asset(Creating 状态,带 next_poll_at)
|
||||
- 失败:删本地文件 → 抛 500
|
||||
7. 调 Ark CreateAsset(url=本地公网 URL),异步审核
|
||||
- 异常:status 置为 FAILED,保留本地文件(因为已占配额和素材数,走删除接口会清理)
|
||||
8. 刷新项目计数 + 配额 used,返回素材
|
||||
"""
|
||||
# 1. project 状态校验
|
||||
if project.status != PrivatePortraitProjectStatus.ACTIVE.value:
|
||||
raise HTTPException(status_code=400, detail=f"项目状态 {project.status} 不可创建素材,仅 active 项目可操作")
|
||||
|
||||
# 2. source_url 校验(只允许公网 http(s))
|
||||
_validate_source_url(payload.source_url, payload.asset_type)
|
||||
|
||||
downloaded: "DownloadedAsset | None" = None
|
||||
try:
|
||||
# 3. URL → 本地下载保存(此处负责 URL 合法性/网络/MIME/大小的校验及抛错)
|
||||
downloaded = await download_url_to_local(
|
||||
api_key_id=api_key_id,
|
||||
asset_type=payload.asset_type,
|
||||
source_url=payload.source_url,
|
||||
requested_filename=payload.name,
|
||||
)
|
||||
file_size_bytes = downloaded.file_size_bytes
|
||||
|
||||
# 4. 配额校验(素材数 + 存储),这里已经拿到真实 file_size_bytes
|
||||
try:
|
||||
await check_asset_quota(
|
||||
db,
|
||||
api_key_id=api_key_id,
|
||||
asset_count_delta=1,
|
||||
file_size_bytes=file_size_bytes,
|
||||
)
|
||||
except HTTPException:
|
||||
# 配额不足 → 立刻清理刚下载好的本地文件,再抛
|
||||
_safe_delete_local_file(downloaded.url)
|
||||
raise
|
||||
|
||||
# 5. Video 时长:优先用 payload.video_duration,否则用探测值
|
||||
effective_video_duration: float | None = None
|
||||
if payload.asset_type == PrivatePortraitAssetType.VIDEO.value:
|
||||
if payload.video_duration is not None and payload.video_duration > 0:
|
||||
effective_video_duration = float(payload.video_duration)
|
||||
elif downloaded.duration_seconds is not None and downloaded.duration_seconds > 0:
|
||||
effective_video_duration = float(downloaded.duration_seconds)
|
||||
else:
|
||||
_safe_delete_local_file(downloaded.url)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Video 素材无法获取时长:请显式传 video_duration(秒),或确保 URL 指向合法的视频文件",
|
||||
)
|
||||
if effective_video_duration > 60:
|
||||
_safe_delete_local_file(downloaded.url)
|
||||
raise HTTPException(status_code=400, detail="视频素材时长不能超过 60 秒")
|
||||
|
||||
# 素材展示名:payload.name → downloaded.suggested_name → filename 去扩展名
|
||||
final_name: str | None = (payload.name or "").strip()[:128] or None
|
||||
if not final_name and downloaded.suggested_name:
|
||||
final_name = (downloaded.suggested_name or "").strip()[:128] or None
|
||||
|
||||
asset = VpV3Asset(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
project_id=project.id,
|
||||
remote_project_name=project.remote_project_name,
|
||||
remote_group_id=project.remote_group_id,
|
||||
remote_asset_id=None,
|
||||
asset_type=payload.asset_type,
|
||||
name=final_name,
|
||||
source_url=payload.source_url, # 本地存储后的 URL
|
||||
preview_url=downloaded.url, # 初始 preview = 本地 URL
|
||||
remote_url=None,
|
||||
remote_url_expired_at=None,
|
||||
upload_resource_id=None, # 不再使用(旧接口兼容保留字段)
|
||||
video_duration=effective_video_duration,
|
||||
video_cover_url=payload.video_cover_url,
|
||||
file_size_bytes=file_size_bytes,
|
||||
mime_type=downloaded.mime_type,
|
||||
status=PrivatePortraitAssetStatus.CREATING.value,
|
||||
poll_count=0,
|
||||
next_poll_at=_bj_now() + timedelta(seconds=2),
|
||||
)
|
||||
db.add(asset)
|
||||
await db.flush()
|
||||
await db.refresh(asset)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 任何 DB 写入前的异常 → 能清理就清理本地文件
|
||||
if downloaded:
|
||||
_safe_delete_local_file(downloaded.url)
|
||||
logger.exception("vp_v3 创建素材(下载/写库阶段)异常:%s", exc)
|
||||
raise HTTPException(status_code=500, detail=f"创建素材失败:{exc}") from exc
|
||||
|
||||
# 6. 调 Ark CreateAsset(到这里 DB 已经 flush 成功了)
|
||||
log_operation_event(
|
||||
domain=DOMAIN,
|
||||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_CREATE_REMOTE_START.value,
|
||||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||
source=PrivatePortraitEventSource.API.value,
|
||||
api_key_id=api_key_id,
|
||||
project_id=project.id,
|
||||
asset_id=asset.id,
|
||||
detail={
|
||||
"remote_project_name": asset.remote_project_name,
|
||||
"remote_group_id": asset.remote_group_id,
|
||||
"source_url": asset.source_url,
|
||||
"asset_type": asset.asset_type,
|
||||
"original_source_url": payload.source_url.strip()[:500],
|
||||
},
|
||||
)
|
||||
try:
|
||||
resp = await ArkPrivateAssetClient().create_asset(
|
||||
project_name=asset.remote_project_name,
|
||||
group_id=asset.remote_group_id,
|
||||
url=asset.source_url,
|
||||
asset_type=asset.asset_type,
|
||||
name=asset.name,
|
||||
)
|
||||
remote_asset_id = resp.get("Id") or resp.get("AssetId") or resp.get("assetId") or resp.get("id")
|
||||
if not remote_asset_id:
|
||||
raise RuntimeError("CreateAsset 未返回素材 Id")
|
||||
asset.remote_asset_id = str(remote_asset_id)
|
||||
asset.raw_response_json = _json(resp)
|
||||
asset.remote_url = resp.get("URL") or resp.get("url") or resp.get("Url") or asset.remote_url
|
||||
if asset.remote_url:
|
||||
asset.preview_url = asset.remote_url
|
||||
asset.next_poll_at = _bj_now() + timedelta(seconds=3)
|
||||
asset.status = PrivatePortraitAssetStatus.CREATING.value
|
||||
log_operation_event(
|
||||
domain=DOMAIN,
|
||||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_CREATE_REMOTE_SUCCESS.value,
|
||||
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||
source=PrivatePortraitEventSource.API.value,
|
||||
api_key_id=api_key_id,
|
||||
project_id=project.id,
|
||||
asset_id=asset.id,
|
||||
detail={"remote_asset_id": remote_asset_id},
|
||||
)
|
||||
await refresh_project_counters(db, [project.id])
|
||||
_ = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||||
return asset
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 火山调用失败 → 保留本地文件(DB 已写好,走删除接口清理),状态 FAILED,带错误
|
||||
asset.status = PrivatePortraitAssetStatus.FAILED.value
|
||||
asset.error_message = str(exc)
|
||||
asset.raw_response_json = _json({"error": str(exc)})
|
||||
log_operation_error(
|
||||
domain=DOMAIN,
|
||||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_CREATE_REMOTE_FAILED.value,
|
||||
source=PrivatePortraitEventSource.API.value,
|
||||
api_key_id=api_key_id,
|
||||
project_id=project.id,
|
||||
asset_id=asset.id,
|
||||
exc=exc,
|
||||
)
|
||||
raise HTTPException(status_code=502, detail=f"提交火山素材创建失败:{exc}") from exc
|
||||
|
||||
|
||||
def _safe_delete_local_file(local_url: str | None) -> None:
|
||||
if not local_url:
|
||||
return
|
||||
try:
|
||||
delete_local_file_by_url(local_url)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("vp_v3 清理本地文件失败(不抛):%s", local_url)
|
||||
|
||||
|
||||
async def list_assets(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
project_id: str | None = None,
|
||||
status: str | None = None,
|
||||
keyword: str | None = None,
|
||||
asset_type: str | None = None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
) -> tuple[list[VpV3Asset], int]:
|
||||
"""分页查询素材列表。"""
|
||||
conds = [VpV3Asset.api_key_id == api_key_id, VpV3Asset.deleted_at.is_(None)]
|
||||
if project_id:
|
||||
conds.append(VpV3Asset.remote_group_id == project_id)
|
||||
if status:
|
||||
conds.append(VpV3Asset.status == status)
|
||||
if keyword:
|
||||
conds.append((VpV3Asset.name.is_not(None)) & (VpV3Asset.name.ilike(f"%{keyword}%")))
|
||||
if asset_type:
|
||||
conds.append(VpV3Asset.asset_type == asset_type)
|
||||
count_result = await db.execute(select(func.count(VpV3Asset.id)).where(*conds))
|
||||
total = int(count_result.scalar() or 0)
|
||||
q = (
|
||||
select(VpV3Asset)
|
||||
.where(*conds)
|
||||
.order_by(VpV3Asset.created_at.desc())
|
||||
.limit(page_size)
|
||||
.offset((page - 1) * page_size)
|
||||
)
|
||||
items = list((await db.execute(q)).scalars().all())
|
||||
return items, total
|
||||
|
||||
|
||||
async def list_selectable_assets(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
project_id: str | None = None,
|
||||
keyword: str | None = None,
|
||||
asset_type: str | None = None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
) -> tuple[list[VpV3Asset], int]:
|
||||
"""AI 创作选择器素材列表:只返回 status=Active 的。"""
|
||||
items, total = await list_assets(
|
||||
db,
|
||||
api_key_id=api_key_id,
|
||||
project_id=project_id,
|
||||
status=PrivatePortraitAssetStatus.ACTIVE.value,
|
||||
keyword=keyword,
|
||||
asset_type=asset_type,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
return items, total
|
||||
|
||||
|
||||
async def get_asset(db: AsyncSession, *, api_key_id: str, asset_id: str) -> VpV3Asset:
|
||||
"""素材详情(权限校验)。"""
|
||||
row = (await db.execute(
|
||||
select(VpV3Asset).where(
|
||||
VpV3Asset.remote_asset_id == asset_id,
|
||||
VpV3Asset.api_key_id == api_key_id,
|
||||
VpV3Asset.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="虚拟素材不存在")
|
||||
return row
|
||||
|
||||
|
||||
async def sync_asset_status(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
asset_id: str,
|
||||
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||
) -> VpV3Asset:
|
||||
"""主动同步素材状态(调 Ark GetAsset)。
|
||||
|
||||
注意:如果素材没有 remote_asset_id(远端还未 CreateAsset 返回),直接跳过并返回当前本地快照。
|
||||
"""
|
||||
asset = await get_asset(db, api_key_id=api_key_id, asset_id=asset_id)
|
||||
if not asset.remote_asset_id:
|
||||
return asset
|
||||
try:
|
||||
resp = await ArkPrivateAssetClient().get_asset(
|
||||
project_name=asset.remote_project_name, asset_id=asset.remote_asset_id,
|
||||
)
|
||||
if execution_guard is not None:
|
||||
await execution_guard()
|
||||
_apply_get_asset_response(asset, resp)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 异常分支也必须推进 poll 计数 + 重算下次轮询时间,避免无限调度且数据库无变化
|
||||
asset.last_poll_at = _bj_now()
|
||||
asset.poll_count = int(asset.poll_count or 0) + 1
|
||||
asset.error_message = f"同步状态失败:{exc}"
|
||||
logger.warning("vp_v3 同步素材状态失败:asset_id=%s err=%s", asset_id, exc)
|
||||
# 异常情况仍然保持 CREATING,按指数退避重算 next_poll_at
|
||||
delays = [3, 6, 12, 30, 60]
|
||||
idx = min(asset.poll_count, len(delays) - 1)
|
||||
asset.next_poll_at = _bj_now() + timedelta(seconds=delays[idx])
|
||||
finally:
|
||||
await db.flush()
|
||||
await refresh_project_counters(db, [asset.project_id])
|
||||
return asset
|
||||
|
||||
|
||||
def _apply_get_asset_response(a: VpV3Asset, resp: dict) -> None:
|
||||
"""把 Ark GetAsset 响应应用到本地记录(状态、URL、审核信息)。"""
|
||||
# 先推进公共轮询字段(无论状态映射结果如何,只要调了一次 GetAsset 都必须记录)
|
||||
a.last_poll_at = _bj_now()
|
||||
a.poll_count = int(a.poll_count or 0) + 1
|
||||
a.moderation_json = _json(resp)
|
||||
a.raw_response_json = _json(resp)
|
||||
|
||||
# Status 映射:火山 Status 字段 → 本地枚举
|
||||
status_raw = str(resp.get("Status") or resp.get("status") or "").lower()
|
||||
if status_raw in {"active", "success", "done", "available"}:
|
||||
a.status = PrivatePortraitAssetStatus.ACTIVE.value
|
||||
elif status_raw in {"creating", "pending", "processing", "auditing"}:
|
||||
a.status = PrivatePortraitAssetStatus.CREATING.value
|
||||
elif status_raw in {"failed", "error", "rejected", "invalid"}:
|
||||
a.status = PrivatePortraitAssetStatus.FAILED.value
|
||||
msg = resp.get("Message") or resp.get("message") or resp.get("Error") or resp.get("error")
|
||||
if msg:
|
||||
a.error_message = str(msg)
|
||||
else:
|
||||
# 未知状态保持原
|
||||
pass
|
||||
|
||||
# URL 续期
|
||||
url = resp.get("URL") or resp.get("url") or resp.get("Url")
|
||||
if url:
|
||||
a.remote_url = url
|
||||
a.preview_url = url
|
||||
a.remote_url_expired_at = None # 无法解析过期时间就不填
|
||||
# 视频时长
|
||||
if not a.video_duration:
|
||||
dur = resp.get("Duration") or resp.get("duration")
|
||||
if dur is not None:
|
||||
try:
|
||||
a.video_duration = float(dur)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
# 大小
|
||||
if not a.file_size_bytes:
|
||||
size = resp.get("FileSize") or resp.get("fileSize") or resp.get("size")
|
||||
if size is not None:
|
||||
try:
|
||||
a.file_size_bytes = int(size)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
# 状态判断下次轮询时间
|
||||
if a.status == PrivatePortraitAssetStatus.CREATING.value:
|
||||
# 指数退避:3s → 6s → 12s → 30s → 60s,最多 60s
|
||||
delays = [3, 6, 12, 30, 60]
|
||||
idx = min(a.poll_count, len(delays) - 1)
|
||||
a.next_poll_at = _bj_now() + timedelta(seconds=delays[idx])
|
||||
elif a.status == PrivatePortraitAssetStatus.FAILED.value:
|
||||
a.next_poll_at = None # 失败不再轮询
|
||||
elif a.status == PrivatePortraitAssetStatus.ACTIVE.value:
|
||||
a.next_poll_at = None # 成功不再轮询
|
||||
|
||||
|
||||
async def soft_delete_asset(db: AsyncSession, *, api_key_id: str, asset_id: str) -> VpV3Asset:
|
||||
"""软删素材(本地先标记为删除中,同步删除本地落盘文件,重新计算项目计数和配额 used,然后 commit 后再投递异步远端删除任务)。"""
|
||||
asset = await get_asset(db, api_key_id=api_key_id, asset_id=asset_id)
|
||||
pid = asset.project_id
|
||||
now = _bj_now()
|
||||
asset.deleted_at = now
|
||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.PENDING.value
|
||||
asset.status = PrivatePortraitAssetStatus.DELETING.value
|
||||
# 本地落盘文件:立刻删(成功失败都不影响状态,避免占磁盘;失败仅 log)
|
||||
if asset.source_url:
|
||||
_safe_delete_local_file(asset.source_url)
|
||||
await db.flush()
|
||||
await refresh_project_counters(db, [pid])
|
||||
q = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||||
return asset
|
||||
|
||||
|
||||
# V3 专属的远端删除服务
|
||||
V3_DOMAIN = "virtual_portrait_v3"
|
||||
|
||||
|
||||
async def _load_v3_asset_delete_snapshot(db: AsyncSession, *, asset_id: str) -> dict | None:
|
||||
"""加载 V3 素材删除快照。"""
|
||||
asset = (
|
||||
await db.execute(
|
||||
select(VpV3Asset).where(VpV3Asset.remote_asset_id == asset_id).limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not asset:
|
||||
return None
|
||||
return {
|
||||
"owner_id": str(asset.id),
|
||||
"owner_type": "asset",
|
||||
"api_key_id": str(asset.api_key_id),
|
||||
"project_id": str(asset.project_id),
|
||||
"remote_id": str(asset.remote_asset_id) if asset.remote_asset_id else None,
|
||||
"remote_project_name": str(asset.remote_project_name or ""),
|
||||
"asset_type": str(asset.asset_type or ""),
|
||||
"remote_delete_status": str(asset.remote_delete_status or ""),
|
||||
}
|
||||
|
||||
|
||||
async def _apply_v3_asset_delete_result(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
asset_id: str,
|
||||
remote_id: str | None,
|
||||
succeeded: bool,
|
||||
skipped: bool = False,
|
||||
error: BaseException | None = None,
|
||||
) -> None:
|
||||
"""应用 V3 素材远端删除结果到数据库。"""
|
||||
asset = (
|
||||
await db.execute(
|
||||
select(VpV3Asset).where(VpV3Asset.id == asset_id).with_for_update().limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not asset:
|
||||
return
|
||||
if asset.remote_delete_status in {
|
||||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||
}:
|
||||
return
|
||||
if remote_id and str(asset.remote_asset_id or "") != remote_id:
|
||||
raise RuntimeError("V3 素材远程 Asset 已变化,旧删除结果已丢弃")
|
||||
now = _bj_now()
|
||||
if skipped:
|
||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
||||
asset.remote_delete_error = None
|
||||
elif succeeded:
|
||||
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
|
||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||||
asset.remote_deleted_at = now
|
||||
asset.remote_delete_error = None
|
||||
else:
|
||||
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
|
||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||||
asset.remote_delete_error = str(error or "远程删除失败")
|
||||
await db.flush()
|
||||
|
||||
|
||||
async def delete_v3_asset_remote(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
asset_id: str,
|
||||
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||
) -> None:
|
||||
"""V3 素材远端删除(异步 Celery 任务调用)。"""
|
||||
snapshot = await _load_v3_asset_delete_snapshot(db, asset_id=asset_id)
|
||||
if snapshot is None:
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
|
||||
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
asset_id=asset_id,
|
||||
message="远程删除跳过:本地素材不存在",
|
||||
)
|
||||
await db.rollback()
|
||||
return
|
||||
|
||||
if snapshot["remote_delete_status"] in {
|
||||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||
}:
|
||||
await db.rollback()
|
||||
return
|
||||
|
||||
remote_id = snapshot["remote_id"]
|
||||
if not remote_id:
|
||||
await _apply_v3_asset_delete_result(
|
||||
db,
|
||||
asset_id=asset_id,
|
||||
remote_id=None,
|
||||
succeeded=False,
|
||||
skipped=True,
|
||||
)
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
|
||||
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=snapshot["project_id"],
|
||||
asset_id=asset_id,
|
||||
message="远程删除跳过:素材没有 remote_asset_id",
|
||||
)
|
||||
return
|
||||
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
|
||||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=snapshot["project_id"],
|
||||
asset_id=asset_id,
|
||||
detail={
|
||||
"remote_asset_id": remote_id,
|
||||
"remote_project_name": snapshot["remote_project_name"],
|
||||
"asset_type": snapshot["asset_type"],
|
||||
},
|
||||
)
|
||||
await db.rollback()
|
||||
|
||||
remote_error: BaseException | None = None
|
||||
succeeded = False
|
||||
try:
|
||||
await ArkPrivateAssetClient(for_celery=True).delete_asset(
|
||||
project_name=snapshot["remote_project_name"],
|
||||
asset_id=remote_id,
|
||||
)
|
||||
succeeded = True
|
||||
except Exception as exc:
|
||||
remote_error = exc
|
||||
# 404 视为幂等成功
|
||||
if "not found" in str(exc).lower() or "404" in str(exc):
|
||||
succeeded = True
|
||||
|
||||
if execution_guard is not None:
|
||||
await execution_guard()
|
||||
|
||||
await _apply_v3_asset_delete_result(
|
||||
db,
|
||||
asset_id=asset_id,
|
||||
remote_id=remote_id,
|
||||
succeeded=succeeded,
|
||||
error=remote_error,
|
||||
)
|
||||
|
||||
if succeeded:
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
|
||||
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=snapshot["project_id"],
|
||||
asset_id=asset_id,
|
||||
message="远程资源不存在,按幂等删除成功处理" if remote_error is not None else None,
|
||||
detail={
|
||||
"remote_asset_id": remote_id,
|
||||
"remote_project_name": snapshot["remote_project_name"],
|
||||
},
|
||||
)
|
||||
else:
|
||||
assert remote_error is not None
|
||||
log_operation_error(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=snapshot["project_id"],
|
||||
asset_id=asset_id,
|
||||
exc=remote_error,
|
||||
)
|
||||
@@ -0,0 +1,184 @@
|
||||
"""VP V3 虚拟素材库专用日志服务。
|
||||
|
||||
统一记录所有 VP V3 相关操作日志到 logs/virtual_portrait_v3/ 目录。
|
||||
按天分文件,便于管理和排查问题。
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# === 日志目录 ===
|
||||
BASE_LOG_DIR = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))),
|
||||
"log", "virtual_portrait_v3",
|
||||
)
|
||||
os.makedirs(BASE_LOG_DIR, exist_ok=True)
|
||||
|
||||
|
||||
class _DailyFileHandler(logging.Handler):
|
||||
"""按天写入不同日志文件的处理器。"""
|
||||
|
||||
def __init__(self, log_dir: str):
|
||||
super().__init__()
|
||||
self.log_dir = log_dir
|
||||
self._current_date = None
|
||||
self._file_handler = None
|
||||
self._open_file()
|
||||
|
||||
def _open_file(self):
|
||||
"""打开当天的日志文件。"""
|
||||
now = datetime.now(timezone.utc)
|
||||
date_str = now.strftime("%Y-%m-%d")
|
||||
|
||||
if date_str == self._current_date and self._file_handler:
|
||||
return
|
||||
|
||||
if self._file_handler:
|
||||
self._file_handler.close()
|
||||
|
||||
self._current_date = date_str
|
||||
filepath = os.path.join(self.log_dir, f"{date_str}.log")
|
||||
self._file_handler = open(filepath, "a", encoding="utf-8")
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
self._open_file()
|
||||
msg = self.format(record)
|
||||
self._file_handler.write(msg + "\n")
|
||||
self._file_handler.flush()
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
def close(self):
|
||||
if self._file_handler:
|
||||
self._file_handler.close()
|
||||
super().close()
|
||||
|
||||
|
||||
def _create_logger(name: str, filename: str | None = None) -> logging.Logger:
|
||||
"""创建专用 Logger。"""
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
# 避免重复添加 handler
|
||||
if logger.handlers:
|
||||
return logger
|
||||
|
||||
# 按天写入文件
|
||||
handler = _DailyFileHandler(BASE_LOG_DIR)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
# 不向上传播到 root logger(避免重复输出到控制台)
|
||||
logger.propagate = False
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
# === 专用 Logger 实例 ===
|
||||
asset_logger = _create_logger("vp_v3.asset")
|
||||
project_logger = _create_logger("vp_v3.project")
|
||||
quota_logger = _create_logger("vp_v3.quota")
|
||||
api_logger = _create_logger("vp_v3.api")
|
||||
|
||||
|
||||
def log_asset_event(
|
||||
event_type: str,
|
||||
api_key_id: str,
|
||||
asset_id: str | None = None,
|
||||
project_id: str | None = None,
|
||||
status: str | None = None,
|
||||
detail: dict | None = None,
|
||||
error: str | None = None,
|
||||
):
|
||||
"""记录素材相关事件。"""
|
||||
log_data = {
|
||||
"event_type": event_type,
|
||||
"api_key_id": api_key_id,
|
||||
"asset_id": asset_id,
|
||||
"project_id": project_id,
|
||||
"status": status,
|
||||
"detail": detail or {},
|
||||
}
|
||||
if error:
|
||||
log_data["error"] = error
|
||||
asset_logger.error(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
else:
|
||||
asset_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
def log_project_event(
|
||||
event_type: str,
|
||||
api_key_id: str,
|
||||
project_id: str | None = None,
|
||||
status: str | None = None,
|
||||
detail: dict | None = None,
|
||||
error: str | None = None,
|
||||
):
|
||||
"""记录项目相关事件。"""
|
||||
log_data = {
|
||||
"event_type": event_type,
|
||||
"api_key_id": api_key_id,
|
||||
"project_id": project_id,
|
||||
"status": status,
|
||||
"detail": detail or {},
|
||||
}
|
||||
if error:
|
||||
log_data["error"] = error
|
||||
project_logger.error(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
else:
|
||||
project_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
def log_quota_event(
|
||||
event_type: str,
|
||||
api_key_id: str,
|
||||
quota_type: str,
|
||||
amount: float,
|
||||
quota_before: float | None = None,
|
||||
quota_after: float | None = None,
|
||||
detail: dict | None = None,
|
||||
):
|
||||
"""记录配额相关事件。"""
|
||||
log_data = {
|
||||
"event_type": event_type,
|
||||
"api_key_id": api_key_id,
|
||||
"quota_type": quota_type,
|
||||
"amount": amount,
|
||||
"quota_before": quota_before,
|
||||
"quota_after": quota_after,
|
||||
"detail": detail or {},
|
||||
}
|
||||
quota_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
def log_api_request(
|
||||
method: str,
|
||||
path: str,
|
||||
api_key_id: str,
|
||||
status_code: int,
|
||||
duration_ms: int,
|
||||
error: str | None = None,
|
||||
):
|
||||
"""记录 API 请求。"""
|
||||
log_data = {
|
||||
"method": method,
|
||||
"path": path,
|
||||
"api_key_id": api_key_id,
|
||||
"status_code": status_code,
|
||||
"duration_ms": duration_ms,
|
||||
}
|
||||
if error:
|
||||
log_data["error"] = error
|
||||
api_logger.error(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
else:
|
||||
api_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
|
||||
@@ -0,0 +1,557 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.enums.private_portrait import (
|
||||
PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
|
||||
PrivatePortraitAssetStatus,
|
||||
PrivatePortraitAssetType,
|
||||
PrivatePortraitEventSource,
|
||||
PrivatePortraitEventStatus,
|
||||
PrivatePortraitEventType,
|
||||
PrivatePortraitProjectStatus,
|
||||
PrivatePortraitRemoteDeleteStatus,
|
||||
)
|
||||
from app.models.virtual_portrait_v3 import VpV3Asset, VpV3Project
|
||||
from app.schemas.virtual_portrait_v3.project import (
|
||||
VpV3ProjectCreate,
|
||||
VpV3ProjectListOut,
|
||||
VpV3ProjectOut,
|
||||
VpV3ProjectUpdate,
|
||||
)
|
||||
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||||
from app.services.private_portrait.ark_client import ArkPrivateAssetClient
|
||||
from app.services.virtual_portrait_v3.quota_service import (
|
||||
_bytes_to_mb,
|
||||
_refresh_quota_used,
|
||||
_slug,
|
||||
check_project_quota,
|
||||
get_quota,
|
||||
remote_group_name,
|
||||
remote_project_name,
|
||||
)
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
DOMAIN = "virtual_portrait_v3"
|
||||
|
||||
# 北京时间(UTC+8)统一基准:V3 业务所有时间写入 / 时间比较唯一参考
|
||||
_BJ_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def _bj_now() -> datetime:
|
||||
"""返回当前北京时间(UTC+8)naive datetime(去掉 tzinfo,与 DB naive 存储一致)。"""
|
||||
return datetime.now(_BJ_TZ).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _json(data) -> str | None:
|
||||
if data is None:
|
||||
return None
|
||||
return json.dumps(data, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def project_to_out(p: VpV3Project) -> VpV3ProjectOut:
|
||||
return VpV3ProjectOut(
|
||||
project_id=p.remote_group_id,
|
||||
name=p.name,
|
||||
description=p.description,
|
||||
status=p.status,
|
||||
asset_count=int(p.asset_count or 0),
|
||||
active_asset_count=int(p.active_asset_count or 0),
|
||||
image_asset_count=int(p.image_asset_count or 0),
|
||||
video_asset_count=int(p.video_asset_count or 0),
|
||||
storage_mb_used=float(p.storage_mb_used or 0),
|
||||
remote_delete_status=p.remote_delete_status or PrivatePortraitRemoteDeleteStatus.NONE.value,
|
||||
error_message=p.error_message,
|
||||
created_at=p.created_at,
|
||||
updated_at=p.updated_at,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def create_project(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
payload: VpV3ProjectCreate,
|
||||
) -> VpV3Project:
|
||||
"""创建虚拟素材项目(同步调用 Ark CreateAssetGroup)。
|
||||
|
||||
1. 配额校验
|
||||
2. 本地落库 status=creating_remote_group
|
||||
3. 调 Ark CreateAssetGroup 拿 remote_group_id
|
||||
4. 本地更新为 active,返回
|
||||
"""
|
||||
await check_project_quota(db, api_key_id=api_key_id, delta=1)
|
||||
|
||||
# slug = _slug(payload.name)
|
||||
proj = VpV3Project(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
name=payload.name.strip()[:128],
|
||||
name_slug=payload.name.strip()[:128],
|
||||
description=payload.description,
|
||||
remote_project_name=remote_project_name(),
|
||||
remote_group_id="",
|
||||
status=PrivatePortraitProjectStatus.CREATING_REMOTE_GROUP.value,
|
||||
asset_count=0,
|
||||
active_asset_count=0,
|
||||
image_asset_count=0,
|
||||
video_asset_count=0,
|
||||
storage_mb_used=0,
|
||||
)
|
||||
db.add(proj)
|
||||
await db.flush()
|
||||
await db.refresh(proj)
|
||||
|
||||
group_name = remote_group_name(api_key_id=api_key_id, project_slug=proj.name_slug,id=proj.id)
|
||||
|
||||
log_operation_event(
|
||||
domain=DOMAIN,
|
||||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_START.value,
|
||||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||
source=PrivatePortraitEventSource.API.value,
|
||||
api_key_id=api_key_id,
|
||||
project_id=proj.id,
|
||||
detail={"remote_group_name": group_name, "remote_project_name": proj.remote_project_name},
|
||||
)
|
||||
try:
|
||||
resp = await ArkPrivateAssetClient().create_asset_group(
|
||||
project_name=proj.remote_project_name,
|
||||
name=group_name,
|
||||
description=payload.description,
|
||||
group_type=PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
|
||||
)
|
||||
remote_group_id = resp.get("Id") or resp.get("GroupId") or resp.get("groupId")
|
||||
if not remote_group_id:
|
||||
raise RuntimeError("CreateAssetGroup 未返回素材组 ID")
|
||||
proj.remote_group_id = str(remote_group_id)
|
||||
proj.remote_group_name = group_name
|
||||
proj.status = PrivatePortraitProjectStatus.ACTIVE.value
|
||||
proj.raw_response_json = _json(resp)
|
||||
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||||
log_operation_event(
|
||||
domain=DOMAIN,
|
||||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_SUCCESS.value,
|
||||
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||
source=PrivatePortraitEventSource.API.value,
|
||||
api_key_id=api_key_id,
|
||||
project_id=proj.id,
|
||||
detail={"remote_group_id": remote_group_id, "group_name": group_name},
|
||||
)
|
||||
return proj
|
||||
except Exception as exc: # noqa: BLE001
|
||||
proj.status = PrivatePortraitProjectStatus.CREATE_GROUP_FAILED.value
|
||||
proj.error_message = str(exc)
|
||||
log_operation_error(
|
||||
domain=DOMAIN,
|
||||
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_FAILED.value,
|
||||
source=PrivatePortraitEventSource.API.value,
|
||||
api_key_id=api_key_id,
|
||||
project_id=proj.id,
|
||||
exc=exc,
|
||||
)
|
||||
raise HTTPException(status_code=502, detail=f"创建虚拟素材项目失败:{exc}") from exc
|
||||
|
||||
|
||||
async def list_projects(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
page: int,
|
||||
page_size: int,
|
||||
keyword: str | None = None,
|
||||
status: str | None = None,
|
||||
) -> tuple[list[VpV3Project], int]:
|
||||
"""按 API Key 分页查询项目列表。"""
|
||||
conds = [VpV3Project.api_key_id == api_key_id, VpV3Project.deleted_at.is_(None)]
|
||||
if keyword:
|
||||
conds.append(VpV3Project.name.ilike(f"%{keyword}%"))
|
||||
if status:
|
||||
conds.append(VpV3Project.status == status)
|
||||
count_result = await db.execute(
|
||||
select(func.count(VpV3Project.id)).where(*conds)
|
||||
)
|
||||
total = int(count_result.scalar() or 0)
|
||||
q = (
|
||||
select(VpV3Project)
|
||||
.where(*conds)
|
||||
.order_by(VpV3Project.created_at.desc())
|
||||
.limit(page_size)
|
||||
.offset((page - 1) * page_size)
|
||||
)
|
||||
items = list((await db.execute(q)).scalars().all())
|
||||
return items, total
|
||||
|
||||
|
||||
async def get_project(db: AsyncSession, *, api_key_id: str, project_id: str) -> VpV3Project:
|
||||
"""获取项目详情(权限校验)。"""
|
||||
row = (await db.execute(
|
||||
select(VpV3Project).where(
|
||||
VpV3Project.remote_group_id == project_id,
|
||||
VpV3Project.api_key_id == api_key_id,
|
||||
VpV3Project.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="虚拟素材项目不存在")
|
||||
return row
|
||||
|
||||
|
||||
async def update_project(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
project_id: str,
|
||||
payload: VpV3ProjectUpdate,
|
||||
) -> VpV3Project:
|
||||
"""更新项目展示信息(名称/描述,不会重新创建远端 Group)。"""
|
||||
proj = await get_project(db, api_key_id=api_key_id, project_id=project_id)
|
||||
changed = False
|
||||
if payload.name is not None and payload.name != proj.name:
|
||||
proj.name = payload.name.strip()[:128]
|
||||
proj.name_slug = _slug(payload.name)
|
||||
changed = True
|
||||
if payload.description is not None and payload.description != proj.description:
|
||||
proj.description = payload.description
|
||||
changed = True
|
||||
if changed:
|
||||
await db.flush()
|
||||
return proj
|
||||
|
||||
|
||||
async def soft_delete_project(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
project_id: str,
|
||||
) -> VpV3Project:
|
||||
"""软删项目和其下所有素材(本地先删,等 commit 后再投递异步远端删除任务)。
|
||||
|
||||
会把 quota used 重新刷新一次。
|
||||
"""
|
||||
proj = await get_project(db, api_key_id=api_key_id, project_id=project_id)
|
||||
now = _bj_now()
|
||||
proj.deleted_at = now
|
||||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.PENDING.value
|
||||
proj.status = PrivatePortraitProjectStatus.DELETING.value
|
||||
# 级联软删其下所有素材
|
||||
await db.execute(
|
||||
VpV3Asset.__table__.update() # type: ignore[attr-defined]
|
||||
.where(
|
||||
VpV3Asset.project_id == proj.id,
|
||||
VpV3Asset.deleted_at.is_(None),
|
||||
)
|
||||
.values(
|
||||
deleted_at=now,
|
||||
remote_delete_status=PrivatePortraitRemoteDeleteStatus.PENDING.value,
|
||||
)
|
||||
)
|
||||
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||||
return proj
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 项目计数刷新(增删素材后调用,用于项目列表快速显示)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def refresh_project_counters(db: AsyncSession, project_ids: list[str]) -> None:
|
||||
"""按真实数据刷新项目 asset 计数和 storage。"""
|
||||
if not project_ids:
|
||||
return
|
||||
for pid in project_ids:
|
||||
row = (await db.execute(
|
||||
select(
|
||||
func.count(VpV3Asset.id),
|
||||
func.sum(case((VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value, 1), else_=0)),
|
||||
func.sum(case((VpV3Asset.asset_type == PrivatePortraitAssetType.IMAGE.value, 1), else_=0)),
|
||||
func.sum(case((VpV3Asset.asset_type == PrivatePortraitAssetType.VIDEO.value, 1), else_=0)),
|
||||
func.sum(case(
|
||||
(VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value,
|
||||
case((VpV3Asset.asset_type == PrivatePortraitAssetType.IMAGE.value, 1), else_=0)),
|
||||
else_=0
|
||||
)),
|
||||
func.sum(case(
|
||||
(VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value,
|
||||
case((VpV3Asset.asset_type == PrivatePortraitAssetType.VIDEO.value, 1), else_=0)),
|
||||
else_=0
|
||||
)),
|
||||
func.coalesce(func.sum(VpV3Asset.file_size_bytes), 0),
|
||||
).where(
|
||||
VpV3Asset.project_id == pid,
|
||||
VpV3Asset.deleted_at.is_(None),
|
||||
)
|
||||
)).one()
|
||||
(total, active, img_cnt, vid_cnt, active_img, active_vid, storage_bytes) = row
|
||||
proj = (await db.execute(
|
||||
select(VpV3Project).where(VpV3Project.id == pid).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
if proj:
|
||||
proj.asset_count = int(total or 0)
|
||||
proj.active_asset_count = int(active or 0)
|
||||
proj.image_asset_count = int(img_cnt or 0)
|
||||
proj.video_asset_count = int(vid_cnt or 0)
|
||||
proj.active_image_asset_count = int(active_img or 0)
|
||||
proj.active_video_asset_count = int(active_vid or 0)
|
||||
proj.storage_mb_used = float(_bytes_to_mb(storage_bytes))
|
||||
|
||||
|
||||
# V3 专属的项目远端删除服务
|
||||
V3_DOMAIN = "virtual_portrait_v3"
|
||||
|
||||
|
||||
async def _load_v3_project_delete_snapshot(db: AsyncSession, *, project_id: str) -> dict | None:
|
||||
"""加载 V3 项目删除快照。"""
|
||||
proj = (
|
||||
await db.execute(
|
||||
select(VpV3Project).where(VpV3Project.id == project_id).limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not proj:
|
||||
return None
|
||||
return {
|
||||
"owner_id": str(proj.id),
|
||||
"owner_type": "project",
|
||||
"api_key_id": str(proj.api_key_id),
|
||||
"remote_id": str(proj.remote_group_id) if proj.remote_group_id else None,
|
||||
"remote_project_name": str(proj.remote_project_name or ""),
|
||||
"remote_delete_status": str(proj.remote_delete_status or ""),
|
||||
}
|
||||
|
||||
|
||||
async def _apply_v3_project_delete_result(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
project_id: str,
|
||||
remote_id: str | None,
|
||||
succeeded: bool,
|
||||
skipped: bool = False,
|
||||
error: BaseException | None = None,
|
||||
) -> None:
|
||||
"""应用 V3 项目远端删除结果到数据库。"""
|
||||
proj = (
|
||||
await db.execute(
|
||||
select(VpV3Project)
|
||||
.where(VpV3Project.id == project_id)
|
||||
.with_for_update()
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not proj:
|
||||
return
|
||||
if proj.remote_delete_status in {
|
||||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||
}:
|
||||
return
|
||||
if remote_id and str(proj.remote_group_id or "") != remote_id:
|
||||
raise RuntimeError("V3 项目远程 Group 已变化,旧删除结果已丢弃")
|
||||
now = _bj_now()
|
||||
if skipped:
|
||||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
||||
proj.remote_delete_error = None
|
||||
elif succeeded:
|
||||
proj.status = PrivatePortraitProjectStatus.DELETED.value
|
||||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||||
proj.remote_deleted_at = now
|
||||
proj.remote_delete_error = None
|
||||
else:
|
||||
proj.status = PrivatePortraitProjectStatus.DELETED.value
|
||||
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||||
proj.remote_delete_error = str(error or "远程删除失败")
|
||||
await db.flush()
|
||||
# 刷新配额
|
||||
quota = await get_quota(db, api_key_id=proj.api_key_id, refresh=False)
|
||||
await _refresh_quota_used(db, quota)
|
||||
|
||||
|
||||
async def delete_v3_project_remote(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
project_id: str,
|
||||
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||
) -> None:
|
||||
"""V3 项目远端删除(异步 Celery 任务调用)。
|
||||
|
||||
会先级联删除项目下所有素材的远端资源,再删除项目的远端 Group。
|
||||
"""
|
||||
# 先删除项目下所有素材的远端资源
|
||||
assets = (
|
||||
await db.execute(
|
||||
select(VpV3Asset).where(
|
||||
VpV3Asset.project_id == project_id,
|
||||
VpV3Asset.deleted_at.is_not(None),
|
||||
VpV3Asset.remote_delete_status == PrivatePortraitRemoteDeleteStatus.PENDING.value,
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
for asset in assets:
|
||||
if asset.remote_asset_id:
|
||||
try:
|
||||
await ArkPrivateAssetClient(for_celery=True).delete_asset(
|
||||
project_name=asset.remote_project_name,
|
||||
asset_id=asset.remote_asset_id,
|
||||
)
|
||||
await _apply_v3_asset_delete_result_for_project(
|
||||
db,
|
||||
asset_id=asset.id,
|
||||
succeeded=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
if "not found" in str(exc).lower() or "404" in str(exc):
|
||||
await _apply_v3_asset_delete_result_for_project(
|
||||
db,
|
||||
asset_id=asset.id,
|
||||
succeeded=True,
|
||||
)
|
||||
else:
|
||||
await _apply_v3_asset_delete_result_for_project(
|
||||
db,
|
||||
asset_id=asset.id,
|
||||
succeeded=False,
|
||||
error=exc,
|
||||
)
|
||||
|
||||
# 再删除项目的远端 Group
|
||||
snapshot = await _load_v3_project_delete_snapshot(db, project_id=project_id)
|
||||
if snapshot is None:
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
|
||||
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=project_id,
|
||||
message="远程删除跳过:本地项目不存在",
|
||||
)
|
||||
await db.rollback()
|
||||
return
|
||||
|
||||
if snapshot["remote_delete_status"] in {
|
||||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||
}:
|
||||
await db.rollback()
|
||||
return
|
||||
|
||||
remote_id = snapshot["remote_id"]
|
||||
if not remote_id:
|
||||
await _apply_v3_project_delete_result(
|
||||
db,
|
||||
project_id=project_id,
|
||||
remote_id=None,
|
||||
succeeded=False,
|
||||
skipped=True,
|
||||
)
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
|
||||
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=project_id,
|
||||
message="远程删除跳过:项目没有 remote_group_id",
|
||||
)
|
||||
return
|
||||
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
|
||||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=project_id,
|
||||
detail={
|
||||
"remote_group_id": remote_id,
|
||||
"remote_project_name": snapshot["remote_project_name"],
|
||||
},
|
||||
)
|
||||
await db.rollback()
|
||||
|
||||
remote_error: BaseException | None = None
|
||||
succeeded = False
|
||||
try:
|
||||
await ArkPrivateAssetClient(for_celery=True).delete_asset_group(
|
||||
project_name=snapshot["remote_project_name"],
|
||||
group_id=remote_id,
|
||||
)
|
||||
succeeded = True
|
||||
except Exception as exc:
|
||||
remote_error = exc
|
||||
# 404 视为幂等成功
|
||||
if "not found" in str(exc).lower() or "404" in str(exc):
|
||||
succeeded = True
|
||||
|
||||
if execution_guard is not None:
|
||||
await execution_guard()
|
||||
|
||||
await _apply_v3_project_delete_result(
|
||||
db,
|
||||
project_id=project_id,
|
||||
remote_id=remote_id,
|
||||
succeeded=succeeded,
|
||||
error=remote_error,
|
||||
)
|
||||
|
||||
if succeeded:
|
||||
log_operation_event(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
|
||||
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=project_id,
|
||||
message="远程 Group 不存在,按幂等删除成功处理" if remote_error is not None else None,
|
||||
detail={
|
||||
"remote_group_id": remote_id,
|
||||
"remote_project_name": snapshot["remote_project_name"],
|
||||
},
|
||||
)
|
||||
else:
|
||||
assert remote_error is not None
|
||||
log_operation_error(
|
||||
domain=V3_DOMAIN,
|
||||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
|
||||
source=PrivatePortraitEventSource.CELERY.value,
|
||||
project_id=project_id,
|
||||
exc=remote_error,
|
||||
)
|
||||
|
||||
|
||||
async def _apply_v3_asset_delete_result_for_project(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
asset_id: str,
|
||||
succeeded: bool,
|
||||
error: BaseException | None = None,
|
||||
) -> None:
|
||||
"""项目删除时级联应用素材删除结果。"""
|
||||
asset = (
|
||||
await db.execute(
|
||||
select(VpV3Asset).where(VpV3Asset.id == asset_id).with_for_update().limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not asset:
|
||||
return
|
||||
if asset.remote_delete_status in {
|
||||
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||
}:
|
||||
return
|
||||
if succeeded:
|
||||
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
|
||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||||
asset.remote_deleted_at = _bj_now()
|
||||
asset.remote_delete_error = None
|
||||
else:
|
||||
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
|
||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||||
asset.remote_delete_error = str(error or "远程删除失败")
|
||||
await db.flush()
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.enums.private_portrait import (
|
||||
PRIVATE_PORTRAIT_REMOTE_PROJECT_NAME,
|
||||
PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
|
||||
PrivatePortraitAssetType,
|
||||
PrivatePortraitAssetStatus,
|
||||
PrivatePortraitLibraryType,
|
||||
PrivatePortraitProjectStatus,
|
||||
)
|
||||
from app.models.virtual_portrait_v3 import (
|
||||
VpV3ApiKeyQuota,
|
||||
VpV3Asset,
|
||||
VpV3Project,
|
||||
)
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
MB_BYTES = 1024 * 1024
|
||||
_SAFE_SLUG = re.compile(r"[^a-zA-Z0-9_-]")
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
if not name:
|
||||
return "unnamed"
|
||||
return _SAFE_SLUG.sub("_", name.strip())[:80] or "unnamed"
|
||||
|
||||
|
||||
def _bytes_to_mb(b: int | float | None) -> float:
|
||||
if not b:
|
||||
return 0.0
|
||||
return round(b / MB_BYTES, 3)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 配额读写(确保 VpV3ApiKeyQuota 记录存在)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _upsert_quota(db: AsyncSession, api_key_id: str) -> VpV3ApiKeyQuota:
|
||||
"""获取配额记录;不存在则创建(默认全 0=不可用)。"""
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
stmt = (
|
||||
insert(VpV3ApiKeyQuota)
|
||||
.values(
|
||||
id=generate_id(),
|
||||
api_key_id=api_key_id,
|
||||
project_limit=0,
|
||||
asset_limit=0,
|
||||
storage_mb_limit=0,
|
||||
project_used=0,
|
||||
asset_used=0,
|
||||
storage_mb_used=0,
|
||||
)
|
||||
.on_conflict_do_nothing(index_elements=["api_key_id"])
|
||||
)
|
||||
await db.execute(stmt)
|
||||
row = (await db.execute(
|
||||
select(VpV3ApiKeyQuota).where(VpV3ApiKeyQuota.api_key_id == api_key_id).limit(1)
|
||||
)).scalar_one()
|
||||
return row
|
||||
|
||||
|
||||
async def _refresh_quota_used(db: AsyncSession, quota: VpV3ApiKeyQuota) -> None:
|
||||
"""按真实数据重算已使用量(最终一致性)。"""
|
||||
project_result = await db.execute(
|
||||
select(func.count(VpV3Project.id)).where(
|
||||
VpV3Project.api_key_id == quota.api_key_id,
|
||||
VpV3Project.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
asset_result = await db.execute(
|
||||
select(
|
||||
func.count(VpV3Asset.id),
|
||||
func.coalesce(func.sum(VpV3Asset.file_size_bytes), 0),
|
||||
).where(
|
||||
VpV3Asset.api_key_id == quota.api_key_id,
|
||||
VpV3Asset.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
project_used = project_result.scalar() or 0
|
||||
asset_row = asset_result.one()
|
||||
asset_used = asset_row[0] or 0
|
||||
storage_bytes = asset_row[1] or 0
|
||||
quota.project_used = int(project_used)
|
||||
quota.asset_used = int(asset_used)
|
||||
quota.storage_mb_used = int(_bytes_to_mb(storage_bytes))
|
||||
|
||||
|
||||
async def get_quota(db: AsyncSession, *, api_key_id: str, refresh: bool = True) -> VpV3ApiKeyQuota:
|
||||
"""获取当前 API Key 的配额(含已使用量)。不存在则创建默认 0。"""
|
||||
quota = await _upsert_quota(db, api_key_id)
|
||||
if refresh:
|
||||
await _refresh_quota_used(db, quota)
|
||||
return quota
|
||||
|
||||
|
||||
async def ensure_quota_enabled(db: AsyncSession, *, api_key_id: str) -> VpV3ApiKeyQuota:
|
||||
"""校验是否已启用虚拟素材库功能,未启用直接 403。返回已刷新的配额。
|
||||
|
||||
判定口径(与后台设置保持一致):只要「项目数上限」或「素材数上限」任一 > 0 即视为启用;
|
||||
存储上限已从配置中移除(不再作为启用条件,也不做硬性限制,仅保留数据库字段做统计展示)。
|
||||
"""
|
||||
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
|
||||
if (quota.project_limit or 0) <= 0 and (quota.asset_limit or 0) <= 0:
|
||||
raise HTTPException(status_code=403, detail="当前 API Key 未开启虚拟素材库功能,请联系管理员配置配额")
|
||||
return quota
|
||||
|
||||
|
||||
def _check(limit: int | None, used: int | float | None, delta: int | float, field: str) -> None:
|
||||
"""通用配额上限校验。
|
||||
|
||||
约定:limit <= 0 视为该维度「未配置 / 不做限制」,此时直接跳过不报错;
|
||||
只有 limit > 0 时才按「已用 + 本次 <= 上限」判断,避免影响已移除的维度(如存储上限)。
|
||||
"""
|
||||
if (limit or 0) <= 0:
|
||||
return # 不限制,直接通过
|
||||
if (used or 0) + delta > limit:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"虚拟素材库配额不足:{field} 上限 {limit},已使用 {used},本次需要 {delta},超出上限",
|
||||
)
|
||||
|
||||
|
||||
async def check_project_quota(db: AsyncSession, *, api_key_id: str, delta: int = 1) -> VpV3ApiKeyQuota:
|
||||
"""创建项目前校验配额。"""
|
||||
quota = await ensure_quota_enabled(db, api_key_id=api_key_id)
|
||||
_check(quota.project_limit, quota.project_used, delta, "项目数")
|
||||
return quota
|
||||
|
||||
|
||||
async def check_asset_quota(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
asset_count_delta: int = 1,
|
||||
file_size_bytes: int | None = None,
|
||||
) -> VpV3ApiKeyQuota:
|
||||
"""上传素材前校验配额。
|
||||
|
||||
注:「存储空间上限」已从业务约束中移除(不再做硬性配额限制),仅保留素材数量上限
|
||||
与项目数量上限的校验;storage_mb_used 字段仍会在 get_quota 中刷新用于统计展示。
|
||||
"""
|
||||
del file_size_bytes # 不再用于配额校验(仅保留形参兼容现有调用点)
|
||||
quota = await ensure_quota_enabled(db, api_key_id=api_key_id)
|
||||
_check(quota.asset_limit, quota.asset_used, asset_count_delta, "素材总数")
|
||||
return quota
|
||||
|
||||
|
||||
def remote_project_name() -> str:
|
||||
"""火山 ProjectName(V3 中转统一共用这个 Project)。"""
|
||||
return PRIVATE_PORTRAIT_REMOTE_PROJECT_NAME
|
||||
|
||||
|
||||
def remote_group_name(*, api_key_id: str, project_slug: str, id: str) -> str:
|
||||
"""火山 GroupName:vp-api-{api_key_id_short}-{id}-{slug} 最多 128 字符。"""
|
||||
short_key = (api_key_id or "")
|
||||
return f"vp-api-{short_key}-{id}-{project_slug}"[:128]
|
||||
@@ -0,0 +1,547 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, UploadFile
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.enums.upload_resource import UploadResourceTypeEnum
|
||||
from app.schemas.virtual_portrait_v3.upload import VpV3UploadOut
|
||||
from app.services.video_cover_service import get_ffmpeg_bin
|
||||
|
||||
logger = logging.getLogger("videogen")
|
||||
|
||||
# 北京时间(UTC+8)统一基准:V3 业务所有时间写入 / 时间比较唯一参考
|
||||
_BJ_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def _bj_now() -> datetime:
|
||||
"""返回当前北京时间(UTC+8)naive datetime(去掉 tzinfo,与 DB naive 存储一致)。"""
|
||||
return datetime.now(_BJ_TZ).replace(tzinfo=None)
|
||||
|
||||
|
||||
VP_V3_IMAGE_MAX_BYTES = 10 * 1024 * 1024
|
||||
VP_V3_VIDEO_MAX_BYTES = 100 * 1024 * 1024
|
||||
VP_V3_MODULE_NAME = "vp_v3_virtual"
|
||||
|
||||
IMAGE_EXT_ALLOWED = {"jpg", "jpeg", "png", "webp", "bmp"}
|
||||
VIDEO_EXT_ALLOWED = {"mp4", "mov", "m4v", "webm"}
|
||||
|
||||
IMAGE_MIME_ALLOWED = {
|
||||
"image/jpeg", "image/jpg", "image/png", "image/webp", "image/bmp",
|
||||
}
|
||||
VIDEO_MIME_ALLOWED = {
|
||||
"video/mp4", "video/quicktime", "video/x-m4v", "video/webm",
|
||||
}
|
||||
|
||||
# URL 下载相关默认值
|
||||
URL_DOWNLOAD_CONNECT_TIMEOUT_SEC = 15
|
||||
URL_DOWNLOAD_READ_TIMEOUT_SEC = 300 # 大文件下载可以久一点,读的时候会按大小上限中断
|
||||
URL_DOWNLOAD_MAX_REDIRECTS = 5
|
||||
URL_DOWNLOAD_CHUNK_BYTES = 1024 * 1024 # 1MB
|
||||
|
||||
|
||||
class DownloadedAsset(NamedTuple):
|
||||
"""URL 下载到本地后的结果。"""
|
||||
url: str # 对外访问 URL(最终要存到 VpV3Asset.source_url 的)
|
||||
filename: str # 落盘后的文件名
|
||||
file_size_bytes: int # 实际文件大小
|
||||
mime_type: str | None # 从响应头/扩展名推断出的 MIME
|
||||
duration_seconds: float | None # 视频:ffprobe 探测到的时长(Image 为 None)
|
||||
suggested_name: str | None # 从 URL 或 Content-Disposition 推断的展示名(无扩展名)
|
||||
|
||||
|
||||
def _max_bytes(asset_type: str) -> int:
|
||||
return VP_V3_VIDEO_MAX_BYTES if asset_type == UploadResourceTypeEnum.VIDEO.value else VP_V3_IMAGE_MAX_BYTES
|
||||
|
||||
|
||||
def _allowed_mime_set(asset_type: str) -> set[str]:
|
||||
return VIDEO_MIME_ALLOWED if asset_type == UploadResourceTypeEnum.VIDEO.value else IMAGE_MIME_ALLOWED
|
||||
|
||||
|
||||
def _safe_ext(filename: str, asset_type: str) -> str:
|
||||
ext = (os.path.splitext(filename or "")[1].lower().lstrip(".") or "").strip()
|
||||
allowed = VIDEO_EXT_ALLOWED if asset_type == UploadResourceTypeEnum.VIDEO.value else IMAGE_EXT_ALLOWED
|
||||
if ext and ext in allowed:
|
||||
return ext
|
||||
# fallback
|
||||
return "mp4" if asset_type == UploadResourceTypeEnum.VIDEO.value else "png"
|
||||
|
||||
|
||||
def _get_ffprobe_bin() -> str:
|
||||
ffmpeg = Path(get_ffmpeg_bin())
|
||||
sibling = ffmpeg.with_name("ffprobe.exe" if ffmpeg.suffix.lower() == ".exe" else "ffprobe")
|
||||
if sibling.exists():
|
||||
return str(sibling)
|
||||
found = shutil.which("ffprobe") or shutil.which("ffprobe.exe")
|
||||
if found:
|
||||
return found
|
||||
raise RuntimeError("未找到 ffprobe,请确保其与 FFMPEG_BIN 同目录或已加入 PATH")
|
||||
|
||||
|
||||
def _probe_duration_optional(video_path: Path) -> float | None:
|
||||
"""尝试 ffprobe 探测视频时长,失败不抛,返回 None 让调用方自己处理。"""
|
||||
import json as _json
|
||||
|
||||
try:
|
||||
ffprobe_bin = _get_ffprobe_bin()
|
||||
# 检查 ffprobe 是否可用
|
||||
if not shutil.which(ffprobe_bin) and ffprobe_bin == "ffprobe":
|
||||
logger.warning("ffprobe 未在系统 PATH 中找到,无法探测视频时长。请安装 ffprobe 并添加到 PATH。")
|
||||
return None
|
||||
|
||||
timeout = int(getattr(settings, "SHOT_FFPROBE_TIMEOUT_SECONDS", 20) or 20)
|
||||
cmd = [
|
||||
ffprobe_bin,
|
||||
"-v", "error",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "json",
|
||||
str(video_path),
|
||||
]
|
||||
completed = subprocess.run(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, timeout=timeout, check=False,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
logger.warning(
|
||||
"ffprobe 执行失败: returncode=%s stderr=%s",
|
||||
completed.returncode, completed.stderr[:200],
|
||||
)
|
||||
return None
|
||||
if not completed.stdout:
|
||||
return None
|
||||
data = _json.loads(completed.stdout or "{}")
|
||||
dur_raw = (data.get("format") or {}).get("duration")
|
||||
if dur_raw is None:
|
||||
return None
|
||||
dur = float(dur_raw)
|
||||
if dur <= 0:
|
||||
return None
|
||||
return dur
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("ffprobe 超时: %s", str(video_path))
|
||||
return None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("ffprobe 探测视频时长失败: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _build_destination(*, api_key_id: str, asset_type: str, original_filename: str, duration_seconds: float | None) -> tuple[Path, str, str]:
|
||||
"""构建 vp_v3 上传存储路径 + 对外访问 URL。
|
||||
|
||||
存储路径:UPLOAD_LOCAL_PATH/api/private_portrait_virtual/{asset_type}/{yyyy}/{mm}/{dd}/{uuid}.{ext}
|
||||
"""
|
||||
now = _bj_now()
|
||||
ext = _safe_ext(original_filename, asset_type)
|
||||
safe_uuid = uuid.uuid4().hex
|
||||
year = f"{now.year:04d}"
|
||||
month = f"{now.month:02d}"
|
||||
day = f"{now.day:02d}"
|
||||
|
||||
sub_type = "videos" if asset_type == UploadResourceTypeEnum.VIDEO.value else "images"
|
||||
rel_dir = Path("api") / "private_portrait_virtual" / sub_type / year / month / day
|
||||
filename = f"vp_v3_{safe_uuid}.{ext}"
|
||||
|
||||
base_dir = Path(settings.UPLOAD_LOCAL_PATH) if settings.UPLOAD_LOCAL_PATH else Path("./storage/uploads")
|
||||
final_path = base_dir / rel_dir / filename
|
||||
|
||||
# URL 前缀 /uploads/...
|
||||
base_url = settings.UPLOAD_BASE_URL.rstrip("/") if settings and getattr(settings, "UPLOAD_BASE_URL", None) else "/uploads"
|
||||
rel_url = f"/{rel_dir.as_posix()}/{filename}".replace("//", "/")
|
||||
url = base_url + rel_url
|
||||
return final_path, url, filename
|
||||
|
||||
|
||||
def _guess_filename_from_url(url: str, cd_header: str | None) -> str:
|
||||
"""优先从 Content-Disposition 拿文件名,其次从 URL path 拿,再 fallback 到 uuid 名。"""
|
||||
# 1. Content-Disposition
|
||||
if cd_header:
|
||||
# filename="a.jpg" 或 filename*=UTF-8''a.jpg
|
||||
import re as _re
|
||||
m1 = _re.search(r"""filename\*\s*=\s*UTF-8''([^;]+)""", cd_header, flags=_re.IGNORECASE)
|
||||
if m1:
|
||||
from urllib.parse import unquote
|
||||
try:
|
||||
return unquote(m1.group(1).strip().strip('"').strip("'"))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
m2 = _re.search(r"""filename\s*=\s*"([^"]+)""", cd_header, flags=_re.IGNORECASE)
|
||||
if m2:
|
||||
return m2.group(1)
|
||||
m3 = _re.search(r"""filename\s*=\s*([^;]+)""", cd_header, flags=_re.IGNORECASE)
|
||||
if m3:
|
||||
return m3.group(1).strip().strip('"').strip("'")
|
||||
# 2. URL path
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
base = os.path.basename(parsed.path or "")
|
||||
if base and "." in base:
|
||||
return base
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return f"vp_v3_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
|
||||
def _guess_ext_from_mime(mime: str | None, asset_type: str) -> str | None:
|
||||
if not mime:
|
||||
return None
|
||||
# 按 asset_type 优先匹配
|
||||
allowed_exts = VIDEO_EXT_ALLOWED if asset_type == UploadResourceTypeEnum.VIDEO.value else IMAGE_EXT_ALLOWED
|
||||
guesses = mimetypes.guess_all_extensions(mime.strip().lower()) or []
|
||||
for g in guesses:
|
||||
ext = g.lower().lstrip(".")
|
||||
if ext in allowed_exts:
|
||||
return ext
|
||||
# 额外的手写映射
|
||||
extra_map = {
|
||||
"image/jpeg": "jpg", "image/jpg": "jpg",
|
||||
"video/quicktime": "mov", "video/x-m4v": "m4v",
|
||||
}
|
||||
if mime.lower() in extra_map and extra_map[mime.lower()] in allowed_exts:
|
||||
return extra_map[mime.lower()]
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1) 上传本地文件(保留旧 API 但走下载流程的也可以共用保存逻辑)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def upload_asset_file(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
api_key_id: str,
|
||||
file: UploadFile,
|
||||
asset_type: str,
|
||||
duration_seconds: float | None = None,
|
||||
) -> VpV3UploadOut:
|
||||
"""V3 虚拟素材上传(独立实现,不经过用户容量账本 UploadResource)。
|
||||
|
||||
- 校验 MIME/扩展名/大小
|
||||
- 落盘到 /uploads/images|videos/vp_v3/{api_key_id_short}/{yyyy}/{mm}/{dd}/
|
||||
- 返回 url + 虚拟 resource_id(hash 形式)
|
||||
"""
|
||||
if asset_type not in {UploadResourceTypeEnum.IMAGE.value, UploadResourceTypeEnum.VIDEO.value}:
|
||||
raise HTTPException(status_code=400, detail="虚拟素材上传仅支持图片或视频")
|
||||
|
||||
max_size = _max_bytes(asset_type)
|
||||
|
||||
temp_path: str | None = None
|
||||
try:
|
||||
# 1. 落临时文件并限制大小
|
||||
size_acc = 0
|
||||
temp_dir = settings.UPLOAD_TEMP_DIR if settings and getattr(settings, "UPLOAD_TEMP_DIR", None) else "./uploads/_tmp_vp_v3"
|
||||
Path(temp_dir).mkdir(parents=True, exist_ok=True)
|
||||
temp_path = os.path.join(temp_dir, f"vp_v3_{uuid.uuid4().hex}")
|
||||
with open(temp_path, "wb") as f:
|
||||
while True:
|
||||
chunk = await file.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size_acc += len(chunk)
|
||||
if size_acc > max_size:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"文件大小超出限制:{asset_type} 最大 {max_size // (1024*1024)} MB",
|
||||
)
|
||||
f.write(chunk)
|
||||
file_size_bytes = size_acc
|
||||
if file_size_bytes <= 0:
|
||||
raise HTTPException(status_code=400, detail="空文件不允许上传")
|
||||
|
||||
# 2. 构建最终路径 + URL
|
||||
final_path, url, safe_filename = _build_destination(
|
||||
api_key_id=api_key_id,
|
||||
asset_type=asset_type,
|
||||
original_filename=file.filename or safe_filename,
|
||||
duration_seconds=duration_seconds,
|
||||
)
|
||||
final_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(temp_path, final_path)
|
||||
temp_path = None
|
||||
|
||||
# 3. 虚拟 resource_id(用于素材删除时的文件清理定位:hash(url))
|
||||
resource_id = "vpv3_" + hashlib.sha256(url.encode()).hexdigest()[:24]
|
||||
|
||||
return VpV3UploadOut(
|
||||
url=url,
|
||||
filename=safe_filename,
|
||||
type=asset_type,
|
||||
resource_id=resource_id,
|
||||
file_size_bytes=file_size_bytes,
|
||||
duration_seconds=duration_seconds,
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("vp_v3 上传失败:%s", exc)
|
||||
raise HTTPException(status_code=500, detail=f"上传失败:{exc}") from exc
|
||||
finally:
|
||||
if temp_path and os.path.exists(temp_path):
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2) URL 下载到本地(新流程:创建素材时一步到位,由 create_asset 内部调用)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def download_url_to_local(
|
||||
*,
|
||||
api_key_id: str,
|
||||
asset_type: str,
|
||||
source_url: str,
|
||||
requested_filename: str | None = None,
|
||||
) -> DownloadedAsset:
|
||||
"""把传入的远程 URL(http/https)下载到本地 vp_v3 上传目录,返回本地 URL + 元信息。
|
||||
|
||||
完整的错误处理:
|
||||
- 非法 URL → 400
|
||||
- 连接/超时 → 502(外部资源不可达)
|
||||
- HTTP 4xx/5xx → 502 带状态码
|
||||
- Content-Type 不在允许列表 → 415
|
||||
- 超出大小上限(读内容时逐 chunk 检查)→ 413
|
||||
- 下载一半失败 → 清理临时文件,不留下半截
|
||||
- 视频可选探测 ffprobe,失败不抛错(调用方自行用 payload.video_duration)
|
||||
"""
|
||||
if asset_type not in {UploadResourceTypeEnum.IMAGE.value, UploadResourceTypeEnum.VIDEO.value}:
|
||||
raise HTTPException(status_code=400, detail="虚拟素材仅支持图片或视频")
|
||||
|
||||
# URL 合法性
|
||||
if not source_url or not isinstance(source_url, str):
|
||||
raise HTTPException(status_code=400, detail="source_url 不能为空")
|
||||
parsed = urlparse(source_url.strip())
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise HTTPException(status_code=400, detail="source_url 必须是合法的 http(s) URL")
|
||||
|
||||
# 拒绝私有/内网地址(SSRF 防御的最小集;生产环境可再严格)
|
||||
import ipaddress
|
||||
host_only = parsed.hostname or ""
|
||||
try:
|
||||
ip_obj = ipaddress.ip_address(host_only)
|
||||
if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_reserved or ip_obj.is_link_local:
|
||||
raise HTTPException(status_code=400, detail="source_url 不允许指向内网/本机地址")
|
||||
except ValueError:
|
||||
# 不是 IP,是域名 → 放行
|
||||
pass
|
||||
|
||||
max_bytes = _max_bytes(asset_type)
|
||||
allowed_mimes = _allowed_mime_set(asset_type)
|
||||
# 用户代理:标成我们服务的 UA,避免一些图片防盗链 403
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (compatible; VideoGenVPV3/1.0; +https://minzhongzc.com/)"
|
||||
if getattr(settings, "VP_V3_DOWNLOAD_UA", None) is None
|
||||
else str(getattr(settings, "VP_V3_DOWNLOAD_UA"))
|
||||
)
|
||||
|
||||
temp_path: str | None = None
|
||||
final_path: Path | None = None
|
||||
# 外层初始化,保证 client.stream 内部 raise 的情况下外层仍然可访问
|
||||
inferred_filename: str = f"vp_v3_{uuid.uuid4().hex[:12]}"
|
||||
mime: str | None = None
|
||||
size_acc: int = 0
|
||||
try:
|
||||
# --- 第一步:下载到临时文件,限制大小 + 校验响应头 ---
|
||||
temp_dir = settings.UPLOAD_TEMP_DIR if settings and getattr(settings, "UPLOAD_TEMP_DIR", None) else "./uploads/_tmp_vp_v3"
|
||||
Path(temp_dir).mkdir(parents=True, exist_ok=True)
|
||||
temp_path = os.path.join(temp_dir, f"vp_v3_url_{uuid.uuid4().hex}")
|
||||
|
||||
transport = httpx.AsyncHTTPTransport(retries=1)
|
||||
timeout = httpx.Timeout(
|
||||
connect=URL_DOWNLOAD_CONNECT_TIMEOUT_SEC,
|
||||
read=URL_DOWNLOAD_READ_TIMEOUT_SEC,
|
||||
write=URL_DOWNLOAD_CONNECT_TIMEOUT_SEC,
|
||||
pool=URL_DOWNLOAD_CONNECT_TIMEOUT_SEC,
|
||||
)
|
||||
headers = {"User-Agent": user_agent, "Accept": "*/*"}
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
transport=transport,
|
||||
follow_redirects=True,
|
||||
max_redirects=URL_DOWNLOAD_MAX_REDIRECTS,
|
||||
verify=bool(getattr(settings, "VP_V3_DOWNLOAD_VERIFY_SSL", True)),
|
||||
) as client:
|
||||
async with client.stream("GET", source_url.strip(), headers=headers) as resp:
|
||||
# HTTP 状态码
|
||||
if resp.status_code >= 400:
|
||||
detail = f"远程资源返回状态码 {resp.status_code}"
|
||||
try:
|
||||
snippet = (await resp.aread())[:200]
|
||||
if snippet:
|
||||
detail += f",响应片段:{snippet.decode('utf-8', errors='ignore')}"
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"source_url 下载失败(HTTP {resp.status_code}):" + detail,
|
||||
)
|
||||
|
||||
# Content-Type 校验(没有就 fallback 到扩展名推断)
|
||||
content_type_raw = resp.headers.get("Content-Type") or ""
|
||||
mime = (content_type_raw.split(";")[0] or "").strip().lower() or None
|
||||
if mime and mime not in allowed_mimes:
|
||||
# 一些 CDN 会用 application/octet-stream,这种情况跳过 MIME 检查,用扩展名兜底
|
||||
if mime != "application/octet-stream":
|
||||
raise HTTPException(
|
||||
status_code=415,
|
||||
detail=(
|
||||
f"不支持的 Content-Type:{mime}。"
|
||||
f"{asset_type} 仅支持:{', '.join(sorted(allowed_mimes))}"
|
||||
),
|
||||
)
|
||||
|
||||
# Content-Length 预估检查(存在且超出就直接拒,不下载)
|
||||
content_length = resp.headers.get("Content-Length")
|
||||
if content_length:
|
||||
try:
|
||||
cl = int(content_length)
|
||||
if cl > max_bytes:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"远程资源太大(Content-Length={cl}),超过 "
|
||||
f"{asset_type} 上限 {max_bytes} 字节"
|
||||
),
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# filename 推断(用于扩展名 + 展示名)
|
||||
cd = resp.headers.get("Content-Disposition")
|
||||
inferred_filename = _guess_filename_from_url(source_url, cd)
|
||||
if requested_filename:
|
||||
# 若用户传了 name 就优先用它做展示名,但扩展名仍然以 mime/url 推断为准
|
||||
try:
|
||||
base_display = os.path.splitext(os.path.basename(requested_filename))[0]
|
||||
old_ext = os.path.splitext(inferred_filename)[1] if inferred_filename else ""
|
||||
inferred_filename = base_display + (old_ext or "")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# 扩展名再精化:如果 MIME 能得出扩展名,优先用
|
||||
ext_from_mime = _guess_ext_from_mime(mime, asset_type)
|
||||
if ext_from_mime:
|
||||
try:
|
||||
stem = os.path.splitext(inferred_filename)[0]
|
||||
inferred_filename = f"{stem}.{ext_from_mime}"
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# 流式下载到 temp_path,逐 chunk 检查大小
|
||||
size_acc = 0
|
||||
with open(temp_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
size_acc += len(chunk)
|
||||
if size_acc > max_bytes:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"远程资源大小超过 {asset_type} 上限 "
|
||||
f"{max_bytes // (1024*1024)} MB"
|
||||
),
|
||||
)
|
||||
f.write(chunk)
|
||||
|
||||
# ======== 以下在 client.stream 退出后、但仍在 httpx.AsyncClient 上下文内执行 ========
|
||||
# 空文件检查
|
||||
file_size_bytes = size_acc
|
||||
if file_size_bytes <= 0:
|
||||
raise HTTPException(status_code=400, detail="远程 URL 返回空文件")
|
||||
|
||||
# --- 第二步:视频可选 ffprobe 探测时长 ---
|
||||
duration: float | None = None
|
||||
if asset_type == UploadResourceTypeEnum.VIDEO.value:
|
||||
duration = _probe_duration_optional(Path(temp_path))
|
||||
|
||||
# --- 第三步:落到最终目录(与 _build_destination 一致的目录结构/权限) ---
|
||||
final_path, url_out, final_filename = _build_destination(
|
||||
api_key_id=api_key_id,
|
||||
asset_type=asset_type,
|
||||
original_filename=inferred_filename,
|
||||
duration_seconds=duration,
|
||||
)
|
||||
final_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(temp_path, final_path)
|
||||
temp_path = None
|
||||
|
||||
# 最终 MIME:按扩展名反推一个(如果之前没拿到)
|
||||
if not mime:
|
||||
mime, _ = mimetypes.guess_type(final_filename)
|
||||
if not mime:
|
||||
mime = "image/png" if asset_type == UploadResourceTypeEnum.IMAGE.value else "video/mp4"
|
||||
|
||||
suggested_name: str | None = None
|
||||
try:
|
||||
stem = os.path.splitext(inferred_filename or final_filename)[0]
|
||||
if stem and not stem.startswith("vp_v3_"):
|
||||
suggested_name = stem[:100] or None
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return DownloadedAsset(
|
||||
url=url_out,
|
||||
filename=final_filename,
|
||||
file_size_bytes=file_size_bytes,
|
||||
mime_type=mime,
|
||||
duration_seconds=duration,
|
||||
suggested_name=suggested_name,
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except (httpx.TimeoutException, httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout) as exc:
|
||||
raise HTTPException(status_code=502, detail=f"远程 URL 连接/读取超时:{exc}") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(status_code=502, detail=f"远程 URL 下载失败:{exc}") from exc
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("vp_v3 URL 下载异常:url=%s err=%s", source_url, exc)
|
||||
raise HTTPException(status_code=500, detail=f"URL 下载保存失败:{exc}") from exc
|
||||
finally:
|
||||
# 任何失败都清掉半截临时文件;但最终文件已经 move 过去了的就不动
|
||||
if temp_path and os.path.exists(temp_path):
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def delete_local_file_by_url(local_url: str) -> bool:
|
||||
"""素材删除时根据 source_url 删除本地落盘文件(非强制,失败不抛)。"""
|
||||
if not local_url:
|
||||
return False
|
||||
try:
|
||||
base_url = settings.UPLOAD_BASE_URL.rstrip("/") if settings and getattr(settings, "UPLOAD_BASE_URL", None) else "/uploads"
|
||||
rel_url = local_url
|
||||
if rel_url.startswith(base_url):
|
||||
rel_url = rel_url[len(base_url):]
|
||||
if not rel_url.startswith("/"):
|
||||
return False
|
||||
# /uploads/api/private_portrait_virtual/... → /api/private_portrait_virtual/... → UPLOAD_LOCAL_PATH/api/private_portrait_virtual/...
|
||||
sub_part = rel_url[len("/uploads"):] if rel_url.startswith("/uploads") else rel_url
|
||||
base_dir = Path(settings.UPLOAD_LOCAL_PATH) if settings.UPLOAD_LOCAL_PATH else Path("./storage/uploads")
|
||||
target = (base_dir / sub_part.lstrip("/")).resolve()
|
||||
base_dir_resolved = base_dir.resolve()
|
||||
# 仅允许删除 base_dir 下的文件(目录穿越防御)
|
||||
if not str(target).startswith(str(base_dir_resolved)):
|
||||
return False
|
||||
if target.is_file():
|
||||
target.unlink(missing_ok=True)
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("vp_v3 清理本地文件失败:%s", local_url)
|
||||
return False
|
||||
Reference in New Issue
Block a user