1、增加调用 AI 视频生成能力和虚拟素材库管理的对外api

2、增加后台apikkey管理
3、增加apikey单独的模型定价
4、增加apikey调用情况
5、完善所有数据的注释增加
This commit is contained in:
2026-08-06 13:13:28 +08:00
parent a55d4d649c
commit 0c511f3451
102 changed files with 13986 additions and 41 deletions
@@ -0,0 +1,59 @@
from app.services.api_v3.auth_service import ApiKeyContext, get_api_key_dependency
from app.services.api_v3.key_service import (
create_api_key,
list_api_keys,
get_api_key,
update_api_key,
delete_api_key,
reset_quota_if_needed,
)
from app.services.api_v3.quota_service import check_quota, can_start_video_task, get_active_video_tasks_count, get_queued_video_tasks
from app.services.api_v3.usage_log_service import record_usage, get_usage_summary, list_usage_logs
from app.services.api_v3.generation_service import submit_video_generation, generate_image_sync
from app.services.api_v3.task_service import create_video_task, create_image_task, get_task, map_task_to_status_response
from app.services.api_v3.upscale_service import (
get_or_create_upscale_config,
save_upscale_config,
build_api_upscale_snapshot,
prepare_api_upscale_task,
)
from app.services.api_v3.engine_service import resolve_video_engine, resolve_image_engine, build_engine_snapshot
from app.services.api_v3.pricing_service import (
calc_api_video_price,
calc_api_image_price,
get_priced_models,
)
__all__ = [
"ApiKeyContext",
"get_api_key_dependency",
"create_api_key",
"list_api_keys",
"get_api_key",
"update_api_key",
"delete_api_key",
"reset_quota_if_needed",
"check_quota",
"can_start_video_task",
"get_active_video_tasks_count",
"get_queued_video_tasks",
"record_usage",
"get_usage_summary",
"list_usage_logs",
"submit_video_generation",
"generate_image_sync",
"create_video_task",
"create_image_task",
"get_task",
"map_task_to_status_response",
"get_or_create_upscale_config",
"save_upscale_config",
"build_api_upscale_snapshot",
"prepare_api_upscale_task",
"resolve_video_engine",
"resolve_image_engine",
"build_engine_snapshot",
"calc_api_video_price",
"calc_api_image_price",
"get_priced_models",
]
@@ -0,0 +1,99 @@
import hashlib
import json
import logging
from datetime import datetime, timezone
from fastapi import Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.dependencies import get_db
from app.models.api.api_key import ApiKey
logger = logging.getLogger("videogen")
security = HTTPBearer(auto_error=False)
class ApiKeyContext:
"""API Key 验证上下文,携带解析后的可调用模型列表。"""
def __init__(self, api_key: ApiKey, callable_models: list[dict]):
self.api_key = api_key
self.api_key_id = api_key.id
self.callable_models = callable_models
async def get_api_key_dependency(
credentials: HTTPAuthorizationCredentials | None = Depends(security),
db: AsyncSession = Depends(get_db),
) -> ApiKeyContext:
"""FastAPI Dependency: 验证 API Key 并返回上下文。
验证流程:
1. 提取 Bearer <REDACTED>
2. SHA-256 哈希后查询数据库
3. 检查 is_active、deleted_at
4. 检查有效期 (valid_from, valid_until)
5. 检查配额 (quota_limit, quota_used)
6. 重置过期周期的配额
"""
if not credentials:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="缺少 Authorization 头,请提供 Bearer <REDACTED>",
)
token_hash = hashlib.sha256(credentials.credentials.encode()).hexdigest()
result = await db.execute(
select(ApiKey).where(
ApiKey.api_key_hash == token_hash,
ApiKey.is_active == True,
ApiKey.deleted_at.is_(None),
).limit(1)
)
key = result.scalar_one_or_none()
if not key:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="无效的 API Key",
)
now = datetime.now(timezone.utc)
# 检查有效期
if key.valid_from and now < key.valid_from:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="API Key 尚未生效",
)
if key.valid_until and now >= key.valid_until:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="API Key 已过期",
)
# 配额周期重置
from app.services.api_v3.key_service import reset_quota_if_needed
key = await reset_quota_if_needed(db, key)
# 检查配额
if key.quota_limit is not None and key.quota_used >= key.quota_limit:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"API Key 配额已用尽 (已用 {key.quota_used:.2f} / 限额 {key.quota_limit:.2f})",
)
# 解析可调用模型
try:
callable_models = json.loads(key.callable_models) if key.callable_models else []
except (json.JSONDecodeError, TypeError):
callable_models = []
# 更新最后使用时间
key.last_used_at = now
return ApiKeyContext(api_key=key, callable_models=callable_models)
@@ -0,0 +1,132 @@
import json
import logging
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.image_engine import ImageEngine
from app.models.video_engine import VideoEngine
logger = logging.getLogger("videogen")
async def resolve_video_engine(
db: AsyncSession,
engine_id: str,
callable_models: list[dict],
) -> VideoEngine:
"""根据 engine_id 解析视频引擎,并验证是否在 api-key 的可调用列表中。"""
# 验证授权
allowed = {m["engine_id"] for m in callable_models if m.get("engine_type") == "video"}
if engine_id not in allowed:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"该 API Key 无权使用引擎 {engine_id}",
)
result = await db.execute(
select(VideoEngine).where(
VideoEngine.id == engine_id,
VideoEngine.is_active == True,
VideoEngine.deleted_at.is_(None),
).limit(1)
)
engine = result.scalar_one_or_none()
if not engine:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"视频引擎 {engine_id} 不存在或未启用",
)
return engine
async def resolve_image_engine(
db: AsyncSession,
engine_id: str,
callable_models: list[dict],
) -> ImageEngine:
"""根据 engine_id 解析图片引擎,并验证是否在 api-key 的可调用列表中。"""
allowed = {m["engine_id"] for m in callable_models if m.get("engine_type") == "image"}
if engine_id not in allowed:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"该 API Key 无权使用引擎 {engine_id}",
)
result = await db.execute(
select(ImageEngine).where(
ImageEngine.id == engine_id,
ImageEngine.is_active == True,
ImageEngine.deleted_at.is_(None),
).limit(1)
)
engine = result.scalar_one_or_none()
if not engine:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"图片引擎 {engine_id} 不存在或未启用",
)
return engine
def build_engine_snapshot(engine: VideoEngine | ImageEngine) -> dict:
"""构建引擎配置快照。"""
snapshot = {
"id": str(engine.id),
"name": str(engine.name),
"provider": str(getattr(engine, "provider", "")),
"api_base": str(engine.api_base),
"model_name": str(engine.model_name),
}
# 可选字段
for field in [
"supported_ratios", "supported_resolutions", "supported_durations",
"default_size", "multi_generation_enabled", "max_generation_count",
]:
val = getattr(engine, field, None)
if val is not None:
if isinstance(val, str):
try:
val = json.loads(val)
except (json.JSONDecodeError, TypeError):
pass
snapshot[field] = val
return snapshot
async def resolve_engine_by_model_name(
db: AsyncSession,
model_name: str,
callable_models: list[dict],
engine_type: str,
) -> tuple[str, VideoEngine | ImageEngine]:
"""根据模型名称查找对应的引擎。
Returns:
(engine_id, engine 对象)
"""
# 在 callable_models 中查找
target = None
for m in callable_models:
if m.get("model_name") == model_name and m.get("engine_type") == engine_type:
target = m
break
if not target:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"该 API Key 无权使用模型 {model_name}",
)
engine_id = target["engine_id"]
if engine_type == "video":
engine = await resolve_video_engine(db, engine_id, callable_models)
else:
engine = await resolve_image_engine(db, engine_id, callable_models)
return engine_id, engine
@@ -0,0 +1,148 @@
"""API v3 文件下载服务。
下载用户提供的图片/视频/音频到本地存储。
"""
import base64
import logging
import os
import re
import uuid
from datetime import datetime
from urllib.parse import urlparse
import httpx
from app.config import settings
logger = logging.getLogger("videogen")
def _get_date_str() -> str:
"""获取当前日期字符串。"""
return datetime.now().strftime("%Y%m%d")
def _get_uploads_dir() -> str:
"""获取上传文件存储目录。"""
upload_dir = os.path.join(os.path.dirname(settings.STORAGE_LOCAL_PATH), "uploads", "api")
os.makedirs(upload_dir, exist_ok=True)
return upload_dir
async def download_file_from_url(url: str, sub_dir: str = "") -> str:
"""从 URL 下载文件到本地。
Args:
url: 文件 URL
sub_dir: 子目录(如 images/videos/audios
Returns:
相对路径: /uploads/api/{sub_dir}/{date}/{filename}
"""
upload_dir = _get_uploads_dir()
date_str = _get_date_str()
# 创建目标目录
dest_dir = os.path.join(upload_dir, sub_dir, date_str)
os.makedirs(dest_dir, exist_ok=True)
# 从 URL 提取扩展名
parsed = urlparse(url)
path = parsed.path
ext = os.path.splitext(path)[1].lower()
if not ext or len(ext) > 10:
ext = ".bin" # 默认扩展名
# 生成唯一文件名
filename = f"{uuid.uuid4().hex}{ext}"
dest_path = os.path.join(dest_dir, filename)
# 下载文件
async with httpx.AsyncClient(timeout=300, follow_redirects=True) as client:
async with client.stream("GET", url) as response:
response.raise_for_status()
with open(dest_path, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=8192):
f.write(chunk)
# 返回相对路径
rel_path = f"/uploads/api/{sub_dir}/{date_str}/{filename}"
logger.info("Downloaded file: %s -> %s", url[:80], rel_path)
return rel_path
def save_base64_file(data: str, sub_dir: str = "") -> str:
"""保存 Base64 编码的文件到本地。
Args:
data: Base64 编码的数据(可包含 data:...;base64, 前缀)
sub_dir: 子目录
Returns:
相对路径: /uploads/api/{sub_dir}/{date}/{filename}
"""
upload_dir = _get_uploads_dir()
date_str = _get_date_str()
# 创建目标目录
dest_dir = os.path.join(upload_dir, sub_dir, date_str)
os.makedirs(dest_dir, exist_ok=True)
# 解析 Base64 数据
if "," in data:
header, b64_data = data.split(",", 1)
# 从 header 提取 MIME 类型
mime_match = re.search(r"data:([^;]+)", header)
mime_type = mime_match.group(1) if mime_match else "application/octet-stream"
# 根据 MIME 类型确定扩展名
ext_map = {
"image/jpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"video/mp4": ".mp4",
"video/webm": ".webm",
"audio/mpeg": ".mp3",
"audio/wav": ".wav",
"audio/ogg": ".ogg",
}
ext = ext_map.get(mime_type, ".bin")
else:
b64_data = data
ext = ".bin"
# 解码并保存
try:
file_data = base64.b64decode(b64_data)
except Exception as exc:
raise ValueError(f"Base64 解码失败: {exc}")
filename = f"{uuid.uuid4().hex}{ext}"
dest_path = os.path.join(dest_dir, filename)
with open(dest_path, "wb") as f:
f.write(file_data)
rel_path = f"/uploads/api/{sub_dir}/{date_str}/{filename}"
logger.info("Saved base64 file: %s (%d bytes)", rel_path, len(file_data))
return rel_path
async def process_media_url(url: str, media_type: str) -> str:
"""处理媒体 URL:下载到本地或保存 Base64。
Args:
url: URL 或 Base64 数据
media_type: image / video / audio
Returns:
相对路径: /uploads/api/{type}/{date}/{filename}
"""
sub_dir = {"image": "images", "video": "videos", "audio": "audios"}.get(media_type, "files")
# 判断是 Base64 还是 URL
if url.startswith("data:"):
return save_base64_file(url, sub_dir)
else:
return await download_file_from_url(url, sub_dir)
@@ -0,0 +1,494 @@
import asyncio
import json
import logging
import time
from datetime import datetime, timezone
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_key import ApiKey
from app.schemas.api_v3.image import ApiImageGenerateRequest, ApiImageGenerateResponse, ApiImageGenerateDataItem
from app.schemas.api_v3.video import ApiVideoCreateRequest, ApiVideoCreateResponse
from app.services.api_v3 import task_service, engine_service, upscale_service
from app.services.api_v3.quota_service import can_start_video_task
from app.services.api_v3.pricing_service import calc_api_video_price, calc_api_image_price, PricingNotConfiguredError
from app.services.api_v3.logging_service import log_model_request
logger = logging.getLogger("videogen")
async def submit_video_generation(
db: AsyncSession,
key: ApiKey,
callable_models: list[dict],
req: ApiVideoCreateRequest,
) -> ApiVideoCreateResponse:
"""提交视频生成任务(异步)。
流程:
1. 检查并发视频任务数
2. 解析引擎
3. 构建超分快照
4. 创建任务记录
5. 入队 Celery 任务
6. 返回 task_id
"""
# 1. 检查是否可以立即启动(并发限制)
can_start = await can_start_video_task(key, db)
# 2. 解析引擎
engine_id, engine = await engine_service.resolve_engine_by_model_name(
db, req.model, callable_models, "video"
)
engine_snapshot = engine_service.build_engine_snapshot(engine)
# 3. 构建超分快照
provider_resolution, upscale_enabled, upscale_snapshot_json = await upscale_service.build_api_upscale_snapshot(
db, key.id, req.resolution or "480p", req.ratio
)
# 如果超分要求不同的生成分辨率,使用超分的
final_provider_resolution = provider_resolution or req.resolution or "480p"
def _get_max_supported_duration(engine) -> int | None:
"""从引擎 supported_durations 获取最大时长。"""
try:
durations = json.loads(engine.supported_durations) if engine.supported_durations else []
return max(durations) if durations else engine.max_duration
except (json.JSONDecodeError, TypeError, ValueError):
return engine.max_duration
# 3.5 验证传入的媒体文件是否符合引擎配置要求
from app.services.api_v3.file_service import process_media_url
from app.services.video_upscale.media_service import probe_video
from fastapi import HTTPException, status # noqa: F401
input_image_count = 0
input_video_count = 0
input_audio_count = 0
input_video_duration = 0.0
local_media_refs = [] # 存储本地路径
# 统计各类媒体数量
for p in req.content:
ptype = p.type
if ptype == "image_url":
input_image_count += 1
elif ptype == "video_url":
input_video_count += 1
elif ptype == "audio_url":
input_audio_count += 1
# 视频引擎校验(本函数仅处理视频生成)
# 校验图片数量限制
if input_image_count > (engine.max_image_count or 0):
raise HTTPException(
status_code=400,
detail=f"该引擎最多支持 {engine.max_image_count} 张参考图片,当前传入 {input_image_count}",
)
# 校验视频数量限制
if input_video_count > (engine.max_video_count or 0):
raise HTTPException(
status_code=400,
detail=f"该引擎最多支持 {engine.max_video_count} 个参考视频,当前传入 {input_video_count}",
)
# 校验音频数量限制
if input_audio_count > (engine.max_audio_count or 0):
raise HTTPException(
status_code=400,
detail=f"该引擎最多支持 {engine.max_audio_count} 个参考音频,当前传入 {input_audio_count}",
)
for p in req.content:
ptype = p.type
if ptype == "text":
local_media_refs.append(p.model_dump(exclude_none=True))
continue
original_url = ""
if ptype == "image_url" and p.image_url:
original_url = p.image_url.get("url", "")
elif ptype == "video_url" and p.video_url:
original_url = p.video_url.get("url", "")
elif ptype == "audio_url" and p.audio_url:
original_url = p.audio_url.get("url", "")
# 下载文件到本地
try:
local_path = await process_media_url(original_url, ptype.replace("_url", ""))
except Exception as exc:
logger.warning("Failed to download media %s: %s", original_url[:80], exc)
local_path = original_url
# 如果是视频/音频,探测实际时长并校验
if ptype in ("video_url", "audio_url") and local_path:
try:
media_info = await probe_video(local_path)
if media_info and media_info.duration_seconds:
duration = media_info.duration_seconds
# 校验最低时长(2秒)
if duration < 2.0:
raise HTTPException(
status_code=400,
detail=f"上传的{ptype.replace('_url', '')}时长不能低于2秒,当前时长: {duration:.1f}",
)
# 校验最高时长(根据引擎 supported_durations 最大值)
max_duration = _get_max_supported_duration(engine)
if max_duration and duration > max_duration:
raise HTTPException(
status_code=400,
detail=f"上传的{ptype.replace('_url', '')}时长不能超过{max_duration}秒,当前时长: {duration:.1f}",
)
if ptype == "video_url":
input_video_duration += duration
except HTTPException:
raise
except Exception as exc:
logger.warning("Failed to probe media duration: %s", exc)
local_media_refs.append({
"type": ptype,
ptype: {"url": local_path},
"role": p.role,
})
# 3.6 计算价格(基于实际探测的视频时长)
try:
estimated_price = await calc_api_video_price(
db,
duration=req.duration or 5,
resolution=req.resolution or "480p",
engine_id=engine_id,
input_video_duration=input_video_duration,
input_image_count=input_image_count,
)
except PricingNotConfiguredError as exc:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=str(exc),
)
# 预检配额
if key.quota_limit is not None and key.quota_used + estimated_price > key.quota_limit:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"配额不足 (需要 {estimated_price:.2f} 元, 剩余 {key.quota_limit - key.quota_used:.2f} 元)",
)
# 预扣配额
if estimated_price > 0:
key.quota_used = round((key.quota_used or 0.0) + estimated_price, 2)
# 4. 创建任务记录(幂等性已在路由层检查)
content_dicts = [p.model_dump(exclude_none=True) for p in req.content]
task = await task_service.create_video_task(
db=db,
api_key_id=key.id,
model_name=req.model,
engine_id=engine_id,
engine_snapshot=engine_snapshot,
content=content_dicts,
ratio=req.ratio,
duration=req.duration,
resolution=req.resolution,
provider_generation_resolution=final_provider_resolution,
upscale_enabled=upscale_enabled,
upscale_snapshot_json=upscale_snapshot_json,
idempotency_key=req.idempotency_key,
local_media_refs=local_media_refs,
)
task.credits_cost = estimated_price # 记录预扣金额
# 根据并发限制决定立即执行还是排队
if can_start:
# 立即执行
task.status = "pending"
task.pipeline_stage = "queued"
else:
# 排队等待
task.status = "queued"
task.pipeline_stage = "waiting_concurrency"
await db.commit()
# 提交时即记录使用日志(配额已预扣)
try:
from app.services.api_v3.usage_log_service import record_usage
quota_before = key.quota_used - estimated_price # 扣减前的余额
quota_after = key.quota_used # 扣减后的余额
price_detail = {
"base_price": getattr(locals(), "base_price", 0),
"per_second_price": getattr(locals(), "per_second_price", 0),
"duration": req.duration,
"resolution": req.resolution,
"ratio": req.ratio,
"total": estimated_price,
}
await record_usage(
db=db,
api_key_id=key.id,
request_type="video_create",
model_name=req.model,
gen_type="video",
status="success",
task_id=task.id,
credits_cost=estimated_price,
price_action="deduct",
resolution=req.resolution,
duration=req.duration,
quota_before=quota_before,
quota_after=quota_after,
price_detail_json=json.dumps(price_detail, ensure_ascii=False),
)
except Exception as log_exc:
logger.error("Failed to record usage on submit: %s", log_exc)
# 记录模型调用日志
log_model_request(
engine_id=engine_id,
model_name=req.model,
task_id=task.id,
params={
"ratio": req.ratio,
"duration": req.duration,
"resolution": req.resolution,
"generate_audio": req.generate_audio,
"watermark": req.watermark,
"content_count": len(req.content),
"queued": not can_start,
},
)
# 5. 只有立即执行的才入队 Celery
if can_start:
from app.tasks.api_generation_tasks import api_create_generation_task
api_create_generation_task.apply_async(
args=[task.id],
queue="gen_api_create",
)
status_str = "queued" if can_start else "pending_queue"
logger.info("API video task created: task_id=%s model=%s key=%s price=%.2f status=%s", task.id, req.model, key.id, estimated_price, status_str)
return ApiVideoCreateResponse(id=task.id)
async def generate_image_sync(
db: AsyncSession,
key: ApiKey,
callable_models: list[dict],
req: "ApiImageGenerateRequest",
start_time: float,
) -> ApiImageGenerateResponse:
"""同步生成图片。
流程:
1. 解析引擎
2. 创建任务记录
3. 调用 Volcano Ark SDK(同步)
4. 下载图片
5. 更新任务状态
6. 记录使用日志
7. 返回结果
"""
from app.services.api_v3.usage_log_service import record_usage
# 1. 解析引擎
engine_id, engine = await engine_service.resolve_engine_by_model_name(
db, req.model, callable_models, "image"
)
engine_snapshot = engine_service.build_engine_snapshot(engine)
# 2. 创建任务记录
task = await task_service.create_image_task(
db=db,
api_key_id=key.id,
model_name=req.model,
engine_id=engine_id,
engine_snapshot=engine_snapshot,
prompt=req.prompt,
size=req.size,
)
# 2.5 验证传入的媒体文件是否符合引擎配置要求
# 校验参考图片数量限制
input_image_count = len(req.image) if req.image else 0
if input_image_count > (engine.max_reference_image_count or 0):
raise HTTPException(
status_code=400,
detail=f"该引擎最多支持 {engine.max_reference_image_count} 张参考图片,当前传入 {input_image_count}",
)
# 校验组图数量限制
generation_count = req.generation_count or 1
if generation_count > (engine.multi_image_max_images or 1):
raise HTTPException(
status_code=400,
detail=f"该引擎最多支持生成 {engine.multi_image_max_images} 张图片,当前请求 {generation_count}",
)
try:
estimated_price = await calc_api_image_price(
db,
image_size=req.size or "2K",
engine_id=engine_id,
input_image_count=input_image_count,
)
except PricingNotConfiguredError as exc:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=str(exc),
)
# 预检配额
if key.quota_limit is not None and key.quota_used + estimated_price > key.quota_limit:
from fastapi import HTTPException, status
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"配额不足 (需要 {estimated_price:.2f} 元, 剩余 {key.quota_limit - key.quota_used:.2f} 元)",
)
# 预扣配额
if estimated_price > 0:
key.quota_used = round((key.quota_used or 0.0) + estimated_price, 2)
task.credits_cost = estimated_price
await db.commit()
# 提交时即记录使用日志(配额已预扣)
try:
quota_before = key.quota_used - estimated_price
quota_after = key.quota_used
price_detail = {
"base_price": getattr(locals(), "base_price", 0),
"size": req.size,
"generation_count": req.generation_count or 1,
"total": estimated_price,
}
await record_usage(
db=db,
api_key_id=key.id,
request_type="image_generate",
model_name=req.model,
gen_type="image",
status="success",
task_id=task.id,
credits_cost=estimated_price,
price_action="deduct",
resolution=req.size,
quota_before=quota_before,
quota_after=quota_after,
price_detail_json=json.dumps(price_detail, ensure_ascii=False),
)
except Exception as log_exc:
logger.error("Failed to record usage on image submit: %s", log_exc)
try:
# 3. 调用 Volcano Ark SDK(同步函数,在线程中执行)
from app.services.image_gen import submit_image_task, download_image
from app.config import settings
import os
# 构建 media_references,下载图片到本地
from app.services.api_v3.file_service import process_media_url
image_refs = []
if req.image:
for url in req.image:
try:
local_path = await process_media_url(url, "image")
except Exception as exc:
logger.warning("Failed to download image %s: %s", url[:80], exc)
local_path = url
image_refs.append({"type": "image", "url": local_path})
# 临时设置 media_references
task.media_references = json.dumps(image_refs, ensure_ascii=False) if image_refs else None
task.image_size = req.size or "2K"
await db.flush()
# 在线程中执行同步 SDK 调用
result = await asyncio.to_thread(
submit_image_task,
db,
engine,
task,
True, # include_media_references
req.generation_count or 1,
)
# 4. 下载图片
items = result.get("items", [])
downloaded_items: list[ApiImageGenerateDataItem] = []
for item in items:
url = item.get("remote_result_url")
if url:
# 下载到本地
date_dir = datetime.now().strftime("%Y%m%d")
dest_dir = f"./storage/generate/api/images/{date_dir}"
os.makedirs(dest_dir, exist_ok=True)
dest_path = os.path.join(dest_dir, f"{task.id}_{item.get('generation_index', 1)}.png")
try:
await download_image(url, dest_path)
except Exception as dl_err:
logger.warning("Image download failed: %s", dl_err)
downloaded_items.append(ApiImageGenerateDataItem(
url=dest_path, # 使用本地路径
size=item.get("size"),
output_format=item.get("output_format"),
))
elif item.get("error_message"):
downloaded_items.append(ApiImageGenerateDataItem(
url=None,
))
# 5. 使用预扣金额(不再重复扣减)
task.credits_cost = estimated_price
# 6. 更新任务状态
task.status = "completed"
task.pipeline_stage = "done"
task.generated_at = datetime.now(timezone.utc)
if downloaded_items and downloaded_items[0].url:
task.image_url = downloaded_items[0].url
await db.commit()
# 提交时已记录使用日志,成功时无需重复记录
duration_ms = int((time.perf_counter() - start_time) * 1000)
return ApiImageGenerateResponse(
created=result.get("created", int(time.time())),
data=downloaded_items,
model=result.get("model", req.model),
)
except Exception as exc:
# 失败:退回预扣配额
if estimated_price > 0:
key.quota_used = round(max(0, (key.quota_used or 0.0) - estimated_price), 2)
task.status = "failed"
task.error_message = str(exc)
task.credits_cost = 0 # 实际消耗为0(已退回)
await db.commit()
duration_ms = int((time.perf_counter() - start_time) * 1000)
quota_after_refund = key.quota_used # 退回后的余额
await record_usage(
db=db,
api_key_id=key.id,
request_type="image_generate",
model_name=req.model,
gen_type="image",
status="failed",
task_id=task.id,
credits_cost=estimated_price,
refund_amount=estimated_price,
request_duration_ms=duration_ms,
error_message=str(exc),
error_code="generation_failed",
price_action="refund",
resolution=req.size,
generation_count=req.generation_count or 1,
quota_before=quota_after_refund,
quota_after=quota_after_refund + estimated_price,
)
await db.commit()
raise
@@ -0,0 +1,151 @@
import hashlib
import json
import logging
import secrets
from datetime import datetime, timezone
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_key import ApiKey
from app.utils.id_gen import generate_id
logger = logging.getLogger("videogen")
API_KEY_PREFIX = "vk_"
async def create_api_key(
db: AsyncSession,
company_name: str,
callable_models: list[dict] | None = None,
quota_limit: float | None = None,
quota_cycle: str | None = None,
valid_from: datetime | None = None,
valid_until: datetime | None = None,
max_concurrent_video_tasks: int | None = None,
description: str | None = None,
) -> tuple[ApiKey, str]:
"""创建新的 API Key。
Returns:
(ApiKey 对象, 明文 API Key) — 明文仅返回这一次。
"""
# 生成密钥: vk_ + 32字节随机hex
raw_key = API_KEY_PREFIX + secrets.token_hex(24) # vk_ + 48位hex = 51字符
key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
key_prefix = raw_key[:8] # 前8位用于展示: vk_xxxxx
api_key = ApiKey(
id=generate_id(),
company_name=company_name,
api_key_hash=key_hash,
api_key_prefix=key_prefix,
description=description,
callable_models=json.dumps(callable_models or [], ensure_ascii=False),
quota_limit=quota_limit,
quota_cycle=quota_cycle,
quota_used=0.0,
valid_from=valid_from,
valid_until=valid_until,
max_concurrent_video_tasks=max_concurrent_video_tasks,
is_active=True,
)
api_key.set_plaintext_key(raw_key) # 加密存储完整 Key
db.add(api_key)
await db.flush()
logger.info("API Key created: id=%s company=%s prefix=%s", api_key.id, company_name, key_prefix)
return api_key, raw_key
async def list_api_keys(
db: AsyncSession,
skip: int = 0,
limit: int = 50,
company_name: str | None = None,
is_active: bool | None = None,
) -> tuple[int, list[ApiKey]]:
"""列出 API Key(分页+筛选)。"""
from sqlalchemy import func
query = select(ApiKey).where(ApiKey.deleted_at.is_(None))
count_query = select(func.count(ApiKey.id)).where(ApiKey.deleted_at.is_(None))
if company_name:
query = query.where(ApiKey.company_name.ilike(f"%{company_name}%"))
count_query = count_query.where(ApiKey.company_name.ilike(f"%{company_name}%"))
if is_active is not None:
query = query.where(ApiKey.is_active == is_active)
count_query = count_query.where(ApiKey.is_active == is_active)
total_result = await db.execute(count_query)
total = total_result.scalar_one()
query = query.order_by(ApiKey.created_at.desc()).offset(skip).limit(limit)
result = await db.execute(query)
keys = list(result.scalars().all())
return total, keys
async def get_api_key(db: AsyncSession, key_id: str) -> ApiKey | None:
"""获取单个 API Key 详情。"""
result = await db.execute(
select(ApiKey).where(ApiKey.id == key_id, ApiKey.deleted_at.is_(None)).limit(1)
)
return result.scalar_one_or_none()
async def update_api_key(db: AsyncSession, key: ApiKey, **kwargs) -> ApiKey:
"""更新 API Key 配置。"""
updatable_fields = {
"company_name", "description", "callable_models",
"quota_limit", "quota_cycle", "valid_from", "valid_until",
"max_concurrent_video_tasks", "is_active",
}
for field, value in kwargs.items():
if field in updatable_fields and value is not None:
if field == "callable_models" and isinstance(value, list):
value = json.dumps(value, ensure_ascii=False)
setattr(key, field, value)
await db.flush()
return key
async def delete_api_key(db: AsyncSession, key: ApiKey) -> None:
"""软删除 API Key。"""
key.deleted_at = datetime.now(timezone.utc)
key.is_active = False
await db.flush()
async def reset_quota_if_needed(db: AsyncSession, key: ApiKey) -> ApiKey:
"""检查并重置过期周期的配额。
- daily: 如果上次重置不是今天,重置 quota_used=0
- monthly: 如果上次重置不是本月,重置 quota_used=0
"""
if key.quota_limit is None or key.quota_cycle is None:
return key
now = datetime.now(timezone.utc)
# 使用 quota_used 的 updated_at 作为周期判断依据
last_reset = key.updated_at or key.created_at
if last_reset is None:
return key
should_reset = False
if key.quota_cycle == "daily":
should_reset = last_reset.date() < now.date()
elif key.quota_cycle == "monthly":
should_reset = (last_reset.year, last_reset.month) < (now.year, now.month)
if should_reset and key.quota_used > 0:
key.quota_used = 0.0
await db.flush()
logger.info("Quota reset for API Key %s (cycle=%s)", key.id, key.quota_cycle)
return key
@@ -0,0 +1,187 @@
"""外部 API v3 日志服务。
按天分类存储在 log/api/ 目录下:
- log/api/requests/YYYY-MM-DD.log — 所有外部请求和响应
- log/api/models/YYYY-MM-DD.log — 模型调用(Volcano Ark SDK
- log/api/upscale/YYYY-MM-DD.log — 超分轮询
- log/api/errors/YYYY-MM-DD.log — 错误日志
"""
import json
import logging
import os
from datetime import datetime, timezone
# === 日志目录 ===
# video-gen-api/log/api/
_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
# 上溯3级: services/api_v3 -> services -> app -> video-gen-api (即项目根目录)
_BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_THIS_DIR)))
BASE_LOG_DIR = os.path.join(_BASE_DIR, "log", "api")
os.makedirs(BASE_LOG_DIR, exist_ok=True)
# 子目录
REQUESTS_LOG_DIR = os.path.join(BASE_LOG_DIR, "requests")
MODELS_LOG_DIR = os.path.join(BASE_LOG_DIR, "models")
UPSCALE_LOG_DIR = os.path.join(BASE_LOG_DIR, "upscale")
ERRORS_LOG_DIR = os.path.join(BASE_LOG_DIR, "errors")
for d in [REQUESTS_LOG_DIR, MODELS_LOG_DIR, UPSCALE_LOG_DIR, ERRORS_LOG_DIR]:
os.makedirs(d, exist_ok=True)
def _get_date_str() -> str:
"""获取当前日期字符串。"""
return datetime.now(timezone.utc).strftime("%Y-%m-%d")
class _DailyFileHandler(logging.Handler):
"""按天写入的日志处理器。"""
def __init__(self, log_dir: str):
super().__init__()
self.log_dir = log_dir
self._current_date = None
self._file_handler = None
self._open_file()
def _open_file(self):
"""打开当天的日志文件。"""
date_str = _get_date_str()
if date_str == self._current_date and self._file_handler:
return
if self._file_handler:
self._file_handler.close()
self._current_date = date_str
filepath = os.path.join(self.log_dir, f"{date_str}.log")
self._file_handler = logging.FileHandler(filepath, encoding="utf-8")
self._file_handler.setFormatter(
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
)
def emit(self, record):
try:
self._open_file()
self._file_handler.emit(record)
except Exception:
self.handleError(record)
def close(self):
if self._file_handler:
self._file_handler.close()
super().close()
def _create_logger(name: str, log_dir: str) -> logging.Logger:
"""创建按天写入的 Logger。"""
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
# 避免重复添加 handler
if not logger.handlers:
handler = _DailyFileHandler(log_dir)
logger.addHandler(handler)
return logger
# === Logger 实例 ===
requests_logger = _create_logger("api_v3.requests", REQUESTS_LOG_DIR)
models_logger = _create_logger("api_v3.models", MODELS_LOG_DIR)
upscale_logger = _create_logger("api_v3.upscale", UPSCALE_LOG_DIR)
errors_logger = _create_logger("api_v3.errors", ERRORS_LOG_DIR)
def _safe_json(obj) -> str:
"""安全地序列化为 JSON。"""
try:
return json.dumps(obj, ensure_ascii=False, default=str)
except Exception:
return str(obj)
# === 请求/响应日志 ===
def log_request(method: str, path: str, api_key_id: str, body: dict | None = None):
"""记录外部请求。"""
requests_logger.info(
f"REQUEST | {method} {path} | key={api_key_id} | body={_safe_json(body)}"
)
def log_response(method: str, path: str, api_key_id: str, status_code: int, body=None, duration_ms: int = 0):
"""记录外部响应。"""
requests_logger.info(
f"RESPONSE | {method} {path} | key={api_key_id} | status={status_code} | duration={duration_ms}ms | body={_safe_json(body)}"
)
def log_request_error(method: str, path: str, api_key_id: str, error: str, status_code: int = 500):
"""记录请求错误。"""
errors_logger.error(
f"REQUEST_ERROR | {method} {path} | key={api_key_id} | status={status_code} | error={error}"
)
# === 模型调用日志 ===
def log_model_request(engine_id: str, model_name: str, task_id: str, params: dict):
"""记录模型调用请求。"""
models_logger.info(
f"MODEL_REQUEST | engine={engine_id} | model={model_name} | task={task_id} | params={_safe_json(params)}"
)
def log_model_response(engine_id: str, model_name: str, task_id: str, success: bool, result: dict | None = None, error: str | None = None):
"""记录模型调用响应。"""
if success:
models_logger.info(
f"MODEL_RESPONSE | engine={engine_id} | model={model_name} | task={task_id} | success | result={_safe_json(result)}"
)
else:
models_logger.error(
f"MODEL_RESPONSE | engine={engine_id} | model={model_name} | task={task_id} | failed | error={error}"
)
errors_logger.error(
f"MODEL_ERROR | engine={engine_id} | model={model_name} | task={task_id} | error={error}"
)
# === 超分轮询日志 ===
def log_upscale_poll_start(task_id: str, api_task_id: str):
"""记录超分轮询开始。"""
upscale_logger.info(f"UPSCALE_POLL_START | task={task_id} | api_task={api_task_id}")
def log_upscale_poll(task_id: str, api_task_id: str, status: str, attempt: int, result: dict | None = None):
"""记录超分轮询状态。"""
upscale_logger.info(
f"UPSCALE_POLL | task={task_id} | api_task={api_task_id} | status={status} | attempt={attempt} | result={_safe_json(result)}"
)
def log_upscale_poll_end(task_id: str, api_task_id: str, success: bool, final_status: str, total_attempts: int):
"""记录超分轮询结束。"""
if success:
upscale_logger.info(
f"UPSCALE_POLL_END | task={task_id} | api_task={api_task_id} | success | status={final_status} | attempts={total_attempts}"
)
else:
upscale_logger.error(
f"UPSCALE_POLL_END | task={task_id} | api_task={api_task_id} | failed | status={final_status} | attempts={total_attempts}"
)
errors_logger.error(
f"UPSCALE_ERROR | task={task_id} | api_task={api_task_id} | status={final_status} | attempts={total_attempts}"
)
# === 通用错误日志 ===
def log_error(category: str, message: str, details: dict | None = None):
"""记录通用错误。"""
errors_logger.error(
f"{category} | {message} | details={_safe_json(details)}"
)
@@ -0,0 +1,173 @@
import logging
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_model_pricing import ApiModelPricing
from app.models.image_engine import ImageEngine
from app.models.video_engine import VideoEngine
logger = logging.getLogger("videogen")
class PricingNotConfiguredError(Exception):
"""模型+分辨率组合未配置价格。"""
def __init__(self, *, model_name: str, resolution: str):
self.model_name = model_name
self.resolution = resolution
super().__init__(
f"模型或引擎 '{self.model_name}' 在分辨率 '{self.resolution}' 下未配置,无法生成"
)
async def resolve_engine_display_name(db: AsyncSession, engine_id: str) -> str:
"""根据引擎 ID 解析展示名称(找不到时原样返回 ID)。"""
if not engine_id:
return engine_id or "unknown"
result = await db.execute(
select(VideoEngine.name).where(VideoEngine.id == engine_id, VideoEngine.deleted_at.is_(None)).limit(1)
)
name = result.scalar_one_or_none()
if name:
return name
result = await db.execute(
select(ImageEngine.name).where(ImageEngine.id == engine_id, ImageEngine.deleted_at.is_(None)).limit(1)
)
name = result.scalar_one_or_none()
return name or engine_id
async def _get_api_pricing(
db: AsyncSession,
*,
gen_type: str,
resolution: str,
engine_id: str | None = None,
) -> ApiModelPricing | None:
"""按引擎精确规则优先获取定价;找不到时回退到同类型同分辨率。
查询优先级:
1. gen_type + engine_id + resolution 精确规则
2. gen_type + resolution 下 base_price 最高规则
"""
gen_type = (gen_type or "").lower().strip()
resolution = (resolution or "").strip()
engine_id = (engine_id or "").strip() or None
if engine_id:
result = await db.execute(
select(ApiModelPricing)
.where(ApiModelPricing.gen_type == gen_type)
.where(ApiModelPricing.model_config_id == engine_id)
.where(ApiModelPricing.resolution == resolution)
.order_by(ApiModelPricing.base_price.desc(), ApiModelPricing.per_second_price.desc())
.limit(1)
)
pricing = result.scalar_one_or_none()
if pricing:
return pricing
result = await db.execute(
select(ApiModelPricing)
.where(ApiModelPricing.gen_type == gen_type)
.where(ApiModelPricing.resolution == resolution)
.order_by(ApiModelPricing.base_price.desc(), ApiModelPricing.per_second_price.desc())
.limit(1)
)
return result.scalar_one_or_none()
async def calc_api_video_price(
db: AsyncSession,
duration: int,
resolution: str,
engine_id: str | None = None,
input_video_duration: float = 0,
input_image_count: int = 0,
) -> float:
"""计算 API 视频生成价格(元)。
未配置价格时抛出 PricingNotConfiguredError。
公式(与 credit_ratios 一致):
base_cost = (base_price + per_second_price × duration) × price_ratio
if 传入视频: += (input_video_base_price + input_video_per_second_price × input_video_duration) × input_video_ratio
if 传入图片: += (input_image_base_price + input_image_per_image_price × input_image_count) × input_image_ratio
"""
if not engine_id:
result = await db.execute(
select(VideoEngine.id)
.where(VideoEngine.is_active == True, VideoEngine.deleted_at.is_(None))
.order_by(VideoEngine.priority.desc())
.limit(1)
)
engine_id = result.scalar_one_or_none()
pricing = await _get_api_pricing(db, gen_type="video", resolution=resolution, engine_id=engine_id)
if not pricing:
raise PricingNotConfiguredError(
model_name=await resolve_engine_display_name(db, engine_id),
resolution=resolution,
)
# 基础价格
base_cost = (pricing.base_price + pricing.per_second_price * duration) * pricing.price_ratio
# 传入视频附加费(每秒 × 倍率)
if input_video_duration > 0:
base_cost += (pricing.input_video_base_price + pricing.input_video_per_second_price * input_video_duration) * pricing.input_video_ratio
# 传入图片附加费(每张 × 倍率)
if input_image_count > 0:
base_cost += (pricing.input_image_base_price + pricing.input_image_per_image_price * input_image_count) * pricing.input_image_ratio
return round(base_cost, 2)
async def calc_api_image_price(
db: AsyncSession,
image_size: str,
engine_id: str | None = None,
input_image_count: int = 0,
) -> float:
"""计算 API 图片生成价格(元)。
未配置价格时抛出 PricingNotConfiguredError。
公式(与 credit_ratios 一致):
base_cost = base_price × price_ratio
if 传入图片: += (input_image_base_price + input_image_per_image_price × input_image_count) × input_image_ratio
"""
if not engine_id:
result = await db.execute(
select(ImageEngine.id)
.where(ImageEngine.is_active == True, ImageEngine.deleted_at.is_(None))
.order_by(ImageEngine.priority.desc())
.limit(1)
)
engine_id = result.scalar_one_or_none()
pricing = await _get_api_pricing(db, gen_type="image", resolution=image_size, engine_id=engine_id)
if not pricing:
raise PricingNotConfiguredError(
model_name=await resolve_engine_display_name(db, engine_id),
resolution=image_size,
)
# 基础价格
base_cost = pricing.base_price * pricing.price_ratio
# 传入图片附加费(每张 × 倍率)
if input_image_count > 0:
base_cost += (pricing.input_image_base_price + pricing.input_image_per_image_price * input_image_count) * pricing.input_image_ratio
return round(base_cost, 2)
async def get_priced_models(db: AsyncSession) -> set[str]:
"""获取所有已配置价格的引擎 ID 集合。
用于过滤 /api/v3/models 接口,仅返回已定价的模型。
"""
result = await db.execute(
select(ApiModelPricing.model_config_id).distinct()
)
return {row[0] for row in result.all()}
@@ -0,0 +1,68 @@
import logging
from datetime import datetime, timezone
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_generation_task import ApiGenerationTask
from app.models.api.api_key import ApiKey
logger = logging.getLogger("videogen")
async def check_quota(key: ApiKey) -> bool:
"""检查 API Key 配额是否充足。
Returns:
True = 配额充足或无限额, False = 已超限。
"""
if key.quota_limit is None:
return True
return key.quota_used < key.quota_limit
async def get_active_video_tasks_count(api_key_id: str, db: AsyncSession) -> int:
"""统计 API Key 当前活跃的视频任务数。
活跃 = status IN ('pending', 'generating', 'processing') AND gen_type='video'
"""
result = await db.execute(
select(func.count(ApiGenerationTask.id)).where(
ApiGenerationTask.api_key_id == api_key_id,
ApiGenerationTask.gen_type == "video",
ApiGenerationTask.status.in_(["pending", "generating", "processing"]),
ApiGenerationTask.deleted_at.is_(None),
)
)
return result.scalar_one() or 0
async def can_start_video_task(key: ApiKey, db: AsyncSession) -> bool:
"""检查是否可以立即启动新的视频任务。
Returns:
True = 可以立即启动, False = 需要排队。
"""
if key.max_concurrent_video_tasks is None:
return True # 无限制
current = await get_active_video_tasks_count(key.id, db)
return current < key.max_concurrent_video_tasks
async def get_queued_video_tasks(key: ApiKey, db: AsyncSession, limit: int = 10) -> list[ApiGenerationTask]:
"""获取排队的视频任务列表(按创建时间排序)。"""
result = await db.execute(
select(ApiGenerationTask).where(
ApiGenerationTask.api_key_id == key.id,
ApiGenerationTask.gen_type == "video",
ApiGenerationTask.status == "queued",
ApiGenerationTask.deleted_at.is_(None),
).order_by(ApiGenerationTask.created_at.asc()).limit(limit)
)
return list(result.scalars().all())
async def increment_quota(db: AsyncSession, key: ApiKey, credits_cost: float) -> None:
"""原子性增加配额使用量。"""
key.quota_used = round((key.quota_used or 0.0) + credits_cost, 2)
await db.flush()
@@ -0,0 +1,215 @@
import json
import logging
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_generation_task import ApiGenerationTask
from app.schemas.api_v3.video import ApiVideoStatusResponse
from app.utils.id_gen import generate_id
logger = logging.getLogger("videogen")
async def create_video_task(
db: AsyncSession,
api_key_id: str,
model_name: str,
engine_id: str,
engine_snapshot: dict,
content: list[dict],
ratio: str | None,
duration: int | None,
resolution: str | None,
provider_generation_resolution: str | None,
upscale_enabled: bool,
upscale_snapshot_json: str | None,
idempotency_key: str | None = None,
local_media_refs: list[dict] | None = None,
) -> ApiGenerationTask:
"""创建视频生成任务记录。"""
# 提取文本提示词
text_parts = [p.get("text", "") for p in content if p.get("type") == "text"]
original_prompt = " ".join(text_parts) if text_parts else content[0].get("text", "") if content else ""
# 构建 media_references(扁平格式,便于外部读取)
# 构建 local_media_json(嵌套格式,与 Volcano SDK 兼容)
from app.services.api_v3.file_service import process_media_url
media_refs = [] # 扁平格式: {"type": "image", "url": "...", "role": "..."}
local_media_refs = [] # 本地下载路径(嵌套格式)
for p in content:
ptype = p.get("type", "")
if ptype == "text":
continue
# 提取原始 URL(从嵌套格式中提取)
original_url = ""
media_type = ptype.replace("_url", "") # image_url -> image
if ptype == "image_url" and p.get("image_url"):
original_url = p["image_url"].get("url", "")
elif ptype == "video_url" and p.get("video_url"):
original_url = p["video_url"].get("url", "")
elif ptype == "audio_url" and p.get("audio_url"):
original_url = p["audio_url"].get("url", "")
# 存储扁平格式到 media_references
media_refs.append({
"type": media_type,
"url": original_url,
"role": p.get("role"),
})
# 下载文件到本地
try:
local_path = await process_media_url(original_url, media_type)
except Exception as exc:
logger.warning("Failed to download media %s: %s", original_url[:80], exc)
local_path = original_url
# 本地路径使用嵌套格式(与 Volcano SDK 兼容)
local_media_refs.append({
"type": ptype,
ptype: {"url": local_path},
"role": p.get("role"),
})
media_references_json = json.dumps(media_refs, ensure_ascii=False) if media_refs else None
local_media_json = json.dumps(local_media_refs, ensure_ascii=False) if local_media_refs else None
now = datetime.now(timezone.utc)
deadline = now + timedelta(hours=24)
task = ApiGenerationTask(
id=generate_id(),
api_key_id=api_key_id,
external_idempotency_key=idempotency_key,
original_prompt=original_prompt,
gen_type="video",
model_name=model_name,
duration=duration,
aspect_ratio=ratio,
resolution=resolution,
provider_generation_resolution=provider_generation_resolution,
generation_count=1,
engine_id=engine_id,
media_references=media_references_json,
local_media_json=local_media_json,
engine_snapshot_json=json.dumps(engine_snapshot, ensure_ascii=False),
status="pending",
pipeline_stage="queued",
deadline_at=deadline,
video_upscale_enabled_snapshot=upscale_enabled,
video_upscale_snapshot_json=upscale_snapshot_json,
)
db.add(task)
await db.flush()
return task
async def create_image_task(
db: AsyncSession,
api_key_id: str,
model_name: str,
engine_id: str,
engine_snapshot: dict,
prompt: str,
size: str | None,
idempotency_key: str | None = None,
) -> ApiGenerationTask:
"""创建图片生成任务记录。"""
task = ApiGenerationTask(
id=generate_id(),
api_key_id=api_key_id,
external_idempotency_key=idempotency_key,
original_prompt=prompt,
gen_type="image",
image_size=size,
generation_count=1,
engine_id=engine_id,
engine_snapshot_json=json.dumps(engine_snapshot, ensure_ascii=False),
status="processing",
pipeline_stage="creating_provider_task",
)
db.add(task)
await db.flush()
return task
async def get_task(db: AsyncSession, task_id: str, api_key_id: str) -> ApiGenerationTask | None:
"""获取任务(带所有权验证)。"""
result = await db.execute(
select(ApiGenerationTask).where(
ApiGenerationTask.id == task_id,
ApiGenerationTask.api_key_id == api_key_id,
ApiGenerationTask.deleted_at.is_(None),
).limit(1)
)
return result.scalar_one_or_none()
async def find_by_idempotency_key(db: AsyncSession, api_key_id: str, idempotency_key: str) -> ApiGenerationTask | None:
"""根据幂等键查找已存在的任务。"""
result = await db.execute(
select(ApiGenerationTask).where(
ApiGenerationTask.api_key_id == api_key_id,
ApiGenerationTask.external_idempotency_key == idempotency_key,
ApiGenerationTask.deleted_at.is_(None),
).limit(1)
)
return result.scalar_one_or_none()
def map_task_to_status_response(task: ApiGenerationTask) -> ApiVideoStatusResponse:
"""将任务对象映射为状态查询响应。"""
from app.config import settings
# 返回完整 URL(包含 BASE_URL
video_url = _make_full_url(task.video_url)
video_cover_url = _make_full_url(task.video_cover_url)
return ApiVideoStatusResponse(
task_id=task.id,
status=_map_status(task.status),
video_url=video_url,
video_cover_url=video_cover_url,
duration=task.duration,
ratio=task.aspect_ratio,
resolution=task.resolution,
error=task.error_message,
created_at=task.created_at,
completed_at=task.generated_at,
)
def _make_full_url(path: str | None) -> str | None:
"""将本地路径转换为完整 URL。"""
if not path:
return None
from app.config import settings
# 如果已经是完整 URL,直接返回
if path.startswith(("http://", "https://")):
return path
# 处理 ./storage/generate/... 格式 → /generate/...
if path.startswith("./storage"):
url_path = path[len("./storage"):]
elif path.startswith("/"):
url_path = path
else:
url_path = f"/{path}"
# 拼接 BASE_URL
base = settings.BASE_URL.rstrip("/")
return f"{base}{url_path}"
def _map_status(status: str) -> str:
"""将内部状态映射为 API 状态。"""
status_map = {
"pending": "queued",
"queued": "pending_queue",
"generating": "generating",
"processing": "generating",
"completed": "completed",
"failed": "failed",
}
return status_map.get(status, status)
@@ -0,0 +1,212 @@
import json
import logging
from datetime import datetime, timezone
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_generation_task import ApiGenerationTask
from app.models.api.api_key_upscale_config import ApiKeyUpscaleConfig
from app.models.api.api_upscale_link import ApiUpscaleLink
from app.utils.id_gen import generate_id
logger = logging.getLogger("videogen")
async def get_or_create_upscale_config(
db: AsyncSession,
api_key_id: str,
) -> ApiKeyUpscaleConfig:
"""获取或创建 API Key 的超分配置。"""
result = await db.execute(
select(ApiKeyUpscaleConfig).where(
ApiKeyUpscaleConfig.api_key_id == api_key_id
).limit(1)
)
config = result.scalar_one_or_none()
if not config:
config = ApiKeyUpscaleConfig(
id=generate_id(),
api_key_id=api_key_id,
enabled=False,
delete_source_after_success=True,
rules_json="[]",
)
db.add(config)
await db.flush()
return config
async def save_upscale_config(
db: AsyncSession,
api_key_id: str,
enabled: bool,
delete_source_after_success: bool,
rules: list[dict],
) -> ApiKeyUpscaleConfig:
"""保存 API Key 的超分配置。"""
config = await get_or_create_upscale_config(db, api_key_id)
config.enabled = enabled
config.delete_source_after_success = delete_source_after_success
config.rules_json = json.dumps(rules, ensure_ascii=False)
await db.flush()
return config
async def build_api_upscale_snapshot(
db: AsyncSession,
api_key_id: str,
target_resolution: str,
aspect_ratio: str | None = None,
) -> tuple[str | None, bool, str | None]:
"""构建 API 超分快照。
读取 api_key_upscale_configs(而非 system_configs),
匹配目标分辨率对应的超分规则。
Returns:
(provider_generation_resolution, enabled, snapshot_json)
"""
config = await get_or_create_upscale_config(db, api_key_id)
if not config.enabled:
return None, False, None
try:
rules = json.loads(config.rules_json) if config.rules_json else []
except (json.JSONDecodeError, TypeError):
return None, False, None
# 匹配规则
matched_rule = None
for rule in rules:
if rule.get("enabled") and rule.get("target_resolution") == target_resolution:
matched_rule = rule
break
if not matched_rule:
return None, False, None
snapshot = {
"enabled": True,
"delete_source_after_success": config.delete_source_after_success,
"rule": matched_rule,
"matched_at": datetime.now(timezone.utc).isoformat(),
# 兼容现有超分流水线的 processor 字段
"processor": {
"max_attempts": 3,
"processor_key": matched_rule.get("processor_key", "volc_large_model_v1"),
},
"target_resolution": target_resolution,
"provider_generation_resolution": matched_rule.get("provider_generation_resolution", target_resolution),
"aspect_ratio": aspect_ratio,
}
provider_resolution = matched_rule.get("provider_generation_resolution", target_resolution)
snapshot_json = json.dumps(snapshot, ensure_ascii=False)
return provider_resolution, True, snapshot_json
async def prepare_api_upscale_task(
db: AsyncSession,
api_task: ApiGenerationTask,
source_local_path: str,
source_width: int = 0,
source_height: int = 0,
source_duration: float = 0.0,
) -> "VideoUpscaleTask | None":
"""为 API 任务创建超分子任务。
复用现有的 VideoUpscaleTask 表和 upscale 执行流水线。
如果已存在超分任务则返回 None(避免重复创建)。
"""
from app.models.video_upscale_task import VideoUpscaleTask
from sqlalchemy import select
# 检查是否已存在超分任务(避免重复创建)
existing = await db.execute(
select(VideoUpscaleTask).where(
VideoUpscaleTask.api_generation_task_id == api_task.id
).limit(1)
)
if existing.scalar_one_or_none():
logger.info("Upscale task already exists for API task %s, skipping", api_task.id)
return None
# 解析快照获取处理器配置
try:
snapshot = json.loads(api_task.video_upscale_snapshot_json) if api_task.video_upscale_snapshot_json else {}
except (json.JSONDecodeError, TypeError):
snapshot = {}
rule = snapshot.get("rule", {})
processor_key = rule.get("processor_key", "volc_large_model_v1")
target_resolution = rule.get("target_resolution", api_task.resolution or "1080p")
# 计算目标尺寸
target_width, target_height = _resolution_to_dimensions(target_resolution, api_task.aspect_ratio)
upscale_task = VideoUpscaleTask(
id=generate_id(),
chat_generation_task_id=None,
generation_record_id=None,
api_generation_task_id=api_task.id, # 关联 API v3 任务
processor_key=processor_key,
target_width=target_width,
target_height=target_height,
effective_target_width=target_width,
effective_target_height=target_height,
source_local_path=api_task.local_path or source_local_path, # 优先使用已下载的本地文件
source_remote_url=api_task.remote_result_url, # 火山 MediaKit 需要远程 URL
input_source_type="provider_remote",
source_file_size_bytes=0,
source_width=source_width,
source_height=source_height,
source_duration_seconds=source_duration,
status="pending",
stage="upscale_queued",
)
db.add(upscale_task)
await db.flush()
# 创建关联记录
link = ApiUpscaleLink(
id=generate_id(),
api_generation_task_id=api_task.id,
video_upscale_task_id=upscale_task.id,
)
db.add(link)
await db.flush()
logger.info(
"API upscale task prepared: api_task=%s upscale_task=%s processor=%s",
api_task.id, upscale_task.id, processor_key,
)
return upscale_task
def _resolution_to_dimensions(resolution: str, aspect_ratio: str | None) -> tuple[int, int]:
"""将分辨率名称转换为像素尺寸。"""
# 标准分辨率映射
resolution_map = {
"480p": (852, 480),
"720p": (1280, 720),
"1080p": (1920, 1080),
"2K": (2560, 1440),
"4K": (3840, 2160),
}
base = resolution_map.get(resolution, (1920, 1080))
# 根据宽高比调整
if aspect_ratio == "9:16":
return (base[1], base[0]) # 竖屏
elif aspect_ratio == "1:1":
return (base[0], base[0]) # 正方形
elif aspect_ratio == "4:3":
return (base[0], int(base[0] * 3 / 4))
return base
@@ -0,0 +1,150 @@
import logging
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.api.api_usage_log import ApiUsageLog
from app.utils.id_gen import generate_id
logger = logging.getLogger("videogen")
async def record_usage(
db: AsyncSession,
api_key_id: str,
request_type: str,
model_name: str,
gen_type: str,
status: str,
task_id: str | None = None,
credits_cost: float = 0.0,
tokens_used: int = 0,
request_duration_ms: int = 0,
error_message: str | None = None,
error_code: str | None = None,
request_payload_json: str | None = None,
price_action: str | None = None,
resolution: str | None = None,
duration: int | None = None,
refund_amount: float | None = None,
quota_before: float | None = None,
quota_after: float | None = None,
price_detail_json: str | None = None,
) -> ApiUsageLog:
"""记录一次 API 调用日志。"""
# 确定 price_action
if price_action:
action = price_action
elif status == "failed":
action = "refund"
else:
action = "deduct"
log = ApiUsageLog(
id=generate_id(),
api_key_id=api_key_id,
api_generation_task_id=task_id,
price_action=action,
request_type=request_type,
model_name=model_name,
gen_type=gen_type,
resolution=resolution,
duration=duration,
credits_cost=credits_cost,
refund_amount=refund_amount or 0.0,
quota_before=quota_before,
quota_after=quota_after,
tokens_used=tokens_used,
request_duration_ms=request_duration_ms,
price_detail_json=price_detail_json,
status=status,
error_message=error_message,
error_code=error_code,
request_payload_json=request_payload_json,
)
db.add(log)
await db.flush()
return log
async def list_usage_logs(
db: AsyncSession,
api_key_id: str | None = None,
skip: int = 0,
limit: int = 50,
start_date: datetime | None = None,
end_date: datetime | None = None,
) -> tuple[int, list[ApiUsageLog]]:
"""查询使用日志(分页+筛选)。"""
query = select(ApiUsageLog)
count_query = select(func.count(ApiUsageLog.id))
filters = []
if api_key_id:
filters.append(ApiUsageLog.api_key_id == api_key_id)
if start_date:
filters.append(ApiUsageLog.created_at >= start_date)
if end_date:
filters.append(ApiUsageLog.created_at <= end_date)
for f in filters:
query = query.where(f)
count_query = count_query.where(f)
total_result = await db.execute(count_query)
total = total_result.scalar_one()
query = query.order_by(ApiUsageLog.created_at.desc()).offset(skip).limit(limit)
result = await db.execute(query)
logs = list(result.scalars().all())
return total, logs
async def get_usage_summary(
db: AsyncSession,
api_key_id: str | None = None,
days: int = 30,
) -> dict:
"""获取使用汇总统计。"""
now = datetime.now(timezone.utc)
start = now - timedelta(days=days)
query = select(
func.count(ApiUsageLog.id).label("total_requests"),
func.coalesce(func.sum(ApiUsageLog.credits_cost), 0).label("total_credits"),
func.coalesce(func.sum(ApiUsageLog.tokens_used), 0).label("total_tokens"),
func.coalesce(func.avg(ApiUsageLog.request_duration_ms), 0).label("avg_duration"),
).where(ApiUsageLog.created_at >= start)
if api_key_id:
query = query.where(ApiUsageLog.api_key_id == api_key_id)
result = await db.execute(query)
row = result.one()
# 成功/失败统计
success_query = select(func.count(ApiUsageLog.id)).where(
ApiUsageLog.created_at >= start,
ApiUsageLog.status == "success",
)
failed_query = select(func.count(ApiUsageLog.id)).where(
ApiUsageLog.created_at >= start,
ApiUsageLog.status == "failed",
)
if api_key_id:
success_query = success_query.where(ApiUsageLog.api_key_id == api_key_id)
failed_query = failed_query.where(ApiUsageLog.api_key_id == api_key_id)
success_result = await db.execute(success_query)
failed_result = await db.execute(failed_query)
return {
"total_requests": row.total_requests or 0,
"total_credits_cost": float(row.total_credits or 0),
"total_tokens_used": int(row.total_tokens or 0),
"avg_duration_ms": int(row.avg_duration or 0),
"success_count": success_result.scalar_one() or 0,
"failed_count": failed_result.scalar_one() or 0,
}
+6 -6
View File
@@ -59,15 +59,15 @@ AI_LOG_ENABLED: bool = True # Set True to enable logging, or use env var AI_LOG
# ── Log output settings ────────────────────────────────────
LOG_DIR = os.path.join(
BASE_LOG_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"log", "AiModel",
"log",
)
LOG_DIR = os.path.join(BASE_LOG_DIR, "AiModel")
# 请求响应日志目录
LOG_R_Q_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"log", "RequestResponse",
)
LOG_R_Q_DIR = os.path.join(BASE_LOG_DIR, "RequestResponse")
# VP V3 虚拟素材库专用日志目录
VP_V3_LOG_DIR = os.path.join(BASE_LOG_DIR, "virtual_portrait_v3")
LOG_FILENAME_FORMAT = "{date}.log" # e.g. 2026-05-12.log
LOG_DATE_FORMAT = "%Y-%m-%d"
@@ -298,6 +298,7 @@ def _base_entry(
step_id: str | None = None,
remote_action: str | None = None,
remote_request_id: str | None = None,
api_key_id: str | None = None,
message: str | None = None,
detail: dict[str, Any] | None = None,
error: str | None = None,
@@ -321,6 +322,7 @@ def _base_entry(
"step_id": step_id,
"remote_action": remote_action,
"remote_request_id": remote_request_id,
"api_key_id": api_key_id,
"message": message,
"detail": detail or {},
"error": error,
@@ -345,6 +347,7 @@ def log_operation_event(
step_id: str | None = None,
remote_action: str | None = None,
remote_request_id: str | None = None,
api_key_id: str | None = None,
message: str | None = None,
detail: dict[str, Any] | None = None,
error: str | None = None,
@@ -369,6 +372,7 @@ def log_operation_event(
step_id=step_id,
remote_action=remote_action,
remote_request_id=remote_request_id,
api_key_id=api_key_id,
message=message,
detail=detail,
error=error,
@@ -390,6 +394,7 @@ def log_module_generation_event(
step_id: str | None = None,
remote_action: str | None = None,
remote_request_id: str | None = None,
api_key_id: str | None = None,
message: str | None = None,
detail: dict[str, Any] | None = None,
error: str | None = None,
@@ -413,6 +418,7 @@ def log_module_generation_event(
step_id=step_id,
remote_action=remote_action,
remote_request_id=remote_request_id,
api_key_id=api_key_id,
message=message,
detail=detail,
error=error,
@@ -30,23 +30,36 @@ def owner_id(owner: VideoUpscaleOwner | None) -> str | None:
def owner_is_generating(owner: VideoUpscaleOwner) -> bool:
if isinstance(owner, ChatGenerationTask):
return owner.status == ChatGenerationTaskStatus.GENERATING.value
if hasattr(owner, "api_key_id"):
# ApiGenerationTask
return owner.status in ("generating", "processing", "pending")
return owner.status == GenerationStatus.generating.value
def owner_is_completed(owner: VideoUpscaleOwner) -> bool:
if isinstance(owner, ChatGenerationTask):
return owner.status == ChatGenerationTaskStatus.COMPLETED.value
if hasattr(owner, "api_key_id"):
# ApiGenerationTask
return owner.status == "completed"
return owner.status == GenerationStatus.completed.value
def set_owner_stage(owner: VideoUpscaleOwner, stage: str) -> None:
owner.pipeline_stage = stage
# ApiGenerationTask 没有 pipeline_stage 字段,使用 stage 字段
if hasattr(owner, "pipeline_stage"):
owner.pipeline_stage = stage
elif hasattr(owner, "stage"):
owner.stage = stage
def upscale_stage_value(owner: VideoUpscaleOwner, chat_stage: ChatGenerationPipelineStage | str) -> str:
value = chat_stage.value if hasattr(chat_stage, "value") else str(chat_stage)
if isinstance(owner, ChatGenerationTask):
return value
if hasattr(owner, "api_key_id"):
# ApiGenerationTask - 直接返回 stage 值
return value
try:
return GenerationRecordPipelineStage(value).value
except ValueError:
@@ -69,6 +82,13 @@ async def load_upscale_owner(
GenerationRecord.id == upscale.generation_record_id,
GenerationRecord.deleted_at.is_(None),
)
elif upscale.api_generation_task_id:
# API v3 任务
from app.models.api.api_generation_task import ApiGenerationTask
query = select(ApiGenerationTask).where(
ApiGenerationTask.id == upscale.api_generation_task_id,
ApiGenerationTask.deleted_at.is_(None),
)
else:
return None
if for_update:
@@ -379,7 +379,8 @@ async def _claim(
return None
if upscale.status in {VideoUpscaleTaskStatus.COMPLETED.value, VideoUpscaleTaskStatus.FAILED.value}:
return None
if not owner_is_generating(task):
# 对于 API v3 任务(有 api_key_id 属性),即使所有者已完成也允许超分继续
if not hasattr(task, "api_key_id") and not owner_is_generating(task):
return None
lease_until = _aware(upscale.lease_until)
if lease_until and lease_until > _now() and upscale.status == VideoUpscaleTaskStatus.PROCESSING.value:
@@ -0,0 +1,13 @@
from app.services.virtual_portrait_v3 import (
quota_service,
project_service,
asset_service,
upload_service,
)
__all__ = [
"quota_service",
"project_service",
"asset_service",
"upload_service",
]
@@ -0,0 +1,674 @@
from __future__ import annotations
import json
import logging
import re
from datetime import datetime, timedelta, timezone
from typing import Awaitable, Callable
from fastapi import HTTPException
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.private_portrait import (
PrivatePortraitAssetStatus,
PrivatePortraitAssetType,
PrivatePortraitEventSource,
PrivatePortraitEventStatus,
PrivatePortraitEventType,
PrivatePortraitProjectStatus,
PrivatePortraitRemoteDeleteStatus,
)
from app.models.virtual_portrait_v3 import VpV3Asset, VpV3Project
from app.schemas.virtual_portrait_v3.asset import (
VpV3AssetCreate,
VpV3AssetListOut,
VpV3AssetOut,
VpV3SelectableAssetListOut,
VpV3SelectableAssetOut,
)
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.ark_client import (
ArkPrivateAssetClient,
ArkPrivateAssetClientError,
)
from app.services.virtual_portrait_v3.project_service import (
refresh_project_counters,
)
from app.services.virtual_portrait_v3.quota_service import (
_bytes_to_mb,
_refresh_quota_used,
check_asset_quota,
get_quota,
remote_project_name,
)
from app.services.virtual_portrait_v3.upload_service import (
delete_local_file_by_url,
download_url_to_local,
)
from app.utils.id_gen import generate_id
logger = logging.getLogger("videogen")
DOMAIN = "virtual_portrait_v3"
URL_RE_REMOTE_URL_EXPR = re.compile(r"^https?://", re.IGNORECASE)
URL_LOCAL_UPLOAD_EXPR = re.compile(r"^/uploads/|^https?://[^/]+/uploads/", re.IGNORECASE)
# 北京时间(UTC+8)统一基准:V3 业务所有时间写入 / 时间比较唯一参考
_BJ_TZ = timezone(timedelta(hours=8))
def _bj_now() -> datetime:
"""返回当前北京时间(UTC+8naive datetime(去掉 tzinfo,与 DB naive 存储一致)。"""
return datetime.now(_BJ_TZ).replace(tzinfo=None)
def _json(data) -> str | None:
if data is None:
return None
return json.dumps(data, ensure_ascii=False, default=str)
def asset_to_out(a: VpV3Asset) -> VpV3AssetOut:
return VpV3AssetOut(
asset_id=a.id,
project_id=a.project_id,
name=a.name,
asset_type=a.asset_type,
status=a.status,
source_url=a.source_url,
preview_url=a.preview_url,
remote_url=a.remote_url,
remote_url_expired_at=a.remote_url_expired_at,
video_duration=a.video_duration,
video_cover_url=a.video_cover_url,
file_size_bytes=a.file_size_bytes,
mime_type=a.mime_type,
moderation_json=a.moderation_json,
error_message=a.error_message,
remote_delete_status=a.remote_delete_status or PrivatePortraitRemoteDeleteStatus.NONE.value,
created_at=a.created_at,
updated_at=a.updated_at,
)
def asset_to_selectable(a: VpV3Asset) -> VpV3SelectableAssetOut:
return VpV3SelectableAssetOut(
asset_id=a.id,
project_id=a.project_id,
name=a.name,
asset_type=a.asset_type,
status=a.status,
source_url=a.source_url,
preview_url=a.preview_url or a.remote_url or a.source_url,
video_duration=a.video_duration,
video_cover_url=a.video_cover_url,
file_size_bytes=a.file_size_bytes,
created_at=a.created_at,
)
def _validate_source_url(url: str, asset_type: str) -> None:
"""创建素材时的 source_url 现在只允许 http(s) 的外部 URL。
旧的 /uploads/* 本地 URL 已不再推荐(直接让系统自己下载保存)。"""
if not url or not url.strip():
raise HTTPException(status_code=400, detail="source_url 不能为空")
stripped = url.strip()
if not stripped.lower().startswith("http://") and not stripped.lower().startswith("https://"):
raise HTTPException(
status_code=400,
detail="source_url 必须是公网可访问的 http(s) URL;本服务会自动下载并保存到本地",
)
if len(stripped) > 2000:
raise HTTPException(status_code=400, detail="source_url 过长(最多 2000 字符)")
# ---------------------------------------------------------------------------
# Asset CRUD
# ---------------------------------------------------------------------------
async def create_asset(
db: AsyncSession,
*,
api_key_id: str,
project: VpV3Project,
payload: VpV3AssetCreate,
) -> VpV3Asset:
"""在项目下创建素材:
**新流程(一步到位)**
1. project 状态校验
2. source_url 格式校验
3. 将 source_url 下载保存到本地 vp_v3 上传目录(占用磁盘,校验 MIME/大小/网络)
- 失败:抛 HTTPException400/413/415/502/500),不留临时文件
4. 配额校验(素材数 + 存储 MB,用下载后的实际 file_size_bytes
- 失败:**立刻删除本地已下载的文件**,避免占用磁盘;再抛 403
5. Video 时长校验(payload.video_duration 优先,否则用 ffprobe 探测到的值;>60s 报错)
- 失败:删本地文件 → 抛 400
6. 写 VpV3AssetCreating 状态,带 next_poll_at
- 失败:删本地文件 → 抛 500
7. 调 Ark CreateAsset(url=本地公网 URL),异步审核
- 异常:status 置为 FAILED,保留本地文件(因为已占配额和素材数,走删除接口会清理)
8. 刷新项目计数 + 配额 used,返回素材
"""
# 1. project 状态校验
if project.status != PrivatePortraitProjectStatus.ACTIVE.value:
raise HTTPException(status_code=400, detail=f"项目状态 {project.status} 不可创建素材,仅 active 项目可操作")
# 2. source_url 校验(只允许公网 http(s)
_validate_source_url(payload.source_url, payload.asset_type)
downloaded: "DownloadedAsset | None" = None
try:
# 3. URL → 本地下载保存(此处负责 URL 合法性/网络/MIME/大小的校验及抛错)
downloaded = await download_url_to_local(
api_key_id=api_key_id,
asset_type=payload.asset_type,
source_url=payload.source_url,
requested_filename=payload.name,
)
file_size_bytes = downloaded.file_size_bytes
# 4. 配额校验(素材数 + 存储),这里已经拿到真实 file_size_bytes
try:
await check_asset_quota(
db,
api_key_id=api_key_id,
asset_count_delta=1,
file_size_bytes=file_size_bytes,
)
except HTTPException:
# 配额不足 → 立刻清理刚下载好的本地文件,再抛
_safe_delete_local_file(downloaded.url)
raise
# 5. Video 时长:优先用 payload.video_duration,否则用探测值
effective_video_duration: float | None = None
if payload.asset_type == PrivatePortraitAssetType.VIDEO.value:
if payload.video_duration is not None and payload.video_duration > 0:
effective_video_duration = float(payload.video_duration)
elif downloaded.duration_seconds is not None and downloaded.duration_seconds > 0:
effective_video_duration = float(downloaded.duration_seconds)
else:
_safe_delete_local_file(downloaded.url)
raise HTTPException(
status_code=400,
detail="Video 素材无法获取时长:请显式传 video_duration(秒),或确保 URL 指向合法的视频文件",
)
if effective_video_duration > 60:
_safe_delete_local_file(downloaded.url)
raise HTTPException(status_code=400, detail="视频素材时长不能超过 60 秒")
# 素材展示名:payload.name → downloaded.suggested_name → filename 去扩展名
final_name: str | None = (payload.name or "").strip()[:128] or None
if not final_name and downloaded.suggested_name:
final_name = (downloaded.suggested_name or "").strip()[:128] or None
asset = VpV3Asset(
id=generate_id(),
api_key_id=api_key_id,
project_id=project.id,
remote_project_name=project.remote_project_name,
remote_group_id=project.remote_group_id,
remote_asset_id=None,
asset_type=payload.asset_type,
name=final_name,
source_url=payload.source_url, # 本地存储后的 URL
preview_url=downloaded.url, # 初始 preview = 本地 URL
remote_url=None,
remote_url_expired_at=None,
upload_resource_id=None, # 不再使用(旧接口兼容保留字段)
video_duration=effective_video_duration,
video_cover_url=payload.video_cover_url,
file_size_bytes=file_size_bytes,
mime_type=downloaded.mime_type,
status=PrivatePortraitAssetStatus.CREATING.value,
poll_count=0,
next_poll_at=_bj_now() + timedelta(seconds=2),
)
db.add(asset)
await db.flush()
await db.refresh(asset)
except HTTPException:
raise
except Exception as exc: # noqa: BLE001
# 任何 DB 写入前的异常 → 能清理就清理本地文件
if downloaded:
_safe_delete_local_file(downloaded.url)
logger.exception("vp_v3 创建素材(下载/写库阶段)异常:%s", exc)
raise HTTPException(status_code=500, detail=f"创建素材失败:{exc}") from exc
# 6. 调 Ark CreateAsset(到这里 DB 已经 flush 成功了)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_CREATE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.API.value,
api_key_id=api_key_id,
project_id=project.id,
asset_id=asset.id,
detail={
"remote_project_name": asset.remote_project_name,
"remote_group_id": asset.remote_group_id,
"source_url": asset.source_url,
"asset_type": asset.asset_type,
"original_source_url": payload.source_url.strip()[:500],
},
)
try:
resp = await ArkPrivateAssetClient().create_asset(
project_name=asset.remote_project_name,
group_id=asset.remote_group_id,
url=asset.source_url,
asset_type=asset.asset_type,
name=asset.name,
)
remote_asset_id = resp.get("Id") or resp.get("AssetId") or resp.get("assetId") or resp.get("id")
if not remote_asset_id:
raise RuntimeError("CreateAsset 未返回素材 Id")
asset.remote_asset_id = str(remote_asset_id)
asset.raw_response_json = _json(resp)
asset.remote_url = resp.get("URL") or resp.get("url") or resp.get("Url") or asset.remote_url
if asset.remote_url:
asset.preview_url = asset.remote_url
asset.next_poll_at = _bj_now() + timedelta(seconds=3)
asset.status = PrivatePortraitAssetStatus.CREATING.value
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_CREATE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.API.value,
api_key_id=api_key_id,
project_id=project.id,
asset_id=asset.id,
detail={"remote_asset_id": remote_asset_id},
)
await refresh_project_counters(db, [project.id])
_ = await get_quota(db, api_key_id=api_key_id, refresh=True)
return asset
except Exception as exc: # noqa: BLE001
# 火山调用失败 → 保留本地文件(DB 已写好,走删除接口清理),状态 FAILED,带错误
asset.status = PrivatePortraitAssetStatus.FAILED.value
asset.error_message = str(exc)
asset.raw_response_json = _json({"error": str(exc)})
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_CREATE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.API.value,
api_key_id=api_key_id,
project_id=project.id,
asset_id=asset.id,
exc=exc,
)
raise HTTPException(status_code=502, detail=f"提交火山素材创建失败:{exc}") from exc
def _safe_delete_local_file(local_url: str | None) -> None:
if not local_url:
return
try:
delete_local_file_by_url(local_url)
except Exception: # noqa: BLE001
logger.warning("vp_v3 清理本地文件失败(不抛):%s", local_url)
async def list_assets(
db: AsyncSession,
*,
api_key_id: str,
project_id: str | None = None,
status: str | None = None,
keyword: str | None = None,
asset_type: str | None = None,
page: int,
page_size: int,
) -> tuple[list[VpV3Asset], int]:
"""分页查询素材列表。"""
conds = [VpV3Asset.api_key_id == api_key_id, VpV3Asset.deleted_at.is_(None)]
if project_id:
conds.append(VpV3Asset.remote_group_id == project_id)
if status:
conds.append(VpV3Asset.status == status)
if keyword:
conds.append((VpV3Asset.name.is_not(None)) & (VpV3Asset.name.ilike(f"%{keyword}%")))
if asset_type:
conds.append(VpV3Asset.asset_type == asset_type)
count_result = await db.execute(select(func.count(VpV3Asset.id)).where(*conds))
total = int(count_result.scalar() or 0)
q = (
select(VpV3Asset)
.where(*conds)
.order_by(VpV3Asset.created_at.desc())
.limit(page_size)
.offset((page - 1) * page_size)
)
items = list((await db.execute(q)).scalars().all())
return items, total
async def list_selectable_assets(
db: AsyncSession,
*,
api_key_id: str,
project_id: str | None = None,
keyword: str | None = None,
asset_type: str | None = None,
page: int,
page_size: int,
) -> tuple[list[VpV3Asset], int]:
"""AI 创作选择器素材列表:只返回 status=Active 的。"""
items, total = await list_assets(
db,
api_key_id=api_key_id,
project_id=project_id,
status=PrivatePortraitAssetStatus.ACTIVE.value,
keyword=keyword,
asset_type=asset_type,
page=page,
page_size=page_size,
)
return items, total
async def get_asset(db: AsyncSession, *, api_key_id: str, asset_id: str) -> VpV3Asset:
"""素材详情(权限校验)。"""
row = (await db.execute(
select(VpV3Asset).where(
VpV3Asset.remote_asset_id == asset_id,
VpV3Asset.api_key_id == api_key_id,
VpV3Asset.deleted_at.is_(None),
).limit(1)
)).scalar_one_or_none()
if not row:
raise HTTPException(status_code=404, detail="虚拟素材不存在")
return row
async def sync_asset_status(
db: AsyncSession,
*,
api_key_id: str,
asset_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> VpV3Asset:
"""主动同步素材状态(调 Ark GetAsset)。
注意:如果素材没有 remote_asset_id(远端还未 CreateAsset 返回),直接跳过并返回当前本地快照。
"""
asset = await get_asset(db, api_key_id=api_key_id, asset_id=asset_id)
if not asset.remote_asset_id:
return asset
try:
resp = await ArkPrivateAssetClient().get_asset(
project_name=asset.remote_project_name, asset_id=asset.remote_asset_id,
)
if execution_guard is not None:
await execution_guard()
_apply_get_asset_response(asset, resp)
except Exception as exc: # noqa: BLE001
# 异常分支也必须推进 poll 计数 + 重算下次轮询时间,避免无限调度且数据库无变化
asset.last_poll_at = _bj_now()
asset.poll_count = int(asset.poll_count or 0) + 1
asset.error_message = f"同步状态失败:{exc}"
logger.warning("vp_v3 同步素材状态失败:asset_id=%s err=%s", asset_id, exc)
# 异常情况仍然保持 CREATING,按指数退避重算 next_poll_at
delays = [3, 6, 12, 30, 60]
idx = min(asset.poll_count, len(delays) - 1)
asset.next_poll_at = _bj_now() + timedelta(seconds=delays[idx])
finally:
await db.flush()
await refresh_project_counters(db, [asset.project_id])
return asset
def _apply_get_asset_response(a: VpV3Asset, resp: dict) -> None:
"""把 Ark GetAsset 响应应用到本地记录(状态、URL、审核信息)。"""
# 先推进公共轮询字段(无论状态映射结果如何,只要调了一次 GetAsset 都必须记录)
a.last_poll_at = _bj_now()
a.poll_count = int(a.poll_count or 0) + 1
a.moderation_json = _json(resp)
a.raw_response_json = _json(resp)
# Status 映射:火山 Status 字段 → 本地枚举
status_raw = str(resp.get("Status") or resp.get("status") or "").lower()
if status_raw in {"active", "success", "done", "available"}:
a.status = PrivatePortraitAssetStatus.ACTIVE.value
elif status_raw in {"creating", "pending", "processing", "auditing"}:
a.status = PrivatePortraitAssetStatus.CREATING.value
elif status_raw in {"failed", "error", "rejected", "invalid"}:
a.status = PrivatePortraitAssetStatus.FAILED.value
msg = resp.get("Message") or resp.get("message") or resp.get("Error") or resp.get("error")
if msg:
a.error_message = str(msg)
else:
# 未知状态保持原
pass
# URL 续期
url = resp.get("URL") or resp.get("url") or resp.get("Url")
if url:
a.remote_url = url
a.preview_url = url
a.remote_url_expired_at = None # 无法解析过期时间就不填
# 视频时长
if not a.video_duration:
dur = resp.get("Duration") or resp.get("duration")
if dur is not None:
try:
a.video_duration = float(dur)
except Exception: # noqa: BLE001
pass
# 大小
if not a.file_size_bytes:
size = resp.get("FileSize") or resp.get("fileSize") or resp.get("size")
if size is not None:
try:
a.file_size_bytes = int(size)
except Exception: # noqa: BLE001
pass
# 状态判断下次轮询时间
if a.status == PrivatePortraitAssetStatus.CREATING.value:
# 指数退避:3s → 6s → 12s → 30s → 60s,最多 60s
delays = [3, 6, 12, 30, 60]
idx = min(a.poll_count, len(delays) - 1)
a.next_poll_at = _bj_now() + timedelta(seconds=delays[idx])
elif a.status == PrivatePortraitAssetStatus.FAILED.value:
a.next_poll_at = None # 失败不再轮询
elif a.status == PrivatePortraitAssetStatus.ACTIVE.value:
a.next_poll_at = None # 成功不再轮询
async def soft_delete_asset(db: AsyncSession, *, api_key_id: str, asset_id: str) -> VpV3Asset:
"""软删素材(本地先标记为删除中,同步删除本地落盘文件,重新计算项目计数和配额 used,然后 commit 后再投递异步远端删除任务)。"""
asset = await get_asset(db, api_key_id=api_key_id, asset_id=asset_id)
pid = asset.project_id
now = _bj_now()
asset.deleted_at = now
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.PENDING.value
asset.status = PrivatePortraitAssetStatus.DELETING.value
# 本地落盘文件:立刻删(成功失败都不影响状态,避免占磁盘;失败仅 log)
if asset.source_url:
_safe_delete_local_file(asset.source_url)
await db.flush()
await refresh_project_counters(db, [pid])
q = await get_quota(db, api_key_id=api_key_id, refresh=True)
return asset
# V3 专属的远端删除服务
V3_DOMAIN = "virtual_portrait_v3"
async def _load_v3_asset_delete_snapshot(db: AsyncSession, *, asset_id: str) -> dict | None:
"""加载 V3 素材删除快照。"""
asset = (
await db.execute(
select(VpV3Asset).where(VpV3Asset.remote_asset_id == asset_id).limit(1)
)
).scalar_one_or_none()
if not asset:
return None
return {
"owner_id": str(asset.id),
"owner_type": "asset",
"api_key_id": str(asset.api_key_id),
"project_id": str(asset.project_id),
"remote_id": str(asset.remote_asset_id) if asset.remote_asset_id else None,
"remote_project_name": str(asset.remote_project_name or ""),
"asset_type": str(asset.asset_type or ""),
"remote_delete_status": str(asset.remote_delete_status or ""),
}
async def _apply_v3_asset_delete_result(
db: AsyncSession,
*,
asset_id: str,
remote_id: str | None,
succeeded: bool,
skipped: bool = False,
error: BaseException | None = None,
) -> None:
"""应用 V3 素材远端删除结果到数据库。"""
asset = (
await db.execute(
select(VpV3Asset).where(VpV3Asset.id == asset_id).with_for_update().limit(1)
)
).scalar_one_or_none()
if not asset:
return
if asset.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if remote_id and str(asset.remote_asset_id or "") != remote_id:
raise RuntimeError("V3 素材远程 Asset 已变化,旧删除结果已丢弃")
now = _bj_now()
if skipped:
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
asset.remote_delete_error = None
elif succeeded:
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
asset.remote_deleted_at = now
asset.remote_delete_error = None
else:
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
asset.remote_delete_error = str(error or "远程删除失败")
await db.flush()
async def delete_v3_asset_remote(
db: AsyncSession,
*,
asset_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> None:
"""V3 素材远端删除(异步 Celery 任务调用)。"""
snapshot = await _load_v3_asset_delete_snapshot(db, asset_id=asset_id)
if snapshot is None:
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
asset_id=asset_id,
message="远程删除跳过:本地素材不存在",
)
await db.rollback()
return
if snapshot["remote_delete_status"] in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
await db.rollback()
return
remote_id = snapshot["remote_id"]
if not remote_id:
await _apply_v3_asset_delete_result(
db,
asset_id=asset_id,
remote_id=None,
succeeded=False,
skipped=True,
)
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=snapshot["project_id"],
asset_id=asset_id,
message="远程删除跳过:素材没有 remote_asset_id",
)
return
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=snapshot["project_id"],
asset_id=asset_id,
detail={
"remote_asset_id": remote_id,
"remote_project_name": snapshot["remote_project_name"],
"asset_type": snapshot["asset_type"],
},
)
await db.rollback()
remote_error: BaseException | None = None
succeeded = False
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset(
project_name=snapshot["remote_project_name"],
asset_id=remote_id,
)
succeeded = True
except Exception as exc:
remote_error = exc
# 404 视为幂等成功
if "not found" in str(exc).lower() or "404" in str(exc):
succeeded = True
if execution_guard is not None:
await execution_guard()
await _apply_v3_asset_delete_result(
db,
asset_id=asset_id,
remote_id=remote_id,
succeeded=succeeded,
error=remote_error,
)
if succeeded:
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=snapshot["project_id"],
asset_id=asset_id,
message="远程资源不存在,按幂等删除成功处理" if remote_error is not None else None,
detail={
"remote_asset_id": remote_id,
"remote_project_name": snapshot["remote_project_name"],
},
)
else:
assert remote_error is not None
log_operation_error(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=snapshot["project_id"],
asset_id=asset_id,
exc=remote_error,
)
@@ -0,0 +1,184 @@
"""VP V3 虚拟素材库专用日志服务。
统一记录所有 VP V3 相关操作日志到 logs/virtual_portrait_v3/ 目录。
按天分文件,便于管理和排查问题。
"""
import json
import logging
import os
from datetime import datetime, timezone
from app.config import settings
# === 日志目录 ===
BASE_LOG_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))),
"log", "virtual_portrait_v3",
)
os.makedirs(BASE_LOG_DIR, exist_ok=True)
class _DailyFileHandler(logging.Handler):
"""按天写入不同日志文件的处理器。"""
def __init__(self, log_dir: str):
super().__init__()
self.log_dir = log_dir
self._current_date = None
self._file_handler = None
self._open_file()
def _open_file(self):
"""打开当天的日志文件。"""
now = datetime.now(timezone.utc)
date_str = now.strftime("%Y-%m-%d")
if date_str == self._current_date and self._file_handler:
return
if self._file_handler:
self._file_handler.close()
self._current_date = date_str
filepath = os.path.join(self.log_dir, f"{date_str}.log")
self._file_handler = open(filepath, "a", encoding="utf-8")
def emit(self, record):
try:
self._open_file()
msg = self.format(record)
self._file_handler.write(msg + "\n")
self._file_handler.flush()
except Exception:
self.handleError(record)
def close(self):
if self._file_handler:
self._file_handler.close()
super().close()
def _create_logger(name: str, filename: str | None = None) -> logging.Logger:
"""创建专用 Logger。"""
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
# 避免重复添加 handler
if logger.handlers:
return logger
# 按天写入文件
handler = _DailyFileHandler(BASE_LOG_DIR)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(asctime)s | %(levelname)s | %(name)s | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
handler.setFormatter(formatter)
logger.addHandler(handler)
# 不向上传播到 root logger(避免重复输出到控制台)
logger.propagate = False
return logger
# === 专用 Logger 实例 ===
asset_logger = _create_logger("vp_v3.asset")
project_logger = _create_logger("vp_v3.project")
quota_logger = _create_logger("vp_v3.quota")
api_logger = _create_logger("vp_v3.api")
def log_asset_event(
event_type: str,
api_key_id: str,
asset_id: str | None = None,
project_id: str | None = None,
status: str | None = None,
detail: dict | None = None,
error: str | None = None,
):
"""记录素材相关事件。"""
log_data = {
"event_type": event_type,
"api_key_id": api_key_id,
"asset_id": asset_id,
"project_id": project_id,
"status": status,
"detail": detail or {},
}
if error:
log_data["error"] = error
asset_logger.error(json.dumps(log_data, ensure_ascii=False, default=str))
else:
asset_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
def log_project_event(
event_type: str,
api_key_id: str,
project_id: str | None = None,
status: str | None = None,
detail: dict | None = None,
error: str | None = None,
):
"""记录项目相关事件。"""
log_data = {
"event_type": event_type,
"api_key_id": api_key_id,
"project_id": project_id,
"status": status,
"detail": detail or {},
}
if error:
log_data["error"] = error
project_logger.error(json.dumps(log_data, ensure_ascii=False, default=str))
else:
project_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
def log_quota_event(
event_type: str,
api_key_id: str,
quota_type: str,
amount: float,
quota_before: float | None = None,
quota_after: float | None = None,
detail: dict | None = None,
):
"""记录配额相关事件。"""
log_data = {
"event_type": event_type,
"api_key_id": api_key_id,
"quota_type": quota_type,
"amount": amount,
"quota_before": quota_before,
"quota_after": quota_after,
"detail": detail or {},
}
quota_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
def log_api_request(
method: str,
path: str,
api_key_id: str,
status_code: int,
duration_ms: int,
error: str | None = None,
):
"""记录 API 请求。"""
log_data = {
"method": method,
"path": path,
"api_key_id": api_key_id,
"status_code": status_code,
"duration_ms": duration_ms,
}
if error:
log_data["error"] = error
api_logger.error(json.dumps(log_data, ensure_ascii=False, default=str))
else:
api_logger.info(json.dumps(log_data, ensure_ascii=False, default=str))
@@ -0,0 +1,557 @@
from __future__ import annotations
import json
import logging
from datetime import datetime, timedelta, timezone
from typing import Awaitable, Callable
from fastapi import HTTPException
from sqlalchemy import case, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.private_portrait import (
PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
PrivatePortraitAssetStatus,
PrivatePortraitAssetType,
PrivatePortraitEventSource,
PrivatePortraitEventStatus,
PrivatePortraitEventType,
PrivatePortraitProjectStatus,
PrivatePortraitRemoteDeleteStatus,
)
from app.models.virtual_portrait_v3 import VpV3Asset, VpV3Project
from app.schemas.virtual_portrait_v3.project import (
VpV3ProjectCreate,
VpV3ProjectListOut,
VpV3ProjectOut,
VpV3ProjectUpdate,
)
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.ark_client import ArkPrivateAssetClient
from app.services.virtual_portrait_v3.quota_service import (
_bytes_to_mb,
_refresh_quota_used,
_slug,
check_project_quota,
get_quota,
remote_group_name,
remote_project_name,
)
from app.utils.id_gen import generate_id
logger = logging.getLogger("videogen")
DOMAIN = "virtual_portrait_v3"
# 北京时间(UTC+8)统一基准:V3 业务所有时间写入 / 时间比较唯一参考
_BJ_TZ = timezone(timedelta(hours=8))
def _bj_now() -> datetime:
"""返回当前北京时间(UTC+8naive datetime(去掉 tzinfo,与 DB naive 存储一致)。"""
return datetime.now(_BJ_TZ).replace(tzinfo=None)
def _json(data) -> str | None:
if data is None:
return None
return json.dumps(data, ensure_ascii=False, default=str)
def project_to_out(p: VpV3Project) -> VpV3ProjectOut:
return VpV3ProjectOut(
project_id=p.remote_group_id,
name=p.name,
description=p.description,
status=p.status,
asset_count=int(p.asset_count or 0),
active_asset_count=int(p.active_asset_count or 0),
image_asset_count=int(p.image_asset_count or 0),
video_asset_count=int(p.video_asset_count or 0),
storage_mb_used=float(p.storage_mb_used or 0),
remote_delete_status=p.remote_delete_status or PrivatePortraitRemoteDeleteStatus.NONE.value,
error_message=p.error_message,
created_at=p.created_at,
updated_at=p.updated_at,
)
# ---------------------------------------------------------------------------
# Project CRUD
# ---------------------------------------------------------------------------
async def create_project(
db: AsyncSession,
*,
api_key_id: str,
payload: VpV3ProjectCreate,
) -> VpV3Project:
"""创建虚拟素材项目(同步调用 Ark CreateAssetGroup)。
1. 配额校验
2. 本地落库 status=creating_remote_group
3. 调 Ark CreateAssetGroup 拿 remote_group_id
4. 本地更新为 active,返回
"""
await check_project_quota(db, api_key_id=api_key_id, delta=1)
# slug = _slug(payload.name)
proj = VpV3Project(
id=generate_id(),
api_key_id=api_key_id,
name=payload.name.strip()[:128],
name_slug=payload.name.strip()[:128],
description=payload.description,
remote_project_name=remote_project_name(),
remote_group_id="",
status=PrivatePortraitProjectStatus.CREATING_REMOTE_GROUP.value,
asset_count=0,
active_asset_count=0,
image_asset_count=0,
video_asset_count=0,
storage_mb_used=0,
)
db.add(proj)
await db.flush()
await db.refresh(proj)
group_name = remote_group_name(api_key_id=api_key_id, project_slug=proj.name_slug,id=proj.id)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.API.value,
api_key_id=api_key_id,
project_id=proj.id,
detail={"remote_group_name": group_name, "remote_project_name": proj.remote_project_name},
)
try:
resp = await ArkPrivateAssetClient().create_asset_group(
project_name=proj.remote_project_name,
name=group_name,
description=payload.description,
group_type=PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
)
remote_group_id = resp.get("Id") or resp.get("GroupId") or resp.get("groupId")
if not remote_group_id:
raise RuntimeError("CreateAssetGroup 未返回素材组 ID")
proj.remote_group_id = str(remote_group_id)
proj.remote_group_name = group_name
proj.status = PrivatePortraitProjectStatus.ACTIVE.value
proj.raw_response_json = _json(resp)
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.API.value,
api_key_id=api_key_id,
project_id=proj.id,
detail={"remote_group_id": remote_group_id, "group_name": group_name},
)
return proj
except Exception as exc: # noqa: BLE001
proj.status = PrivatePortraitProjectStatus.CREATE_GROUP_FAILED.value
proj.error_message = str(exc)
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.VIRTUAL_ASSET_GROUP_CREATE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.API.value,
api_key_id=api_key_id,
project_id=proj.id,
exc=exc,
)
raise HTTPException(status_code=502, detail=f"创建虚拟素材项目失败:{exc}") from exc
async def list_projects(
db: AsyncSession,
*,
api_key_id: str,
page: int,
page_size: int,
keyword: str | None = None,
status: str | None = None,
) -> tuple[list[VpV3Project], int]:
"""按 API Key 分页查询项目列表。"""
conds = [VpV3Project.api_key_id == api_key_id, VpV3Project.deleted_at.is_(None)]
if keyword:
conds.append(VpV3Project.name.ilike(f"%{keyword}%"))
if status:
conds.append(VpV3Project.status == status)
count_result = await db.execute(
select(func.count(VpV3Project.id)).where(*conds)
)
total = int(count_result.scalar() or 0)
q = (
select(VpV3Project)
.where(*conds)
.order_by(VpV3Project.created_at.desc())
.limit(page_size)
.offset((page - 1) * page_size)
)
items = list((await db.execute(q)).scalars().all())
return items, total
async def get_project(db: AsyncSession, *, api_key_id: str, project_id: str) -> VpV3Project:
"""获取项目详情(权限校验)。"""
row = (await db.execute(
select(VpV3Project).where(
VpV3Project.remote_group_id == project_id,
VpV3Project.api_key_id == api_key_id,
VpV3Project.deleted_at.is_(None),
).limit(1)
)).scalar_one_or_none()
if not row:
raise HTTPException(status_code=404, detail="虚拟素材项目不存在")
return row
async def update_project(
db: AsyncSession,
*,
api_key_id: str,
project_id: str,
payload: VpV3ProjectUpdate,
) -> VpV3Project:
"""更新项目展示信息(名称/描述,不会重新创建远端 Group)。"""
proj = await get_project(db, api_key_id=api_key_id, project_id=project_id)
changed = False
if payload.name is not None and payload.name != proj.name:
proj.name = payload.name.strip()[:128]
proj.name_slug = _slug(payload.name)
changed = True
if payload.description is not None and payload.description != proj.description:
proj.description = payload.description
changed = True
if changed:
await db.flush()
return proj
async def soft_delete_project(
db: AsyncSession,
*,
api_key_id: str,
project_id: str,
) -> VpV3Project:
"""软删项目和其下所有素材(本地先删,等 commit 后再投递异步远端删除任务)。
会把 quota used 重新刷新一次。
"""
proj = await get_project(db, api_key_id=api_key_id, project_id=project_id)
now = _bj_now()
proj.deleted_at = now
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.PENDING.value
proj.status = PrivatePortraitProjectStatus.DELETING.value
# 级联软删其下所有素材
await db.execute(
VpV3Asset.__table__.update() # type: ignore[attr-defined]
.where(
VpV3Asset.project_id == proj.id,
VpV3Asset.deleted_at.is_(None),
)
.values(
deleted_at=now,
remote_delete_status=PrivatePortraitRemoteDeleteStatus.PENDING.value,
)
)
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
return proj
# ---------------------------------------------------------------------------
# 项目计数刷新(增删素材后调用,用于项目列表快速显示)
# ---------------------------------------------------------------------------
async def refresh_project_counters(db: AsyncSession, project_ids: list[str]) -> None:
"""按真实数据刷新项目 asset 计数和 storage。"""
if not project_ids:
return
for pid in project_ids:
row = (await db.execute(
select(
func.count(VpV3Asset.id),
func.sum(case((VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value, 1), else_=0)),
func.sum(case((VpV3Asset.asset_type == PrivatePortraitAssetType.IMAGE.value, 1), else_=0)),
func.sum(case((VpV3Asset.asset_type == PrivatePortraitAssetType.VIDEO.value, 1), else_=0)),
func.sum(case(
(VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value,
case((VpV3Asset.asset_type == PrivatePortraitAssetType.IMAGE.value, 1), else_=0)),
else_=0
)),
func.sum(case(
(VpV3Asset.status == PrivatePortraitAssetStatus.ACTIVE.value,
case((VpV3Asset.asset_type == PrivatePortraitAssetType.VIDEO.value, 1), else_=0)),
else_=0
)),
func.coalesce(func.sum(VpV3Asset.file_size_bytes), 0),
).where(
VpV3Asset.project_id == pid,
VpV3Asset.deleted_at.is_(None),
)
)).one()
(total, active, img_cnt, vid_cnt, active_img, active_vid, storage_bytes) = row
proj = (await db.execute(
select(VpV3Project).where(VpV3Project.id == pid).limit(1)
)).scalar_one_or_none()
if proj:
proj.asset_count = int(total or 0)
proj.active_asset_count = int(active or 0)
proj.image_asset_count = int(img_cnt or 0)
proj.video_asset_count = int(vid_cnt or 0)
proj.active_image_asset_count = int(active_img or 0)
proj.active_video_asset_count = int(active_vid or 0)
proj.storage_mb_used = float(_bytes_to_mb(storage_bytes))
# V3 专属的项目远端删除服务
V3_DOMAIN = "virtual_portrait_v3"
async def _load_v3_project_delete_snapshot(db: AsyncSession, *, project_id: str) -> dict | None:
"""加载 V3 项目删除快照。"""
proj = (
await db.execute(
select(VpV3Project).where(VpV3Project.id == project_id).limit(1)
)
).scalar_one_or_none()
if not proj:
return None
return {
"owner_id": str(proj.id),
"owner_type": "project",
"api_key_id": str(proj.api_key_id),
"remote_id": str(proj.remote_group_id) if proj.remote_group_id else None,
"remote_project_name": str(proj.remote_project_name or ""),
"remote_delete_status": str(proj.remote_delete_status or ""),
}
async def _apply_v3_project_delete_result(
db: AsyncSession,
*,
project_id: str,
remote_id: str | None,
succeeded: bool,
skipped: bool = False,
error: BaseException | None = None,
) -> None:
"""应用 V3 项目远端删除结果到数据库。"""
proj = (
await db.execute(
select(VpV3Project)
.where(VpV3Project.id == project_id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not proj:
return
if proj.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if remote_id and str(proj.remote_group_id or "") != remote_id:
raise RuntimeError("V3 项目远程 Group 已变化,旧删除结果已丢弃")
now = _bj_now()
if skipped:
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
proj.remote_delete_error = None
elif succeeded:
proj.status = PrivatePortraitProjectStatus.DELETED.value
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
proj.remote_deleted_at = now
proj.remote_delete_error = None
else:
proj.status = PrivatePortraitProjectStatus.DELETED.value
proj.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
proj.remote_delete_error = str(error or "远程删除失败")
await db.flush()
# 刷新配额
quota = await get_quota(db, api_key_id=proj.api_key_id, refresh=False)
await _refresh_quota_used(db, quota)
async def delete_v3_project_remote(
db: AsyncSession,
*,
project_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> None:
"""V3 项目远端删除(异步 Celery 任务调用)。
会先级联删除项目下所有素材的远端资源,再删除项目的远端 Group。
"""
# 先删除项目下所有素材的远端资源
assets = (
await db.execute(
select(VpV3Asset).where(
VpV3Asset.project_id == project_id,
VpV3Asset.deleted_at.is_not(None),
VpV3Asset.remote_delete_status == PrivatePortraitRemoteDeleteStatus.PENDING.value,
)
)
).scalars().all()
for asset in assets:
if asset.remote_asset_id:
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset(
project_name=asset.remote_project_name,
asset_id=asset.remote_asset_id,
)
await _apply_v3_asset_delete_result_for_project(
db,
asset_id=asset.id,
succeeded=True,
)
except Exception as exc:
if "not found" in str(exc).lower() or "404" in str(exc):
await _apply_v3_asset_delete_result_for_project(
db,
asset_id=asset.id,
succeeded=True,
)
else:
await _apply_v3_asset_delete_result_for_project(
db,
asset_id=asset.id,
succeeded=False,
error=exc,
)
# 再删除项目的远端 Group
snapshot = await _load_v3_project_delete_snapshot(db, project_id=project_id)
if snapshot is None:
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=project_id,
message="远程删除跳过:本地项目不存在",
)
await db.rollback()
return
if snapshot["remote_delete_status"] in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
await db.rollback()
return
remote_id = snapshot["remote_id"]
if not remote_id:
await _apply_v3_project_delete_result(
db,
project_id=project_id,
remote_id=None,
succeeded=False,
skipped=True,
)
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=project_id,
message="远程删除跳过:项目没有 remote_group_id",
)
return
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=project_id,
detail={
"remote_group_id": remote_id,
"remote_project_name": snapshot["remote_project_name"],
},
)
await db.rollback()
remote_error: BaseException | None = None
succeeded = False
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset_group(
project_name=snapshot["remote_project_name"],
group_id=remote_id,
)
succeeded = True
except Exception as exc:
remote_error = exc
# 404 视为幂等成功
if "not found" in str(exc).lower() or "404" in str(exc):
succeeded = True
if execution_guard is not None:
await execution_guard()
await _apply_v3_project_delete_result(
db,
project_id=project_id,
remote_id=remote_id,
succeeded=succeeded,
error=remote_error,
)
if succeeded:
log_operation_event(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=project_id,
message="远程 Group 不存在,按幂等删除成功处理" if remote_error is not None else None,
detail={
"remote_group_id": remote_id,
"remote_project_name": snapshot["remote_project_name"],
},
)
else:
assert remote_error is not None
log_operation_error(
domain=V3_DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.CELERY.value,
project_id=project_id,
exc=remote_error,
)
async def _apply_v3_asset_delete_result_for_project(
db: AsyncSession,
*,
asset_id: str,
succeeded: bool,
error: BaseException | None = None,
) -> None:
"""项目删除时级联应用素材删除结果。"""
asset = (
await db.execute(
select(VpV3Asset).where(VpV3Asset.id == asset_id).with_for_update().limit(1)
)
).scalar_one_or_none()
if not asset:
return
if asset.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if succeeded:
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
asset.remote_deleted_at = _bj_now()
asset.remote_delete_error = None
else:
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
asset.remote_delete_error = str(error or "远程删除失败")
await db.flush()
@@ -0,0 +1,163 @@
from __future__ import annotations
import re
from fastapi import HTTPException
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.private_portrait import (
PRIVATE_PORTRAIT_REMOTE_PROJECT_NAME,
PRIVATE_PORTRAIT_VIRTUAL_GROUP_TYPE,
PrivatePortraitAssetType,
PrivatePortraitAssetStatus,
PrivatePortraitLibraryType,
PrivatePortraitProjectStatus,
)
from app.models.virtual_portrait_v3 import (
VpV3ApiKeyQuota,
VpV3Asset,
VpV3Project,
)
from app.utils.id_gen import generate_id
MB_BYTES = 1024 * 1024
_SAFE_SLUG = re.compile(r"[^a-zA-Z0-9_-]")
def _slug(name: str) -> str:
if not name:
return "unnamed"
return _SAFE_SLUG.sub("_", name.strip())[:80] or "unnamed"
def _bytes_to_mb(b: int | float | None) -> float:
if not b:
return 0.0
return round(b / MB_BYTES, 3)
# ---------------------------------------------------------------------------
# 配额读写(确保 VpV3ApiKeyQuota 记录存在)
# ---------------------------------------------------------------------------
async def _upsert_quota(db: AsyncSession, api_key_id: str) -> VpV3ApiKeyQuota:
"""获取配额记录;不存在则创建(默认全 0=不可用)。"""
from sqlalchemy.dialects.postgresql import insert
stmt = (
insert(VpV3ApiKeyQuota)
.values(
id=generate_id(),
api_key_id=api_key_id,
project_limit=0,
asset_limit=0,
storage_mb_limit=0,
project_used=0,
asset_used=0,
storage_mb_used=0,
)
.on_conflict_do_nothing(index_elements=["api_key_id"])
)
await db.execute(stmt)
row = (await db.execute(
select(VpV3ApiKeyQuota).where(VpV3ApiKeyQuota.api_key_id == api_key_id).limit(1)
)).scalar_one()
return row
async def _refresh_quota_used(db: AsyncSession, quota: VpV3ApiKeyQuota) -> None:
"""按真实数据重算已使用量(最终一致性)。"""
project_result = await db.execute(
select(func.count(VpV3Project.id)).where(
VpV3Project.api_key_id == quota.api_key_id,
VpV3Project.deleted_at.is_(None),
)
)
asset_result = await db.execute(
select(
func.count(VpV3Asset.id),
func.coalesce(func.sum(VpV3Asset.file_size_bytes), 0),
).where(
VpV3Asset.api_key_id == quota.api_key_id,
VpV3Asset.deleted_at.is_(None),
)
)
project_used = project_result.scalar() or 0
asset_row = asset_result.one()
asset_used = asset_row[0] or 0
storage_bytes = asset_row[1] or 0
quota.project_used = int(project_used)
quota.asset_used = int(asset_used)
quota.storage_mb_used = int(_bytes_to_mb(storage_bytes))
async def get_quota(db: AsyncSession, *, api_key_id: str, refresh: bool = True) -> VpV3ApiKeyQuota:
"""获取当前 API Key 的配额(含已使用量)。不存在则创建默认 0。"""
quota = await _upsert_quota(db, api_key_id)
if refresh:
await _refresh_quota_used(db, quota)
return quota
async def ensure_quota_enabled(db: AsyncSession, *, api_key_id: str) -> VpV3ApiKeyQuota:
"""校验是否已启用虚拟素材库功能,未启用直接 403。返回已刷新的配额。
判定口径(与后台设置保持一致):只要「项目数上限」或「素材数上限」任一 > 0 即视为启用;
存储上限已从配置中移除(不再作为启用条件,也不做硬性限制,仅保留数据库字段做统计展示)。
"""
quota = await get_quota(db, api_key_id=api_key_id, refresh=True)
if (quota.project_limit or 0) <= 0 and (quota.asset_limit or 0) <= 0:
raise HTTPException(status_code=403, detail="当前 API Key 未开启虚拟素材库功能,请联系管理员配置配额")
return quota
def _check(limit: int | None, used: int | float | None, delta: int | float, field: str) -> None:
"""通用配额上限校验。
约定:limit <= 0 视为该维度「未配置 / 不做限制」,此时直接跳过不报错;
只有 limit > 0 时才按「已用 + 本次 <= 上限」判断,避免影响已移除的维度(如存储上限)。
"""
if (limit or 0) <= 0:
return # 不限制,直接通过
if (used or 0) + delta > limit:
raise HTTPException(
status_code=403,
detail=f"虚拟素材库配额不足:{field} 上限 {limit},已使用 {used},本次需要 {delta},超出上限",
)
async def check_project_quota(db: AsyncSession, *, api_key_id: str, delta: int = 1) -> VpV3ApiKeyQuota:
"""创建项目前校验配额。"""
quota = await ensure_quota_enabled(db, api_key_id=api_key_id)
_check(quota.project_limit, quota.project_used, delta, "项目数")
return quota
async def check_asset_quota(
db: AsyncSession,
*,
api_key_id: str,
asset_count_delta: int = 1,
file_size_bytes: int | None = None,
) -> VpV3ApiKeyQuota:
"""上传素材前校验配额。
注:「存储空间上限」已从业务约束中移除(不再做硬性配额限制),仅保留素材数量上限
与项目数量上限的校验;storage_mb_used 字段仍会在 get_quota 中刷新用于统计展示。
"""
del file_size_bytes # 不再用于配额校验(仅保留形参兼容现有调用点)
quota = await ensure_quota_enabled(db, api_key_id=api_key_id)
_check(quota.asset_limit, quota.asset_used, asset_count_delta, "素材总数")
return quota
def remote_project_name() -> str:
"""火山 ProjectNameV3 中转统一共用这个 Project)。"""
return PRIVATE_PORTRAIT_REMOTE_PROJECT_NAME
def remote_group_name(*, api_key_id: str, project_slug: str, id: str) -> str:
"""火山 GroupNamevp-api-{api_key_id_short}-{id}-{slug} 最多 128 字符。"""
short_key = (api_key_id or "")
return f"vp-api-{short_key}-{id}-{project_slug}"[:128]
@@ -0,0 +1,547 @@
from __future__ import annotations
import hashlib
import logging
import mimetypes
import os
import shutil
import subprocess
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import NamedTuple
from urllib.parse import urlparse
import httpx
from fastapi import HTTPException, UploadFile
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.upload_resource import UploadResourceTypeEnum
from app.schemas.virtual_portrait_v3.upload import VpV3UploadOut
from app.services.video_cover_service import get_ffmpeg_bin
logger = logging.getLogger("videogen")
# 北京时间(UTC+8)统一基准:V3 业务所有时间写入 / 时间比较唯一参考
_BJ_TZ = timezone(timedelta(hours=8))
def _bj_now() -> datetime:
"""返回当前北京时间(UTC+8naive datetime(去掉 tzinfo,与 DB naive 存储一致)。"""
return datetime.now(_BJ_TZ).replace(tzinfo=None)
VP_V3_IMAGE_MAX_BYTES = 10 * 1024 * 1024
VP_V3_VIDEO_MAX_BYTES = 100 * 1024 * 1024
VP_V3_MODULE_NAME = "vp_v3_virtual"
IMAGE_EXT_ALLOWED = {"jpg", "jpeg", "png", "webp", "bmp"}
VIDEO_EXT_ALLOWED = {"mp4", "mov", "m4v", "webm"}
IMAGE_MIME_ALLOWED = {
"image/jpeg", "image/jpg", "image/png", "image/webp", "image/bmp",
}
VIDEO_MIME_ALLOWED = {
"video/mp4", "video/quicktime", "video/x-m4v", "video/webm",
}
# URL 下载相关默认值
URL_DOWNLOAD_CONNECT_TIMEOUT_SEC = 15
URL_DOWNLOAD_READ_TIMEOUT_SEC = 300 # 大文件下载可以久一点,读的时候会按大小上限中断
URL_DOWNLOAD_MAX_REDIRECTS = 5
URL_DOWNLOAD_CHUNK_BYTES = 1024 * 1024 # 1MB
class DownloadedAsset(NamedTuple):
"""URL 下载到本地后的结果。"""
url: str # 对外访问 URL(最终要存到 VpV3Asset.source_url 的)
filename: str # 落盘后的文件名
file_size_bytes: int # 实际文件大小
mime_type: str | None # 从响应头/扩展名推断出的 MIME
duration_seconds: float | None # 视频:ffprobe 探测到的时长(Image 为 None)
suggested_name: str | None # 从 URL 或 Content-Disposition 推断的展示名(无扩展名)
def _max_bytes(asset_type: str) -> int:
return VP_V3_VIDEO_MAX_BYTES if asset_type == UploadResourceTypeEnum.VIDEO.value else VP_V3_IMAGE_MAX_BYTES
def _allowed_mime_set(asset_type: str) -> set[str]:
return VIDEO_MIME_ALLOWED if asset_type == UploadResourceTypeEnum.VIDEO.value else IMAGE_MIME_ALLOWED
def _safe_ext(filename: str, asset_type: str) -> str:
ext = (os.path.splitext(filename or "")[1].lower().lstrip(".") or "").strip()
allowed = VIDEO_EXT_ALLOWED if asset_type == UploadResourceTypeEnum.VIDEO.value else IMAGE_EXT_ALLOWED
if ext and ext in allowed:
return ext
# fallback
return "mp4" if asset_type == UploadResourceTypeEnum.VIDEO.value else "png"
def _get_ffprobe_bin() -> str:
ffmpeg = Path(get_ffmpeg_bin())
sibling = ffmpeg.with_name("ffprobe.exe" if ffmpeg.suffix.lower() == ".exe" else "ffprobe")
if sibling.exists():
return str(sibling)
found = shutil.which("ffprobe") or shutil.which("ffprobe.exe")
if found:
return found
raise RuntimeError("未找到 ffprobe,请确保其与 FFMPEG_BIN 同目录或已加入 PATH")
def _probe_duration_optional(video_path: Path) -> float | None:
"""尝试 ffprobe 探测视频时长,失败不抛,返回 None 让调用方自己处理。"""
import json as _json
try:
ffprobe_bin = _get_ffprobe_bin()
# 检查 ffprobe 是否可用
if not shutil.which(ffprobe_bin) and ffprobe_bin == "ffprobe":
logger.warning("ffprobe 未在系统 PATH 中找到,无法探测视频时长。请安装 ffprobe 并添加到 PATH。")
return None
timeout = int(getattr(settings, "SHOT_FFPROBE_TIMEOUT_SECONDS", 20) or 20)
cmd = [
ffprobe_bin,
"-v", "error",
"-show_entries", "format=duration",
"-of", "json",
str(video_path),
]
completed = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, timeout=timeout, check=False,
)
if completed.returncode != 0:
logger.warning(
"ffprobe 执行失败: returncode=%s stderr=%s",
completed.returncode, completed.stderr[:200],
)
return None
if not completed.stdout:
return None
data = _json.loads(completed.stdout or "{}")
dur_raw = (data.get("format") or {}).get("duration")
if dur_raw is None:
return None
dur = float(dur_raw)
if dur <= 0:
return None
return dur
except subprocess.TimeoutExpired:
logger.warning("ffprobe 超时: %s", str(video_path))
return None
except Exception as exc: # noqa: BLE001
logger.warning("ffprobe 探测视频时长失败: %s", exc)
return None
def _build_destination(*, api_key_id: str, asset_type: str, original_filename: str, duration_seconds: float | None) -> tuple[Path, str, str]:
"""构建 vp_v3 上传存储路径 + 对外访问 URL。
存储路径:UPLOAD_LOCAL_PATH/api/private_portrait_virtual/{asset_type}/{yyyy}/{mm}/{dd}/{uuid}.{ext}
"""
now = _bj_now()
ext = _safe_ext(original_filename, asset_type)
safe_uuid = uuid.uuid4().hex
year = f"{now.year:04d}"
month = f"{now.month:02d}"
day = f"{now.day:02d}"
sub_type = "videos" if asset_type == UploadResourceTypeEnum.VIDEO.value else "images"
rel_dir = Path("api") / "private_portrait_virtual" / sub_type / year / month / day
filename = f"vp_v3_{safe_uuid}.{ext}"
base_dir = Path(settings.UPLOAD_LOCAL_PATH) if settings.UPLOAD_LOCAL_PATH else Path("./storage/uploads")
final_path = base_dir / rel_dir / filename
# URL 前缀 /uploads/...
base_url = settings.UPLOAD_BASE_URL.rstrip("/") if settings and getattr(settings, "UPLOAD_BASE_URL", None) else "/uploads"
rel_url = f"/{rel_dir.as_posix()}/{filename}".replace("//", "/")
url = base_url + rel_url
return final_path, url, filename
def _guess_filename_from_url(url: str, cd_header: str | None) -> str:
"""优先从 Content-Disposition 拿文件名,其次从 URL path 拿,再 fallback 到 uuid 名。"""
# 1. Content-Disposition
if cd_header:
# filename="a.jpg" 或 filename*=UTF-8''a.jpg
import re as _re
m1 = _re.search(r"""filename\*\s*=\s*UTF-8''([^;]+)""", cd_header, flags=_re.IGNORECASE)
if m1:
from urllib.parse import unquote
try:
return unquote(m1.group(1).strip().strip('"').strip("'"))
except Exception: # noqa: BLE001
pass
m2 = _re.search(r"""filename\s*=\s*"([^"]+)""", cd_header, flags=_re.IGNORECASE)
if m2:
return m2.group(1)
m3 = _re.search(r"""filename\s*=\s*([^;]+)""", cd_header, flags=_re.IGNORECASE)
if m3:
return m3.group(1).strip().strip('"').strip("'")
# 2. URL path
try:
parsed = urlparse(url)
base = os.path.basename(parsed.path or "")
if base and "." in base:
return base
except Exception: # noqa: BLE001
pass
return f"vp_v3_{uuid.uuid4().hex[:12]}"
def _guess_ext_from_mime(mime: str | None, asset_type: str) -> str | None:
if not mime:
return None
# 按 asset_type 优先匹配
allowed_exts = VIDEO_EXT_ALLOWED if asset_type == UploadResourceTypeEnum.VIDEO.value else IMAGE_EXT_ALLOWED
guesses = mimetypes.guess_all_extensions(mime.strip().lower()) or []
for g in guesses:
ext = g.lower().lstrip(".")
if ext in allowed_exts:
return ext
# 额外的手写映射
extra_map = {
"image/jpeg": "jpg", "image/jpg": "jpg",
"video/quicktime": "mov", "video/x-m4v": "m4v",
}
if mime.lower() in extra_map and extra_map[mime.lower()] in allowed_exts:
return extra_map[mime.lower()]
return None
# ---------------------------------------------------------------------------
# 1) 上传本地文件(保留旧 API 但走下载流程的也可以共用保存逻辑)
# ---------------------------------------------------------------------------
async def upload_asset_file(
db: AsyncSession,
*,
api_key_id: str,
file: UploadFile,
asset_type: str,
duration_seconds: float | None = None,
) -> VpV3UploadOut:
"""V3 虚拟素材上传(独立实现,不经过用户容量账本 UploadResource)。
- 校验 MIME/扩展名/大小
- 落盘到 /uploads/images|videos/vp_v3/{api_key_id_short}/{yyyy}/{mm}/{dd}/
- 返回 url + 虚拟 resource_idhash 形式)
"""
if asset_type not in {UploadResourceTypeEnum.IMAGE.value, UploadResourceTypeEnum.VIDEO.value}:
raise HTTPException(status_code=400, detail="虚拟素材上传仅支持图片或视频")
max_size = _max_bytes(asset_type)
temp_path: str | None = None
try:
# 1. 落临时文件并限制大小
size_acc = 0
temp_dir = settings.UPLOAD_TEMP_DIR if settings and getattr(settings, "UPLOAD_TEMP_DIR", None) else "./uploads/_tmp_vp_v3"
Path(temp_dir).mkdir(parents=True, exist_ok=True)
temp_path = os.path.join(temp_dir, f"vp_v3_{uuid.uuid4().hex}")
with open(temp_path, "wb") as f:
while True:
chunk = await file.read(1024 * 1024)
if not chunk:
break
size_acc += len(chunk)
if size_acc > max_size:
raise HTTPException(
status_code=400,
detail=f"文件大小超出限制:{asset_type} 最大 {max_size // (1024*1024)} MB",
)
f.write(chunk)
file_size_bytes = size_acc
if file_size_bytes <= 0:
raise HTTPException(status_code=400, detail="空文件不允许上传")
# 2. 构建最终路径 + URL
final_path, url, safe_filename = _build_destination(
api_key_id=api_key_id,
asset_type=asset_type,
original_filename=file.filename or safe_filename,
duration_seconds=duration_seconds,
)
final_path.parent.mkdir(parents=True, exist_ok=True)
shutil.move(temp_path, final_path)
temp_path = None
# 3. 虚拟 resource_id(用于素材删除时的文件清理定位:hash(url))
resource_id = "vpv3_" + hashlib.sha256(url.encode()).hexdigest()[:24]
return VpV3UploadOut(
url=url,
filename=safe_filename,
type=asset_type,
resource_id=resource_id,
file_size_bytes=file_size_bytes,
duration_seconds=duration_seconds,
)
except HTTPException:
raise
except Exception as exc: # noqa: BLE001
logger.exception("vp_v3 上传失败:%s", exc)
raise HTTPException(status_code=500, detail=f"上传失败:{exc}") from exc
finally:
if temp_path and os.path.exists(temp_path):
try:
os.remove(temp_path)
except Exception: # noqa: BLE001
pass
# ---------------------------------------------------------------------------
# 2) URL 下载到本地(新流程:创建素材时一步到位,由 create_asset 内部调用)
# ---------------------------------------------------------------------------
async def download_url_to_local(
*,
api_key_id: str,
asset_type: str,
source_url: str,
requested_filename: str | None = None,
) -> DownloadedAsset:
"""把传入的远程 URLhttp/https)下载到本地 vp_v3 上传目录,返回本地 URL + 元信息。
完整的错误处理:
- 非法 URL → 400
- 连接/超时 → 502(外部资源不可达)
- HTTP 4xx/5xx → 502 带状态码
- Content-Type 不在允许列表 → 415
- 超出大小上限(读内容时逐 chunk 检查)→ 413
- 下载一半失败 → 清理临时文件,不留下半截
- 视频可选探测 ffprobe,失败不抛错(调用方自行用 payload.video_duration
"""
if asset_type not in {UploadResourceTypeEnum.IMAGE.value, UploadResourceTypeEnum.VIDEO.value}:
raise HTTPException(status_code=400, detail="虚拟素材仅支持图片或视频")
# URL 合法性
if not source_url or not isinstance(source_url, str):
raise HTTPException(status_code=400, detail="source_url 不能为空")
parsed = urlparse(source_url.strip())
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise HTTPException(status_code=400, detail="source_url 必须是合法的 http(s) URL")
# 拒绝私有/内网地址(SSRF 防御的最小集;生产环境可再严格)
import ipaddress
host_only = parsed.hostname or ""
try:
ip_obj = ipaddress.ip_address(host_only)
if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_reserved or ip_obj.is_link_local:
raise HTTPException(status_code=400, detail="source_url 不允许指向内网/本机地址")
except ValueError:
# 不是 IP,是域名 → 放行
pass
max_bytes = _max_bytes(asset_type)
allowed_mimes = _allowed_mime_set(asset_type)
# 用户代理:标成我们服务的 UA,避免一些图片防盗链 403
user_agent = (
"Mozilla/5.0 (compatible; VideoGenVPV3/1.0; +https://minzhongzc.com/)"
if getattr(settings, "VP_V3_DOWNLOAD_UA", None) is None
else str(getattr(settings, "VP_V3_DOWNLOAD_UA"))
)
temp_path: str | None = None
final_path: Path | None = None
# 外层初始化,保证 client.stream 内部 raise 的情况下外层仍然可访问
inferred_filename: str = f"vp_v3_{uuid.uuid4().hex[:12]}"
mime: str | None = None
size_acc: int = 0
try:
# --- 第一步:下载到临时文件,限制大小 + 校验响应头 ---
temp_dir = settings.UPLOAD_TEMP_DIR if settings and getattr(settings, "UPLOAD_TEMP_DIR", None) else "./uploads/_tmp_vp_v3"
Path(temp_dir).mkdir(parents=True, exist_ok=True)
temp_path = os.path.join(temp_dir, f"vp_v3_url_{uuid.uuid4().hex}")
transport = httpx.AsyncHTTPTransport(retries=1)
timeout = httpx.Timeout(
connect=URL_DOWNLOAD_CONNECT_TIMEOUT_SEC,
read=URL_DOWNLOAD_READ_TIMEOUT_SEC,
write=URL_DOWNLOAD_CONNECT_TIMEOUT_SEC,
pool=URL_DOWNLOAD_CONNECT_TIMEOUT_SEC,
)
headers = {"User-Agent": user_agent, "Accept": "*/*"}
async with httpx.AsyncClient(
timeout=timeout,
transport=transport,
follow_redirects=True,
max_redirects=URL_DOWNLOAD_MAX_REDIRECTS,
verify=bool(getattr(settings, "VP_V3_DOWNLOAD_VERIFY_SSL", True)),
) as client:
async with client.stream("GET", source_url.strip(), headers=headers) as resp:
# HTTP 状态码
if resp.status_code >= 400:
detail = f"远程资源返回状态码 {resp.status_code}"
try:
snippet = (await resp.aread())[:200]
if snippet:
detail += f",响应片段:{snippet.decode('utf-8', errors='ignore')}"
except Exception: # noqa: BLE001
pass
raise HTTPException(
status_code=502,
detail=f"source_url 下载失败(HTTP {resp.status_code}):" + detail,
)
# Content-Type 校验(没有就 fallback 到扩展名推断)
content_type_raw = resp.headers.get("Content-Type") or ""
mime = (content_type_raw.split(";")[0] or "").strip().lower() or None
if mime and mime not in allowed_mimes:
# 一些 CDN 会用 application/octet-stream,这种情况跳过 MIME 检查,用扩展名兜底
if mime != "application/octet-stream":
raise HTTPException(
status_code=415,
detail=(
f"不支持的 Content-Type{mime}"
f"{asset_type} 仅支持:{', '.join(sorted(allowed_mimes))}"
),
)
# Content-Length 预估检查(存在且超出就直接拒,不下载)
content_length = resp.headers.get("Content-Length")
if content_length:
try:
cl = int(content_length)
if cl > max_bytes:
raise HTTPException(
status_code=413,
detail=(
f"远程资源太大(Content-Length={cl}),超过 "
f"{asset_type} 上限 {max_bytes} 字节"
),
)
except ValueError:
pass
# filename 推断(用于扩展名 + 展示名)
cd = resp.headers.get("Content-Disposition")
inferred_filename = _guess_filename_from_url(source_url, cd)
if requested_filename:
# 若用户传了 name 就优先用它做展示名,但扩展名仍然以 mime/url 推断为准
try:
base_display = os.path.splitext(os.path.basename(requested_filename))[0]
old_ext = os.path.splitext(inferred_filename)[1] if inferred_filename else ""
inferred_filename = base_display + (old_ext or "")
except Exception: # noqa: BLE001
pass
# 扩展名再精化:如果 MIME 能得出扩展名,优先用
ext_from_mime = _guess_ext_from_mime(mime, asset_type)
if ext_from_mime:
try:
stem = os.path.splitext(inferred_filename)[0]
inferred_filename = f"{stem}.{ext_from_mime}"
except Exception: # noqa: BLE001
pass
# 流式下载到 temp_path,逐 chunk 检查大小
size_acc = 0
with open(temp_path, "wb") as f:
async for chunk in resp.aiter_bytes():
if not chunk:
continue
size_acc += len(chunk)
if size_acc > max_bytes:
raise HTTPException(
status_code=413,
detail=(
f"远程资源大小超过 {asset_type} 上限 "
f"{max_bytes // (1024*1024)} MB"
),
)
f.write(chunk)
# ======== 以下在 client.stream 退出后、但仍在 httpx.AsyncClient 上下文内执行 ========
# 空文件检查
file_size_bytes = size_acc
if file_size_bytes <= 0:
raise HTTPException(status_code=400, detail="远程 URL 返回空文件")
# --- 第二步:视频可选 ffprobe 探测时长 ---
duration: float | None = None
if asset_type == UploadResourceTypeEnum.VIDEO.value:
duration = _probe_duration_optional(Path(temp_path))
# --- 第三步:落到最终目录(与 _build_destination 一致的目录结构/权限) ---
final_path, url_out, final_filename = _build_destination(
api_key_id=api_key_id,
asset_type=asset_type,
original_filename=inferred_filename,
duration_seconds=duration,
)
final_path.parent.mkdir(parents=True, exist_ok=True)
shutil.move(temp_path, final_path)
temp_path = None
# 最终 MIME:按扩展名反推一个(如果之前没拿到)
if not mime:
mime, _ = mimetypes.guess_type(final_filename)
if not mime:
mime = "image/png" if asset_type == UploadResourceTypeEnum.IMAGE.value else "video/mp4"
suggested_name: str | None = None
try:
stem = os.path.splitext(inferred_filename or final_filename)[0]
if stem and not stem.startswith("vp_v3_"):
suggested_name = stem[:100] or None
except Exception: # noqa: BLE001
pass
return DownloadedAsset(
url=url_out,
filename=final_filename,
file_size_bytes=file_size_bytes,
mime_type=mime,
duration_seconds=duration,
suggested_name=suggested_name,
)
except HTTPException:
raise
except (httpx.TimeoutException, httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout) as exc:
raise HTTPException(status_code=502, detail=f"远程 URL 连接/读取超时:{exc}") from exc
except httpx.HTTPError as exc:
raise HTTPException(status_code=502, detail=f"远程 URL 下载失败:{exc}") from exc
except Exception as exc: # noqa: BLE001
logger.exception("vp_v3 URL 下载异常:url=%s err=%s", source_url, exc)
raise HTTPException(status_code=500, detail=f"URL 下载保存失败:{exc}") from exc
finally:
# 任何失败都清掉半截临时文件;但最终文件已经 move 过去了的就不动
if temp_path and os.path.exists(temp_path):
try:
os.remove(temp_path)
except Exception: # noqa: BLE001
pass
def delete_local_file_by_url(local_url: str) -> bool:
"""素材删除时根据 source_url 删除本地落盘文件(非强制,失败不抛)。"""
if not local_url:
return False
try:
base_url = settings.UPLOAD_BASE_URL.rstrip("/") if settings and getattr(settings, "UPLOAD_BASE_URL", None) else "/uploads"
rel_url = local_url
if rel_url.startswith(base_url):
rel_url = rel_url[len(base_url):]
if not rel_url.startswith("/"):
return False
# /uploads/api/private_portrait_virtual/... → /api/private_portrait_virtual/... → UPLOAD_LOCAL_PATH/api/private_portrait_virtual/...
sub_part = rel_url[len("/uploads"):] if rel_url.startswith("/uploads") else rel_url
base_dir = Path(settings.UPLOAD_LOCAL_PATH) if settings.UPLOAD_LOCAL_PATH else Path("./storage/uploads")
target = (base_dir / sub_part.lstrip("/")).resolve()
base_dir_resolved = base_dir.resolve()
# 仅允许删除 base_dir 下的文件(目录穿越防御)
if not str(target).startswith(str(base_dir_resolved)):
return False
if target.is_file():
target.unlink(missing_ok=True)
return True
except Exception: # noqa: BLE001
logger.exception("vp_v3 清理本地文件失败:%s", local_url)
return False