拆镜复刻开发完成
This commit is contained in:
@@ -1,103 +1,28 @@
|
||||
# app/services/celery_download_recovery_service.py
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||
|
||||
from app.config import settings
|
||||
|
||||
try:
|
||||
from redis.exceptions import RedisError
|
||||
except ImportError:
|
||||
RedisError = RuntimeError # type: ignore[assignment]
|
||||
from app.services.redis_registry_service import (
|
||||
close_registry_redis,
|
||||
datetime_to_epoch,
|
||||
ensure_aware_utc,
|
||||
get_registry_redis,
|
||||
redis_get_due_registry_ids,
|
||||
redis_get_registry_payloads,
|
||||
redis_postpone_registry_item,
|
||||
redis_remove_registry_item,
|
||||
redis_upsert_registry_item,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger("video_gen")
|
||||
|
||||
_redis_client: Optional[Any] = None
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ensure_aware_utc(value: Optional[datetime]) -> Optional[datetime]:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def datetime_to_epoch(value: Optional[datetime]) -> int:
|
||||
checked_value = ensure_aware_utc(value) or utc_now()
|
||||
return int(checked_value.timestamp())
|
||||
|
||||
|
||||
def _registry_redis_url() -> str:
|
||||
return settings.CELERY_BROKER_URL or settings.REDIS_URL or ""
|
||||
|
||||
|
||||
async def get_registry_redis() -> Optional[Any]:
|
||||
global _redis_client
|
||||
|
||||
if _redis_client is not None:
|
||||
return _redis_client
|
||||
|
||||
redis_url = _registry_redis_url()
|
||||
if not redis_url:
|
||||
return None
|
||||
|
||||
try:
|
||||
from redis.asyncio import Redis
|
||||
except ImportError as exc:
|
||||
logger.warning(
|
||||
"下载容灾 Redis 注册表不可用，redis 依赖未安装。error=%s",
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
redis_client = Redis.from_url(redis_url, decode_responses=True)
|
||||
await redis_client.ping()
|
||||
_redis_client = redis_client
|
||||
return _redis_client
|
||||
except (RedisError, OSError, RuntimeError) as exc:
|
||||
logger.warning(
|
||||
"下载容灾 Redis 注册表不可用，降级为仅 DB 容灾。error=%s",
|
||||
exc,
|
||||
)
|
||||
_redis_client = None
|
||||
return None
|
||||
|
||||
|
||||
async def close_registry_redis() -> None:
|
||||
global _redis_client
|
||||
|
||||
client = _redis_client
|
||||
_redis_client = None
|
||||
|
||||
if client is None:
|
||||
return
|
||||
|
||||
try:
|
||||
close_method = getattr(client, "close", None)
|
||||
if close_method is None:
|
||||
return
|
||||
|
||||
close_result = close_method()
|
||||
if inspect.isawaitable(close_result):
|
||||
await close_result
|
||||
except (RedisError, OSError, RuntimeError) as exc:
|
||||
logger.debug(
|
||||
"关闭下载容灾 Redis 注册表连接失败。error=%s",
|
||||
exc,
|
||||
)
|
||||
# 说明：
|
||||
# - 本文件保留旧函数名，作为下载容灾兼容层。
|
||||
# - 底层 Redis Hash/ZSet 操作已迁移到 redis_registry_service.py。
|
||||
# - 下载 active key、Redis URL 选择逻辑不变，避免影响已稳定下载模块。
|
||||
|
||||
|
||||
def build_download_active_payload(
|
||||
@@ -131,32 +56,12 @@ def build_download_active_payload(
|
||||
"attempt": int(attempt or 0),
|
||||
"queue": queue,
|
||||
"priority": priority,
|
||||
"enqueue_at": (
|
||||
datetime_to_epoch(checked_enqueue_at)
|
||||
if checked_enqueue_at
|
||||
else None
|
||||
),
|
||||
"started_at": (
|
||||
datetime_to_epoch(checked_started_at)
|
||||
if checked_started_at
|
||||
else None
|
||||
),
|
||||
"enqueue_at": datetime_to_epoch(checked_enqueue_at) if checked_enqueue_at else None,
|
||||
"started_at": datetime_to_epoch(checked_started_at) if checked_started_at else None,
|
||||
"updated_at": datetime_to_epoch(checked_updated_at),
|
||||
"lease_until": (
|
||||
datetime_to_epoch(checked_lease_until)
|
||||
if checked_lease_until
|
||||
else None
|
||||
),
|
||||
"next_retry_at": (
|
||||
datetime_to_epoch(checked_next_retry_at)
|
||||
if checked_next_retry_at
|
||||
else None
|
||||
),
|
||||
"check_at": (
|
||||
datetime_to_epoch(checked_check_at)
|
||||
if checked_check_at
|
||||
else None
|
||||
),
|
||||
"lease_until": datetime_to_epoch(checked_lease_until) if checked_lease_until else None,
|
||||
"next_retry_at": datetime_to_epoch(checked_next_retry_at) if checked_next_retry_at else None,
|
||||
"check_at": datetime_to_epoch(checked_check_at) if checked_check_at else None,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
@@ -167,56 +72,23 @@ async def upsert_download_active(
|
||||
payload: Dict[str, Any],
|
||||
check_at: Optional[Union[datetime, int, float]],
|
||||
) -> None:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return
|
||||
|
||||
if isinstance(check_at, datetime):
|
||||
score = datetime_to_epoch(check_at)
|
||||
elif check_at is None:
|
||||
score = datetime_to_epoch(utc_now())
|
||||
else:
|
||||
score = int(float(check_at))
|
||||
|
||||
updated_payload = dict(payload)
|
||||
updated_payload["check_at"] = score
|
||||
|
||||
try:
|
||||
pipe: Any = redis.pipeline(transaction=True)
|
||||
pipe.hset(
|
||||
settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
record_id,
|
||||
json.dumps(updated_payload, ensure_ascii=False, default=str),
|
||||
)
|
||||
pipe.zadd(
|
||||
settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
{record_id: score},
|
||||
)
|
||||
await pipe.execute()
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"写入下载容灾 Redis 注册表失败。record_id=%s, error=%s",
|
||||
record_id,
|
||||
exc,
|
||||
)
|
||||
await redis_upsert_registry_item(
|
||||
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
item_id=record_id,
|
||||
payload=payload,
|
||||
check_at=check_at,
|
||||
log_context="download_active",
|
||||
)
|
||||
|
||||
|
||||
async def remove_download_active(record_id: str) -> None:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return
|
||||
|
||||
try:
|
||||
pipe: Any = redis.pipeline(transaction=True)
|
||||
pipe.hdel(settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, record_id)
|
||||
pipe.zrem(settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY, record_id)
|
||||
await pipe.execute()
|
||||
except (RedisError, OSError, RuntimeError) as exc:
|
||||
logger.warning(
|
||||
"删除下载容灾 Redis 注册表失败。record_id=%s, error=%s",
|
||||
record_id,
|
||||
exc,
|
||||
)
|
||||
await redis_remove_registry_item(
|
||||
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
item_id=record_id,
|
||||
log_context="download_active",
|
||||
)
|
||||
|
||||
|
||||
async def get_due_download_record_ids(
|
||||
@@ -224,68 +96,22 @@ async def get_due_download_record_ids(
|
||||
limit: Optional[int] = None,
|
||||
now: Optional[datetime] = None,
|
||||
) -> List[str]:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return []
|
||||
|
||||
batch_limit = int(limit or settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100)
|
||||
score = datetime_to_epoch(now or utc_now())
|
||||
|
||||
try:
|
||||
result = await redis.zrangebyscore(
|
||||
settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
min="-inf",
|
||||
max=score,
|
||||
start=0,
|
||||
num=batch_limit,
|
||||
)
|
||||
return [str(item) for item in result]
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"扫描下载容灾 Redis ZSet 失败。error=%s",
|
||||
exc,
|
||||
)
|
||||
return []
|
||||
return await redis_get_due_registry_ids(
|
||||
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
limit=limit or int(settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100),
|
||||
now=now,
|
||||
log_context="download_active",
|
||||
)
|
||||
|
||||
|
||||
async def get_download_active_payloads(
|
||||
record_ids: Iterable[str],
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
cleaned_record_ids = [str(item) for item in record_ids if item]
|
||||
if not cleaned_record_ids:
|
||||
return {}
|
||||
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return {}
|
||||
|
||||
try:
|
||||
raw_values = await redis.hmget(
|
||||
settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
cleaned_record_ids,
|
||||
)
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"读取下载容灾 Redis Hash 失败。error=%s",
|
||||
exc,
|
||||
)
|
||||
return {}
|
||||
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for record_id, raw in zip(cleaned_record_ids, raw_values):
|
||||
if not raw:
|
||||
continue
|
||||
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
if isinstance(value, dict):
|
||||
result[record_id] = value
|
||||
|
||||
return result
|
||||
return await redis_get_registry_payloads(
|
||||
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
item_ids=record_ids,
|
||||
log_context="download_active",
|
||||
)
|
||||
|
||||
|
||||
async def postpone_download_active_check(
|
||||
@@ -294,45 +120,14 @@ async def postpone_download_active_check(
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
check_at: Optional[Union[datetime, int, float]] = None,
|
||||
) -> None:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return
|
||||
if check_at is None:
|
||||
check_at = utc_now().timestamp() + int(settings.DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS or 300)
|
||||
|
||||
if isinstance(check_at, datetime):
|
||||
score = datetime_to_epoch(check_at)
|
||||
elif check_at is None:
|
||||
score = datetime_to_epoch(utc_now()) + int(
|
||||
settings.DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS or 300
|
||||
)
|
||||
else:
|
||||
score = int(float(check_at))
|
||||
|
||||
try:
|
||||
pipe: Any = redis.pipeline(transaction=True)
|
||||
pipe.zadd(
|
||||
settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
{record_id: score},
|
||||
)
|
||||
|
||||
if payload is not None:
|
||||
updated_payload = dict(payload)
|
||||
updated_payload["check_at"] = score
|
||||
updated_payload["updated_at"] = datetime_to_epoch(utc_now())
|
||||
|
||||
pipe.hset(
|
||||
settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
record_id,
|
||||
json.dumps(
|
||||
updated_payload,
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
|
||||
await pipe.execute()
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"刷新下载容灾 Redis 检查时间失败。record_id=%s, error=%s",
|
||||
record_id,
|
||||
exc,
|
||||
)
|
||||
await redis_postpone_registry_item(
|
||||
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||
item_id=record_id,
|
||||
payload=payload,
|
||||
check_at=check_at,
|
||||
log_context="download_active",
|
||||
)
|
||||
|
||||
@@ -23,3 +23,15 @@ async def notify_chat_generation_task_finished(db: AsyncSession, task: ChatGener
|
||||
await handle_chat_generation_task_completed(db, task)
|
||||
elif task.status == "failed":
|
||||
await handle_chat_generation_task_failed(db, task)
|
||||
return
|
||||
|
||||
if task.generation_mode == "shot_replicate":
|
||||
from app.services.shot_replicate_flow_service import (
|
||||
handle_chat_generation_task_completed,
|
||||
handle_chat_generation_task_failed,
|
||||
)
|
||||
if task.status == "completed":
|
||||
await handle_chat_generation_task_completed(db, task)
|
||||
elif task.status == "failed":
|
||||
await handle_chat_generation_task_failed(db, task)
|
||||
return
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
@@ -16,13 +17,21 @@ from app.services.celery_download_recovery_service import (
|
||||
postpone_download_active_check,
|
||||
remove_download_active,
|
||||
)
|
||||
from app.services.generation_log_service import log_task_event
|
||||
from app.services.generation_log_service import log_provider_call, log_task_event
|
||||
from app.services.generation_module_hook_service import notify_chat_generation_task_finished
|
||||
from app.services.generation_provider_service import poll_provider_task
|
||||
from app.services.generation_refund_service import mark_chat_generation_task_failed_and_refund_once
|
||||
from app.services.redis_registry_service import (
|
||||
redis_get_due_registry_ids,
|
||||
redis_get_registry_payloads,
|
||||
redis_postpone_registry_item,
|
||||
redis_remove_registry_item,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("video_gen")
|
||||
|
||||
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate"}
|
||||
ALLOWED_GENERATION_MODES = {"chatapi_async", "hot_opening_replicate", "shot_replicate"}
|
||||
POLL_QUEUE = "gen_provider_poll"
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -58,6 +67,52 @@ def _is_final_task_state(task: ChatGenerationTask) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _is_success(status: str | None) -> bool:
|
||||
return str(status or "").lower() in ("succeeded", "success", "completed", "done")
|
||||
|
||||
|
||||
def _is_failed(status: str | None) -> bool:
|
||||
return str(status or "").lower() in ("failed", "error", "canceled", "cancelled")
|
||||
|
||||
|
||||
def _engine_snapshot(task: ChatGenerationTask) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(task.engine_snapshot_json or "{}")
|
||||
return value if isinstance(value, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _poll_queue_timeout_at(now: datetime | None = None) -> datetime:
|
||||
current_time = now or _now()
|
||||
return current_time + timedelta(seconds=int(settings.POLL_TASK_QUEUE_TIMEOUT_SECONDS or 120))
|
||||
|
||||
|
||||
async def _remove_poll_active(task_id: str) -> None:
|
||||
await redis_remove_registry_item(
|
||||
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
||||
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||
item_id=task_id,
|
||||
log_context="poll_active",
|
||||
)
|
||||
|
||||
|
||||
async def _postpone_poll_active(
|
||||
*,
|
||||
task_id: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
check_at: datetime | int | float | None = None,
|
||||
) -> None:
|
||||
await redis_postpone_registry_item(
|
||||
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
||||
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||
item_id=task_id,
|
||||
payload=payload,
|
||||
check_at=check_at or _poll_queue_timeout_at(),
|
||||
log_context="poll_active",
|
||||
)
|
||||
|
||||
|
||||
async def recover_one_download_task(
|
||||
db: AsyncSession,
|
||||
task: ChatGenerationTask,
|
||||
@@ -221,7 +276,7 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||
select(ChatGenerationTask)
|
||||
.where(
|
||||
ChatGenerationTask.deleted_at.is_(None),
|
||||
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate"]),
|
||||
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate", "shot_replicate"]),
|
||||
ChatGenerationTask.status == "generating",
|
||||
ChatGenerationTask.remote_result_url.is_not(None),
|
||||
ChatGenerationTask.pipeline_stage.in_(
|
||||
@@ -249,134 +304,363 @@ async def recover_download_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||
return {"checked": len(checked_ids), "results": results}
|
||||
|
||||
|
||||
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||
"""启动时生成链路容灾扫描。
|
||||
async def _mark_timeout(
|
||||
db: AsyncSession,
|
||||
task: ChatGenerationTask,
|
||||
*,
|
||||
error_message: str = "任务超时",
|
||||
) -> str:
|
||||
await mark_chat_generation_task_failed_and_refund_once(
|
||||
db,
|
||||
task=task,
|
||||
error_message=error_message,
|
||||
pipeline_stage="timeout",
|
||||
)
|
||||
await notify_chat_generation_task_finished(db, task)
|
||||
await db.commit()
|
||||
await _remove_poll_active(task.id)
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="TASK_TIMEOUT",
|
||||
to_status="failed",
|
||||
to_stage="timeout",
|
||||
)
|
||||
return "mark_timeout"
|
||||
|
||||
只在 Celery worker 启动时跑一次，不引入 beat，不新增第四条启动命令。
|
||||
用于把 queued/creating/waiting_remote/polling/result_ready 等中间态重新投递到现有三个队列。
|
||||
|
||||
async def _mark_failed(
|
||||
db: AsyncSession,
|
||||
task: ChatGenerationTask,
|
||||
*,
|
||||
error_message: str,
|
||||
event_type: str = "POLL_FAILED",
|
||||
detail: Any = None,
|
||||
) -> str:
|
||||
await mark_chat_generation_task_failed_and_refund_once(
|
||||
db,
|
||||
task=task,
|
||||
error_message=error_message,
|
||||
pipeline_stage="failed",
|
||||
)
|
||||
await notify_chat_generation_task_finished(db, task)
|
||||
await db.commit()
|
||||
await _remove_poll_active(task.id)
|
||||
await log_task_event(task, event_type=event_type, message=task.error_message, detail=detail)
|
||||
return "mark_failed"
|
||||
|
||||
|
||||
async def _try_final_poll_before_timeout(db: AsyncSession, task: ChatGenerationTask) -> str:
|
||||
"""超时前最后查一次供应商，避免 Celery 中断导致本地假超时。
|
||||
|
||||
如果供应商已经成功，继续进入下载；如果仍 running 或查询失败，再按超时处理。
|
||||
"""
|
||||
from app.tasks.generation_download_tasks import enqueue_download_task
|
||||
|
||||
if not (task.provider_task_id or task.seedance_task_id):
|
||||
return await _mark_timeout(db, task)
|
||||
|
||||
try:
|
||||
poll_result = await poll_provider_task(db, task)
|
||||
status = poll_result.get("status")
|
||||
response_data = poll_result.get("response_data")
|
||||
except Exception as exc:
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="FINAL_POLL_BEFORE_TIMEOUT_ERROR",
|
||||
message=str(exc),
|
||||
)
|
||||
return await _mark_timeout(db, task)
|
||||
|
||||
try:
|
||||
provider_response = json.loads(response_data or "{}")
|
||||
except Exception:
|
||||
provider_response = {"raw": response_data}
|
||||
|
||||
snapshot = _engine_snapshot(task)
|
||||
await log_provider_call(
|
||||
task,
|
||||
provider=snapshot.get("provider") or "ark",
|
||||
api_type=f"{task.gen_type}_final_poll_before_timeout",
|
||||
model=snapshot.get("model_name"),
|
||||
engine_id=task.engine_id,
|
||||
status="success",
|
||||
provider_task_id=task.seedance_task_id or task.provider_task_id,
|
||||
response_data=provider_response,
|
||||
)
|
||||
|
||||
if _is_success(status):
|
||||
if task.gen_type == "image":
|
||||
task.remote_result_url = poll_result.get("image_url")
|
||||
task.image_tokens_used = poll_result.get("image_tokens", 0) or 0
|
||||
else:
|
||||
task.remote_result_url = poll_result.get("video_url")
|
||||
task.video_tokens_used = poll_result.get("video_tokens", 0) or 0
|
||||
|
||||
task.provider_response_json = response_data
|
||||
if not task.remote_result_url:
|
||||
return await _mark_failed(
|
||||
db,
|
||||
task,
|
||||
error_message="供应商任务成功但未返回结果URL",
|
||||
detail=poll_result,
|
||||
)
|
||||
|
||||
task.pipeline_stage = "result_ready"
|
||||
task.retry_count = 0
|
||||
await db.commit()
|
||||
await _remove_poll_active(task.id)
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="POLL_SUCCESS_AFTER_TIMEOUT_RECOVERY",
|
||||
to_stage="result_ready",
|
||||
detail=poll_result,
|
||||
)
|
||||
await enqueue_download_task(db, task, recover=True, reason="final_poll_before_timeout_success")
|
||||
return "recover_timeout_success_to_download"
|
||||
|
||||
if _is_failed(status):
|
||||
task.provider_response_json = response_data
|
||||
return await _mark_failed(
|
||||
db,
|
||||
task,
|
||||
error_message=poll_result.get("error") or f"供应商任务失败: {status}",
|
||||
detail=poll_result,
|
||||
)
|
||||
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="FINAL_POLL_BEFORE_TIMEOUT_PENDING",
|
||||
message=f"status={status}",
|
||||
detail=poll_result,
|
||||
)
|
||||
return await _mark_timeout(db, task)
|
||||
|
||||
|
||||
async def recover_one_generation_task(
|
||||
db: AsyncSession,
|
||||
task: ChatGenerationTask,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
source: str = "startup_db",
|
||||
) -> str:
|
||||
from app.tasks.generation_create_tasks import chatapi_create_generation_task
|
||||
from app.tasks.generation_download_tasks import enqueue_download_task
|
||||
from app.tasks.generation_poll_tasks import poll_generation_task
|
||||
from app.tasks.generation_poll_tasks import poll_generation_task, register_poll_active
|
||||
|
||||
current_time = _now()
|
||||
results: dict[str, int] = {}
|
||||
redis_payload = payload or {}
|
||||
|
||||
query_result = await db.execute(
|
||||
select(ChatGenerationTask)
|
||||
.where(
|
||||
ChatGenerationTask.deleted_at.is_(None),
|
||||
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate"]),
|
||||
ChatGenerationTask.status == "generating",
|
||||
ChatGenerationTask.pipeline_stage.in_(
|
||||
[
|
||||
"queued",
|
||||
"preparing",
|
||||
"creating_provider_task",
|
||||
"waiting_remote",
|
||||
"polling",
|
||||
"result_ready",
|
||||
]
|
||||
),
|
||||
)
|
||||
.order_by(ChatGenerationTask.updated_at.asc())
|
||||
.limit(int(settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100))
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
tasks = query_result.scalars().all()
|
||||
if not task:
|
||||
return "skip_missing_task"
|
||||
if task.generation_mode not in ALLOWED_GENERATION_MODES:
|
||||
await _remove_poll_active(task.id)
|
||||
return "clean_invalid_mode"
|
||||
if _is_final_task_state(task):
|
||||
await _remove_poll_active(task.id)
|
||||
return "clean_final_state"
|
||||
if task.status != "generating":
|
||||
await _remove_poll_active(task.id)
|
||||
return "clean_not_generating"
|
||||
|
||||
for task in tasks:
|
||||
if task.deadline_at and _is_expired(task.deadline_at, current_time):
|
||||
await mark_chat_generation_task_failed_and_refund_once(
|
||||
db,
|
||||
task=task,
|
||||
error_message="任务超时",
|
||||
pipeline_stage="timeout",
|
||||
)
|
||||
await notify_chat_generation_task_finished(db, task)
|
||||
if task.deadline_at and _is_expired(task.deadline_at, current_time):
|
||||
if task.pipeline_stage in ("waiting_remote", "polling"):
|
||||
return await _try_final_poll_before_timeout(db, task)
|
||||
return await _mark_timeout(db, task)
|
||||
|
||||
if task.pipeline_stage in ("queued", "preparing", "creating_provider_task"):
|
||||
if task.provider_task_id or task.seedance_task_id:
|
||||
task.pipeline_stage = "waiting_remote"
|
||||
await db.commit()
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="TASK_TIMEOUT",
|
||||
to_status="failed",
|
||||
to_stage="timeout",
|
||||
event_type="GENERATION_RECOVERY_ENQUEUE",
|
||||
message=f"{source} 发现创建阶段已存在供应商任务ID，恢复投递轮询队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
|
||||
)
|
||||
action = "mark_timeout"
|
||||
poll_generation_task.apply_async(args=[task.id], queue=POLL_QUEUE, countdown=0)
|
||||
await register_poll_active(
|
||||
task,
|
||||
check_at=_poll_queue_timeout_at(),
|
||||
reason=f"{source}_create_stage_has_provider_id",
|
||||
)
|
||||
return "recover_poll_from_create_stage"
|
||||
|
||||
elif task.pipeline_stage in ("queued", "preparing", "creating_provider_task"):
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="GENERATION_RECOVERY_ENQUEUE",
|
||||
message=f"{source} 发现创建阶段任务未完成，恢复投递创建队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
|
||||
)
|
||||
chatapi_create_generation_task.apply_async(
|
||||
args=[task.id],
|
||||
queue="gen_chatapi_create",
|
||||
countdown=0,
|
||||
)
|
||||
return "recover_create"
|
||||
|
||||
if task.pipeline_stage in ("waiting_remote", "polling"):
|
||||
if task.remote_result_url:
|
||||
await _remove_poll_active(task.id)
|
||||
await enqueue_download_task(
|
||||
db,
|
||||
task,
|
||||
recover=True,
|
||||
reason=f"{source}_waiting_remote_has_result",
|
||||
)
|
||||
return "recover_waiting_has_result"
|
||||
|
||||
if task.provider_task_id or task.seedance_task_id:
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="GENERATION_RECOVERY_ENQUEUE",
|
||||
message="启动时发现创建阶段任务未完成，恢复投递创建队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage},
|
||||
message=f"{source} 发现远程等待/轮询阶段任务未完成，恢复投递轮询队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
|
||||
)
|
||||
chatapi_create_generation_task.apply_async(
|
||||
task.pipeline_stage = "waiting_remote"
|
||||
await db.commit()
|
||||
poll_generation_task.apply_async(
|
||||
args=[task.id],
|
||||
queue="gen_chatapi_create",
|
||||
queue=POLL_QUEUE,
|
||||
countdown=0,
|
||||
)
|
||||
action = "recover_create"
|
||||
await register_poll_active(
|
||||
task,
|
||||
check_at=_poll_queue_timeout_at(),
|
||||
reason=f"{source}_recover_poll",
|
||||
)
|
||||
return "recover_poll"
|
||||
|
||||
elif task.pipeline_stage in ("waiting_remote", "polling"):
|
||||
if task.remote_result_url:
|
||||
await enqueue_download_task(
|
||||
db,
|
||||
task,
|
||||
recover=True,
|
||||
reason="startup_waiting_remote_has_result",
|
||||
)
|
||||
action = "recover_waiting_has_result"
|
||||
elif task.provider_task_id or task.seedance_task_id:
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="GENERATION_RECOVERY_ENQUEUE",
|
||||
message="启动时发现远程等待/轮询阶段任务未完成，恢复投递轮询队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage},
|
||||
)
|
||||
task.pipeline_stage = "waiting_remote"
|
||||
await db.commit()
|
||||
poll_generation_task.apply_async(
|
||||
args=[task.id],
|
||||
queue="gen_provider_poll",
|
||||
countdown=0,
|
||||
)
|
||||
action = "recover_poll"
|
||||
else:
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="GENERATION_RECOVERY_ENQUEUE",
|
||||
message="启动时发现任务缺少供应商任务ID，恢复投递创建队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage},
|
||||
)
|
||||
task.pipeline_stage = "queued"
|
||||
await db.commit()
|
||||
chatapi_create_generation_task.apply_async(
|
||||
args=[task.id],
|
||||
queue="gen_chatapi_create",
|
||||
countdown=0,
|
||||
)
|
||||
action = "recover_create_missing_provider_id"
|
||||
await log_task_event(
|
||||
task,
|
||||
event_type="GENERATION_RECOVERY_ENQUEUE",
|
||||
message=f"{source} 发现任务缺少供应商任务ID，恢复投递创建队列",
|
||||
detail={"pipeline_stage": task.pipeline_stage, "payload": redis_payload},
|
||||
)
|
||||
task.pipeline_stage = "queued"
|
||||
await db.commit()
|
||||
await _remove_poll_active(task.id)
|
||||
chatapi_create_generation_task.apply_async(
|
||||
args=[task.id],
|
||||
queue="gen_chatapi_create",
|
||||
countdown=0,
|
||||
)
|
||||
return "recover_create_missing_provider_id"
|
||||
|
||||
elif task.pipeline_stage == "result_ready":
|
||||
if task.remote_result_url:
|
||||
await enqueue_download_task(
|
||||
db,
|
||||
task,
|
||||
recover=True,
|
||||
reason="startup_generation_result_ready",
|
||||
)
|
||||
action = "recover_result_ready"
|
||||
else:
|
||||
action = "skip_result_ready_no_url"
|
||||
if task.pipeline_stage == "result_ready":
|
||||
await _remove_poll_active(task.id)
|
||||
if task.remote_result_url:
|
||||
await enqueue_download_task(
|
||||
db,
|
||||
task,
|
||||
recover=True,
|
||||
reason=f"{source}_generation_result_ready",
|
||||
)
|
||||
return "recover_result_ready"
|
||||
return "skip_result_ready_no_url"
|
||||
|
||||
return f"skip_stage_{task.pipeline_stage}"
|
||||
|
||||
|
||||
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||
"""启动时生成链路容灾扫描。
|
||||
|
||||
不新增 Celery beat，不新增 worker 命令；worker 启动时由 Redis 锁保证只投递一次。
|
||||
恢复顺序：
|
||||
1. Redis poll active_index 到期任务；
|
||||
2. DB fallback 扫描 queued/creating/waiting_remote/polling/result_ready；
|
||||
3. 下载阶段仍由 recover_download_tasks_once 兜底。
|
||||
"""
|
||||
checked_ids: set[str] = set()
|
||||
results: dict[str, int] = {}
|
||||
|
||||
due_poll_ids = await redis_get_due_registry_ids(
|
||||
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||
limit=int(settings.POLL_RECOVERY_BATCH_SIZE or settings.GENERATION_RECOVERY_BATCH_SIZE or 100),
|
||||
log_context="poll_active",
|
||||
)
|
||||
poll_payloads = await redis_get_registry_payloads(
|
||||
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
||||
item_ids=due_poll_ids,
|
||||
log_context="poll_active",
|
||||
)
|
||||
|
||||
for task_id in due_poll_ids:
|
||||
result = await db.execute(
|
||||
select(ChatGenerationTask)
|
||||
.where(
|
||||
ChatGenerationTask.id == task_id,
|
||||
ChatGenerationTask.deleted_at.is_(None),
|
||||
)
|
||||
.with_for_update()
|
||||
.limit(1)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
if task is None:
|
||||
await _remove_poll_active(task_id)
|
||||
action = "clean_missing_poll_task"
|
||||
else:
|
||||
action = f"skip_stage_{task.pipeline_stage}"
|
||||
|
||||
checked_ids.add(task.id)
|
||||
action = await recover_one_generation_task(
|
||||
db,
|
||||
task,
|
||||
payload=poll_payloads.get(task_id),
|
||||
source="startup_poll_redis",
|
||||
)
|
||||
results[action] = results.get(action, 0) + 1
|
||||
|
||||
batch_size = int(settings.GENERATION_RECOVERY_BATCH_SIZE or settings.DOWNLOAD_RECOVERY_BATCH_SIZE or 100)
|
||||
max_rounds = max(1, int(settings.GENERATION_RECOVERY_MAX_ROUNDS or 1))
|
||||
total_db_checked = 0
|
||||
|
||||
for _round in range(max_rounds):
|
||||
query_result = await db.execute(
|
||||
select(ChatGenerationTask)
|
||||
.where(
|
||||
ChatGenerationTask.deleted_at.is_(None),
|
||||
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate", "shot_replicate"]),
|
||||
ChatGenerationTask.status == "generating",
|
||||
ChatGenerationTask.pipeline_stage.in_(
|
||||
[
|
||||
"queued",
|
||||
"preparing",
|
||||
"creating_provider_task",
|
||||
"waiting_remote",
|
||||
"polling",
|
||||
"result_ready",
|
||||
]
|
||||
),
|
||||
)
|
||||
.order_by(ChatGenerationTask.updated_at.asc())
|
||||
.limit(batch_size)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
tasks = query_result.scalars().all()
|
||||
if not tasks:
|
||||
break
|
||||
|
||||
progressed_this_round = 0
|
||||
for task in tasks:
|
||||
if task.id in checked_ids:
|
||||
continue
|
||||
action = await recover_one_generation_task(
|
||||
db,
|
||||
task,
|
||||
payload=None,
|
||||
source="startup_db",
|
||||
)
|
||||
results[action] = results.get(action, 0) + 1
|
||||
checked_ids.add(task.id)
|
||||
total_db_checked += 1
|
||||
progressed_this_round += 1
|
||||
|
||||
if len(tasks) < batch_size or progressed_this_round <= 0:
|
||||
break
|
||||
|
||||
# 下载阶段单独跑 DB fallback。
|
||||
download_result = await recover_download_tasks_once(db)
|
||||
return {
|
||||
"checked": len(tasks),
|
||||
"checked": len(checked_ids),
|
||||
"db_checked": total_db_checked,
|
||||
"results": results,
|
||||
"download_recovery": download_result,
|
||||
}
|
||||
|
||||
@@ -180,7 +180,7 @@ async def mark_chat_generation_task_failed_and_refund_once(
|
||||
select(ChatGenerationTask)
|
||||
.where(
|
||||
ChatGenerationTask.id == task_id,
|
||||
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate"]),
|
||||
ChatGenerationTask.generation_mode.in_(["chatapi_async", "hot_opening_replicate", "shot_replicate"]),
|
||||
ChatGenerationTask.deleted_at.is_(None),
|
||||
)
|
||||
.with_for_update()
|
||||
|
||||
@@ -44,7 +44,7 @@ from app.services.generation_billing_service import charge_module_prompt_usage
|
||||
from app.services.generation_refund_service import mark_chat_generation_task_failed_and_refund_once
|
||||
from app.services.generation_task_factory_service import create_chat_generation_task_for_module
|
||||
from app.services.hot_opening_video_prompt_service import build_final_video_prompt, optimize_hot_opening_video_prompt, patch_video_prompt_schema_from_client
|
||||
from app.services.module_generation_log_service import log_module_event_file, log_module_prompt_event
|
||||
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
|
||||
from app.services.llm import optimize_prompt
|
||||
from app.services.resource_accounting_service import soft_delete_chat_task_resources
|
||||
from app.services.resource_signed_url_service import build_resource_signed_url
|
||||
@@ -206,6 +206,28 @@ async def log_module_event(
|
||||
)
|
||||
|
||||
|
||||
|
||||
def _log_project_error(
|
||||
*,
|
||||
project: ModuleGenerationProject | None,
|
||||
event_type: str,
|
||||
message: str,
|
||||
exc: BaseException | None = None,
|
||||
step: ModuleGenerationStep | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
log_module_error(
|
||||
module=(project.module if project else MODULE),
|
||||
event_type=event_type,
|
||||
project_id=(project.id if project else None),
|
||||
step_id=(step.id if step else None),
|
||||
user_id=(project.user_id if project else None),
|
||||
message=message,
|
||||
detail=detail,
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
|
||||
async def _get_project_for_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
@@ -1039,6 +1061,17 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
||||
step.completed_at = _now()
|
||||
project.status = ModuleProjectStatusEnum.FAILED.value
|
||||
project.error_message = f"图片 AI 提词生成失败: {exc}"
|
||||
log_module_prompt_event(
|
||||
event_type="module_prompt_error",
|
||||
project_id=project.id,
|
||||
step_id=step.id,
|
||||
user_id=project.user_id,
|
||||
module=project.module,
|
||||
prompt_type=ModulePromptTypeEnum.IMAGE_PROMPT.value,
|
||||
request=locals().get("request_log", {}),
|
||||
error=str(exc),
|
||||
)
|
||||
_log_project_error(project=project, step=step, event_type="IMAGE_PROMPT_FAILED", message=project.error_message, exc=exc)
|
||||
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.IMAGE_PROMPT_FAILED.value, message=project.error_message)
|
||||
return step
|
||||
|
||||
@@ -1317,6 +1350,17 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
||||
step.completed_at = _now()
|
||||
project.status = ModuleProjectStatusEnum.FAILED.value
|
||||
project.error_message = f"视频 AI 提词生成失败: {exc}"
|
||||
log_module_prompt_event(
|
||||
event_type="module_prompt_error",
|
||||
project_id=project.id,
|
||||
step_id=step.id,
|
||||
user_id=project.user_id,
|
||||
module=project.module,
|
||||
prompt_type=ModulePromptTypeEnum.VIDEO_PROMPT.value,
|
||||
request=locals().get("request_log", {}),
|
||||
error=str(exc),
|
||||
)
|
||||
_log_project_error(project=project, step=step, event_type="VIDEO_PROMPT_FAILED", message=project.error_message, exc=exc)
|
||||
await log_module_event(db, project=project, step=step, event_type=ModuleEventTypeEnum.VIDEO_PROMPT_FAILED.value, message=project.error_message)
|
||||
return step
|
||||
|
||||
@@ -1517,6 +1561,16 @@ async def mark_hot_opening_step_dispatch_failed(
|
||||
step.completed_at = _now()
|
||||
project.status = ModuleProjectStatusEnum.FAILED.value
|
||||
project.error_message = error_message
|
||||
log_module_error(
|
||||
module=project.module,
|
||||
event_type="CELERY_DISPATCH_FAILED",
|
||||
project_id=project.id,
|
||||
step_id=step.id,
|
||||
user_id=project.user_id,
|
||||
message=error_message,
|
||||
detail={"reason": "celery_dispatch_failed", "chat_task_id": step.chat_task_id},
|
||||
error=error_message,
|
||||
)
|
||||
await log_module_event(
|
||||
db,
|
||||
project=project,
|
||||
|
||||
@@ -628,7 +628,7 @@ async def optimize_hot_opening_video_prompt(
|
||||
) -> tuple[dict[str, Any], str, dict[str, Any]]:
|
||||
duration = int(video_config["duration"])
|
||||
references = [
|
||||
{"type": "video", "url": material_video_url},
|
||||
{"type": "video", "url": _build_file_url_or_data_uri(material_video_url)},
|
||||
{"type": "image", "url": _build_file_url_or_data_uri(generated_image_url)},
|
||||
]
|
||||
client_schema = build_dynamic_schema(video_config)
|
||||
|
||||
@@ -3,12 +3,14 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from app.services.log_config import LOG_DATE_FORMAT, LOG_DIR, is_enabled
|
||||
|
||||
MAX_LOG_FIELD_LENGTH = 20000
|
||||
MAX_TRACEBACK_LENGTH = 12000
|
||||
MODULE_LOG_ROOT = os.path.join(os.path.dirname(LOG_DIR), "ModuleGeneration")
|
||||
|
||||
|
||||
@@ -33,6 +35,23 @@ def _safe_dump_value(value: Any) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def build_exception_detail(exc: BaseException | None, extra: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""构造统一异常日志 detail。日志方法必须吞异常，业务不能被日志影响。"""
|
||||
detail: dict[str, Any] = dict(extra or {})
|
||||
if exc is not None:
|
||||
tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
||||
if len(tb) > MAX_TRACEBACK_LENGTH:
|
||||
tb = tb[:MAX_TRACEBACK_LENGTH] + f"...<traceback_truncated:{len(tb) - MAX_TRACEBACK_LENGTH}>"
|
||||
detail.update(
|
||||
{
|
||||
"exception_type": type(exc).__name__,
|
||||
"exception_message": str(exc),
|
||||
"traceback": tb,
|
||||
}
|
||||
)
|
||||
return detail
|
||||
|
||||
|
||||
def _append_module_log(module: str, entry: dict[str, Any]) -> None:
|
||||
if not is_enabled():
|
||||
return
|
||||
@@ -92,10 +111,7 @@ def log_module_prompt_event(
|
||||
token_usage: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""记录模块 AI 提词请求/响应到 JSONL 文件。
|
||||
|
||||
与模块事件共用同一个服务，但按 module 分目录，方便按模块排查。
|
||||
"""
|
||||
"""记录模块 AI 提词/分析请求和响应到 JSONL 文件。"""
|
||||
entry = {
|
||||
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"log_type": "module_prompt",
|
||||
@@ -123,8 +139,15 @@ def log_module_error(
|
||||
message: str | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
exc: BaseException | None = None,
|
||||
) -> None:
|
||||
"""记录模块异常日志。"""
|
||||
"""记录模块异常日志。
|
||||
|
||||
- 兼容原有 detail/error 参数。
|
||||
- 新增 exc 后自动记录 exception_type、message、traceback。
|
||||
- 日志写入失败会被底层吞掉，不影响主流程。
|
||||
"""
|
||||
merged_detail = build_exception_detail(exc, detail)
|
||||
entry = {
|
||||
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"log_type": "module_error",
|
||||
@@ -134,7 +157,7 @@ def log_module_error(
|
||||
"step_id": step_id,
|
||||
"user_id": user_id,
|
||||
"message": message,
|
||||
"detail": _safe_dump_value(detail or {}),
|
||||
"error": error,
|
||||
"detail": _safe_dump_value(merged_detail),
|
||||
"error": error if error is not None else (str(exc) if exc is not None else None),
|
||||
}
|
||||
_append_module_log(module, entry)
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
# app/services/redis_registry_service.py
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||
|
||||
from app.config import settings
|
||||
|
||||
try:
|
||||
from redis.exceptions import RedisError
|
||||
except ImportError: # pragma: no cover - redis 未安装时降级
|
||||
RedisError = RuntimeError # type: ignore[assignment]
|
||||
|
||||
|
||||
logger = logging.getLogger("video_gen")
|
||||
|
||||
_redis_clients: Dict[tuple[int, int, int], Any] = {}
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ensure_aware_utc(value: Optional[datetime]) -> Optional[datetime]:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def datetime_to_epoch(value: Optional[datetime]) -> int:
|
||||
checked_value = ensure_aware_utc(value) or utc_now()
|
||||
return int(checked_value.timestamp())
|
||||
|
||||
|
||||
def normalize_registry_score(value: Optional[Union[datetime, int, float]]) -> int:
|
||||
if isinstance(value, datetime):
|
||||
return datetime_to_epoch(value)
|
||||
if value is None:
|
||||
return datetime_to_epoch(utc_now())
|
||||
return int(float(value))
|
||||
|
||||
|
||||
def registry_redis_url() -> str:
|
||||
"""Celery 容灾注册表统一使用 Celery broker Redis。
|
||||
|
||||
不能改成只读 settings.REDIS_URL，否则线上 CELERY_BROKER_URL 使用独立
|
||||
Redis DB 时，旧下载 active 注册表会被写到另一个库，导致恢复扫描失效。
|
||||
"""
|
||||
return settings.CELERY_BROKER_URL or settings.REDIS_URL or ""
|
||||
|
||||
|
||||
def _is_supported_redis_url(redis_url: str) -> bool:
|
||||
if not redis_url:
|
||||
return False
|
||||
lowered = redis_url.lower()
|
||||
return lowered.startswith(("redis://", "rediss://", "unix://"))
|
||||
|
||||
|
||||
async def get_registry_redis() -> Optional[Any]:
|
||||
"""获取 Celery 容灾 Redis 连接。
|
||||
|
||||
重点：redis.asyncio 的连接/连接池绑定 event loop，不能跨 loop 复用。
|
||||
Celery -P threads 或 worker_ready + task 线程混用时，如果使用单个全局
|
||||
Redis 客户端，会触发 got Future attached to a different loop。
|
||||
|
||||
因此这里按 pid + thread_id + event_loop_id 缓存客户端，确保同一个客户端
|
||||
只在创建它的事件循环里使用。Redis 不可用时返回 None，调用方降级为
|
||||
DB fallback，不能影响生成主链路。
|
||||
"""
|
||||
redis_url = registry_redis_url()
|
||||
if not _is_supported_redis_url(redis_url):
|
||||
if redis_url:
|
||||
logger.warning(
|
||||
"Celery 容灾 Redis 注册表仅支持 redis/rediss/unix URL，当前 broker 不是 Redis，降级为 DB 容灾。url=%s",
|
||||
redis_url,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
from redis.asyncio import Redis
|
||||
except ImportError as exc:
|
||||
logger.warning(
|
||||
"Celery 容灾 Redis 注册表不可用，redis 依赖未安装。error=%s",
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
client_key = (os.getpid(), threading.get_ident(), id(loop))
|
||||
cached = _redis_clients.get(client_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
try:
|
||||
redis_client = Redis.from_url(redis_url, decode_responses=True)
|
||||
await redis_client.ping()
|
||||
_redis_clients[client_key] = redis_client
|
||||
return redis_client
|
||||
except (RedisError, OSError, RuntimeError) as exc:
|
||||
logger.warning(
|
||||
"Celery 容灾 Redis 注册表不可用，降级为仅 DB 容灾。error=%s",
|
||||
exc,
|
||||
)
|
||||
_redis_clients.pop(client_key, None)
|
||||
return None
|
||||
|
||||
|
||||
async def close_registry_redis() -> None:
|
||||
"""关闭当前进程内已缓存的 Redis 注册表连接。
|
||||
|
||||
关闭动作尽量只关闭当前 event loop 对应的客户端；如果调用方处于进程
|
||||
退出阶段，则逐个尝试关闭，失败忽略，避免影响 worker 退出。
|
||||
"""
|
||||
if not _redis_clients:
|
||||
return
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
current_key = (os.getpid(), threading.get_ident(), id(loop))
|
||||
items = [(current_key, _redis_clients.pop(current_key, None))]
|
||||
except RuntimeError:
|
||||
items = list(_redis_clients.items())
|
||||
_redis_clients.clear()
|
||||
|
||||
for _, client in items:
|
||||
if client is None:
|
||||
continue
|
||||
try:
|
||||
close_method = getattr(client, "close", None) or getattr(client, "aclose", None)
|
||||
if close_method is None:
|
||||
continue
|
||||
close_result = close_method()
|
||||
if inspect.isawaitable(close_result):
|
||||
await close_result
|
||||
except (RedisError, OSError, RuntimeError) as exc:
|
||||
logger.debug("关闭 Celery 容灾 Redis 注册表连接失败。error=%s", exc)
|
||||
|
||||
|
||||
async def redis_upsert_registry_item(
|
||||
*,
|
||||
hash_key: str,
|
||||
zset_key: str,
|
||||
item_id: str,
|
||||
payload: Dict[str, Any],
|
||||
check_at: Optional[Union[datetime, int, float]],
|
||||
log_context: str = "registry",
|
||||
) -> None:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return
|
||||
|
||||
score = normalize_registry_score(check_at)
|
||||
updated_payload = dict(payload)
|
||||
updated_payload["check_at"] = score
|
||||
updated_payload["updated_at"] = updated_payload.get("updated_at") or datetime_to_epoch(utc_now())
|
||||
|
||||
try:
|
||||
pipe: Any = redis.pipeline(transaction=True)
|
||||
pipe.hset(hash_key, item_id, json.dumps(updated_payload, ensure_ascii=False, default=str))
|
||||
pipe.zadd(zset_key, {item_id: score})
|
||||
await pipe.execute()
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"写入 Redis 注册表失败。context=%s, item_id=%s, error=%s",
|
||||
log_context,
|
||||
item_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
async def redis_remove_registry_item(
|
||||
*,
|
||||
hash_key: str,
|
||||
zset_key: str,
|
||||
item_id: str,
|
||||
log_context: str = "registry",
|
||||
) -> None:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return
|
||||
|
||||
try:
|
||||
pipe: Any = redis.pipeline(transaction=True)
|
||||
pipe.hdel(hash_key, item_id)
|
||||
pipe.zrem(zset_key, item_id)
|
||||
await pipe.execute()
|
||||
except (RedisError, OSError, RuntimeError) as exc:
|
||||
logger.warning(
|
||||
"删除 Redis 注册表失败。context=%s, item_id=%s, error=%s",
|
||||
log_context,
|
||||
item_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
async def redis_get_due_registry_ids(
|
||||
*,
|
||||
zset_key: str,
|
||||
limit: Optional[int] = None,
|
||||
now: Optional[datetime] = None,
|
||||
log_context: str = "registry",
|
||||
) -> List[str]:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return []
|
||||
|
||||
batch_limit = int(limit or 100)
|
||||
score = datetime_to_epoch(now or utc_now())
|
||||
|
||||
try:
|
||||
result = await redis.zrangebyscore(
|
||||
zset_key,
|
||||
min="-inf",
|
||||
max=score,
|
||||
start=0,
|
||||
num=batch_limit,
|
||||
)
|
||||
return [str(item) for item in result]
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning("扫描 Redis ZSet 失败。context=%s, error=%s", log_context, exc)
|
||||
return []
|
||||
|
||||
|
||||
async def redis_get_registry_payloads(
|
||||
*,
|
||||
hash_key: str,
|
||||
item_ids: Iterable[str],
|
||||
log_context: str = "registry",
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
cleaned_item_ids = [str(item) for item in item_ids if item]
|
||||
if not cleaned_item_ids:
|
||||
return {}
|
||||
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return {}
|
||||
|
||||
try:
|
||||
raw_values = await redis.hmget(hash_key, cleaned_item_ids)
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning("读取 Redis Hash 失败。context=%s, error=%s", log_context, exc)
|
||||
return {}
|
||||
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
for item_id, raw in zip(cleaned_item_ids, raw_values):
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
result[item_id] = value
|
||||
return result
|
||||
|
||||
|
||||
async def redis_postpone_registry_item(
|
||||
*,
|
||||
hash_key: str,
|
||||
zset_key: str,
|
||||
item_id: str,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
check_at: Optional[Union[datetime, int, float]] = None,
|
||||
log_context: str = "registry",
|
||||
) -> None:
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return
|
||||
|
||||
score = normalize_registry_score(check_at)
|
||||
|
||||
try:
|
||||
pipe: Any = redis.pipeline(transaction=True)
|
||||
pipe.zadd(zset_key, {item_id: score})
|
||||
|
||||
if payload is not None:
|
||||
updated_payload = dict(payload)
|
||||
updated_payload["check_at"] = score
|
||||
updated_payload["updated_at"] = datetime_to_epoch(utc_now())
|
||||
pipe.hset(hash_key, item_id, json.dumps(updated_payload, ensure_ascii=False, default=str))
|
||||
|
||||
await pipe.execute()
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"刷新 Redis 注册表检查时间失败。context=%s, item_id=%s, error=%s",
|
||||
log_context,
|
||||
item_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
async def redis_acquire_lock(
|
||||
*,
|
||||
lock_key: str,
|
||||
ttl_seconds: int,
|
||||
token: Optional[str] = None,
|
||||
log_context: str = "lock",
|
||||
) -> Optional[str]:
|
||||
"""尝试获取 Redis 分布式锁。
|
||||
|
||||
返回 token 表示抢锁成功；返回 None 表示 Redis 不可用或锁已被其他 worker 持有。
|
||||
"""
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return None
|
||||
|
||||
lock_token = token or uuid.uuid4().hex
|
||||
ttl = max(1, int(ttl_seconds or 60))
|
||||
|
||||
try:
|
||||
acquired = await redis.set(lock_key, lock_token, nx=True, ex=ttl)
|
||||
return lock_token if acquired else None
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning("获取 Redis 锁失败。context=%s, lock_key=%s, error=%s", log_context, lock_key, exc)
|
||||
return None
|
||||
|
||||
|
||||
async def redis_release_lock(
|
||||
*,
|
||||
lock_key: str,
|
||||
token: str,
|
||||
log_context: str = "lock",
|
||||
) -> bool:
|
||||
"""只释放 token 匹配的锁，避免误删其他 worker 新抢到的锁。"""
|
||||
redis = await get_registry_redis()
|
||||
if redis is None:
|
||||
return False
|
||||
|
||||
script = """
|
||||
if redis.call('get', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('del', KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
"""
|
||||
try:
|
||||
released = await redis.eval(script, 1, lock_key, token)
|
||||
return bool(released)
|
||||
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
logger.warning("释放 Redis 锁失败。context=%s, lock_key=%s, error=%s", log_context, lock_key, exc)
|
||||
return False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.enums.shot_replicate import ShotSplitStatusEnum
|
||||
from app.models.shot_replicate_segment import ShotReplicateSegment
|
||||
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
||||
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
|
||||
from app.tasks.celery_app import celery_app
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _ensure_aware(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _expired(value: datetime | None, now: datetime | None = None) -> bool:
|
||||
checked = _ensure_aware(value)
|
||||
if checked is None:
|
||||
return True
|
||||
return checked <= (now or _now())
|
||||
|
||||
|
||||
def _queue_timeout(segment: ShotReplicateSegment, now: datetime | None = None) -> bool:
|
||||
enqueued_at = _ensure_aware(segment.split_enqueued_at)
|
||||
if enqueued_at is None:
|
||||
return True
|
||||
return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now())
|
||||
|
||||
|
||||
async def recover_one_split_segment(db: AsyncSession, segment: ShotReplicateSegment, *, source: str = "startup_db") -> str:
|
||||
from app.tasks.shot_replicate_tasks import split_one_segment
|
||||
|
||||
if not segment:
|
||||
return "skip_missing_segment"
|
||||
if segment.deleted_at is not None:
|
||||
return "skip_deleted"
|
||||
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value:
|
||||
return "skip_completed"
|
||||
if segment.split_status == ShotSplitStatusEnum.FAILED.value:
|
||||
return "skip_failed"
|
||||
|
||||
current_time = _now()
|
||||
should_recover = False
|
||||
|
||||
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
|
||||
should_recover = _queue_timeout(segment, current_time)
|
||||
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
|
||||
should_recover = _expired(segment.split_lease_until, current_time)
|
||||
elif segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
|
||||
should_recover = _expired(segment.split_next_retry_at, current_time)
|
||||
|
||||
if not should_recover:
|
||||
return f"skip_{segment.split_status}_not_due"
|
||||
|
||||
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
|
||||
segment.split_status = ShotSplitStatusEnum.FAILED.value
|
||||
segment.split_last_error = segment.split_last_error or f"{source} 恢复时超过最大重试次数"
|
||||
segment.split_lease_until = None
|
||||
segment.split_next_retry_at = None
|
||||
await refresh_task_set_split_summary(db, segment.task_set_id)
|
||||
await db.commit()
|
||||
return "mark_failed_max_retry"
|
||||
|
||||
segment.split_status = ShotSplitStatusEnum.PENDING.value
|
||||
segment.split_enqueued_at = current_time
|
||||
segment.split_lease_until = None
|
||||
segment.split_next_retry_at = None
|
||||
await refresh_task_set_split_summary(db, segment.task_set_id)
|
||||
await db.commit()
|
||||
|
||||
if celery_app:
|
||||
split_one_segment.apply_async(
|
||||
args=[segment.id],
|
||||
queue="gen_result_download",
|
||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||
countdown=0,
|
||||
)
|
||||
return f"recover_{source}"
|
||||
|
||||
|
||||
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||
"""拆镜 ffmpeg 任务容灾恢复。独立扫描 shot_replicate_segments，不复用 Chat 下载 active registry。"""
|
||||
batch_size = int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50)
|
||||
result = await db.execute(
|
||||
select(ShotReplicateSegment)
|
||||
.where(
|
||||
ShotReplicateSegment.deleted_at.is_(None),
|
||||
ShotReplicateSegment.split_status.in_(
|
||||
[
|
||||
ShotSplitStatusEnum.PENDING.value,
|
||||
ShotSplitStatusEnum.PROCESSING.value,
|
||||
ShotSplitStatusEnum.RETRY_WAITING.value,
|
||||
]
|
||||
),
|
||||
)
|
||||
.order_by(ShotReplicateSegment.updated_at.asc())
|
||||
.limit(batch_size)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
segments = list(result.scalars().all())
|
||||
|
||||
checked = 0
|
||||
results: dict[str, int] = {}
|
||||
touched_task_set_ids: set[str] = set()
|
||||
for segment in segments:
|
||||
action = await recover_one_split_segment(db, segment, source="startup_db")
|
||||
checked += 1
|
||||
touched_task_set_ids.add(segment.task_set_id)
|
||||
results[action] = results.get(action, 0) + 1
|
||||
|
||||
for task_set_id in touched_task_set_ids:
|
||||
await refresh_task_set_split_summary(db, task_set_id)
|
||||
await db.commit()
|
||||
|
||||
return {"checked": checked, "results": results}
|
||||
@@ -0,0 +1,529 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.config import settings
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.enums.shot_replicate import (
|
||||
ModuleCodeEnum,
|
||||
ShotAnalysisStatusEnum,
|
||||
ShotSegmentAnalysisStatusEnum,
|
||||
ShotSegmentReplicateStatusEnum,
|
||||
ShotSegmentSourceModeEnum,
|
||||
ShotSplitStatusEnum,
|
||||
ShotTaskSetStatusEnum,
|
||||
)
|
||||
from app.models.shot_replicate_segment import ShotReplicateSegment
|
||||
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
||||
from app.models.user import User
|
||||
from app.schemas.shot_replicate import (
|
||||
ShotAISuggestionOut,
|
||||
ShotSegmentDetailOut,
|
||||
ShotSegmentListOut,
|
||||
ShotSegmentOut,
|
||||
ShotSplitByAIOut,
|
||||
ShotSplitByAIRequest,
|
||||
ShotSplitCustomOut,
|
||||
ShotSplitCustomRequest,
|
||||
ShotTaskSetCreate,
|
||||
ShotTaskSetDetailOut,
|
||||
ShotTaskSetListOut,
|
||||
ShotTaskSetOut,
|
||||
)
|
||||
from app.services.module_generation_log_service import log_module_event_file
|
||||
from app.services.upload_video_asset_service import (
|
||||
build_time_node,
|
||||
validate_split_range,
|
||||
validate_upload_video_asset,
|
||||
)
|
||||
from app.tasks.celery_app import celery_app
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
|
||||
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _normalize_suggestions(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value, start=1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
start = item.get("拆镜开始秒")
|
||||
end = item.get("拆镜结束秒")
|
||||
try:
|
||||
start_f = float(start)
|
||||
end_f = float(end)
|
||||
except Exception:
|
||||
continue
|
||||
if start_f < 0 or end_f <= start_f:
|
||||
continue
|
||||
normalized.append(
|
||||
{
|
||||
"index": idx,
|
||||
"start_second": start_f,
|
||||
"end_second": end_f,
|
||||
"duration_seconds": round(end_f - start_f, 3),
|
||||
"time_node": str(item.get("拆镜时间节点") or build_time_node(start_f, end_f)),
|
||||
"content": str(item.get("对应时间节点内的内容") or "无"),
|
||||
"category": str(item.get("分类") or "无"),
|
||||
"audience": str(item.get("受众人群") or "无"),
|
||||
"raw": item,
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _task_set_to_out(task_set: ShotReplicateTaskSet) -> ShotTaskSetOut:
|
||||
return ShotTaskSetOut.model_validate(task_set)
|
||||
|
||||
|
||||
def _task_set_to_detail_out(task_set: ShotReplicateTaskSet) -> ShotTaskSetDetailOut:
|
||||
suggestions = [ShotAISuggestionOut(**{k: v for k, v in item.items() if k != "raw"}) for item in _normalize_suggestions(task_set.ai_suggestion_json)]
|
||||
base = ShotTaskSetDetailOut.model_validate(task_set)
|
||||
base.ai_suggestions = suggestions
|
||||
return base
|
||||
|
||||
|
||||
def _segment_to_out(segment: ShotReplicateSegment) -> ShotSegmentOut:
|
||||
data = ShotSegmentOut.model_validate(segment)
|
||||
data.segment_name = f"片段{segment.segment_index}"
|
||||
return data
|
||||
|
||||
|
||||
def _segment_to_detail_out(segment: ShotReplicateSegment) -> ShotSegmentDetailOut:
|
||||
data = ShotSegmentDetailOut.model_validate(segment)
|
||||
data.segment_name = f"片段{segment.segment_index}"
|
||||
return data
|
||||
|
||||
|
||||
async def get_task_set_for_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
task_set_id: str,
|
||||
user: User,
|
||||
for_update: bool = False,
|
||||
) -> ShotReplicateTaskSet:
|
||||
query = select(ShotReplicateTaskSet).where(
|
||||
ShotReplicateTaskSet.id == task_set_id,
|
||||
ShotReplicateTaskSet.deleted_at.is_(None),
|
||||
)
|
||||
if not user.is_admin:
|
||||
query = query.where(ShotReplicateTaskSet.user_id == user.id)
|
||||
if for_update:
|
||||
query = query.with_for_update()
|
||||
result = await db.execute(query.limit(1))
|
||||
task_set = result.scalar_one_or_none()
|
||||
if not task_set:
|
||||
raise HTTPException(status_code=404, detail="拆镜总任务集不存在")
|
||||
return task_set
|
||||
|
||||
|
||||
async def get_segment_for_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
segment_id: str,
|
||||
user: User,
|
||||
for_update: bool = False,
|
||||
) -> ShotReplicateSegment:
|
||||
query = select(ShotReplicateSegment).where(
|
||||
ShotReplicateSegment.id == segment_id,
|
||||
ShotReplicateSegment.deleted_at.is_(None),
|
||||
)
|
||||
if not user.is_admin:
|
||||
query = query.where(ShotReplicateSegment.user_id == user.id)
|
||||
if for_update:
|
||||
query = query.with_for_update()
|
||||
result = await db.execute(query.limit(1))
|
||||
segment = result.scalar_one_or_none()
|
||||
if not segment:
|
||||
raise HTTPException(status_code=404, detail="拆镜片段不存在")
|
||||
return segment
|
||||
|
||||
|
||||
async def create_task_set(db: AsyncSession, *, current_user: User, req: ShotTaskSetCreate) -> ShotReplicateTaskSet:
|
||||
if req.idempotency_key:
|
||||
existing_result = await db.execute(
|
||||
select(ShotReplicateTaskSet).where(
|
||||
ShotReplicateTaskSet.user_id == current_user.id,
|
||||
ShotReplicateTaskSet.idempotency_key == req.idempotency_key,
|
||||
ShotReplicateTaskSet.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
existing = existing_result.scalar_one_or_none()
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
asset = validate_upload_video_asset(req.video_url, req.video_duration_seconds)
|
||||
task_set = ShotReplicateTaskSet(
|
||||
id=generate_id(),
|
||||
user_id=current_user.id,
|
||||
title=req.title or "拆镜复刻任务",
|
||||
video_url=asset.url,
|
||||
video_path=str(asset.path),
|
||||
video_duration_seconds=asset.duration_seconds,
|
||||
status=ShotTaskSetStatusEnum.PENDING_ANALYSIS.value,
|
||||
analysis_status=ShotAnalysisStatusEnum.PENDING.value,
|
||||
split_status=ShotSplitStatusEnum.NONE.value,
|
||||
segment_count=0,
|
||||
completed_segment_count=0,
|
||||
failed_segment_count=0,
|
||||
idempotency_key=req.idempotency_key,
|
||||
)
|
||||
db.add(task_set)
|
||||
await db.flush()
|
||||
log_module_event_file(
|
||||
module=MODULE,
|
||||
event_type="SHOT_TASK_SET_CREATED",
|
||||
project_id=task_set.id,
|
||||
user_id=task_set.user_id,
|
||||
message="创建拆镜总任务集",
|
||||
detail={
|
||||
"task_set_id": task_set.id,
|
||||
"title": task_set.title,
|
||||
"video_url": task_set.video_url,
|
||||
"video_path": task_set.video_path,
|
||||
"video_duration_seconds": task_set.video_duration_seconds,
|
||||
"idempotency_key": task_set.idempotency_key,
|
||||
},
|
||||
)
|
||||
return task_set
|
||||
|
||||
|
||||
async def list_task_sets(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
status: str | None = None,
|
||||
analysis_status: str | None = None,
|
||||
split_status: str | None = None,
|
||||
keyword: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> ShotTaskSetListOut:
|
||||
query = select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.deleted_at.is_(None))
|
||||
if not current_user.is_admin:
|
||||
query = query.where(ShotReplicateTaskSet.user_id == current_user.id)
|
||||
if status:
|
||||
query = query.where(ShotReplicateTaskSet.status == status)
|
||||
if analysis_status:
|
||||
query = query.where(ShotReplicateTaskSet.analysis_status == analysis_status)
|
||||
if split_status:
|
||||
query = query.where(ShotReplicateTaskSet.split_status == split_status)
|
||||
if keyword:
|
||||
like = f"%{keyword.strip()}%"
|
||||
query = query.where(
|
||||
(ShotReplicateTaskSet.title.ilike(like))
|
||||
| (ShotReplicateTaskSet.original_video_content.ilike(like))
|
||||
| (ShotReplicateTaskSet.original_video_category.ilike(like))
|
||||
)
|
||||
|
||||
total_result = await db.execute(select(func.count()).select_from(query.subquery()))
|
||||
total = int(total_result.scalar() or 0)
|
||||
rows = await db.execute(
|
||||
query.order_by(ShotReplicateTaskSet.created_at.desc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
)
|
||||
return ShotTaskSetListOut(total=total, page=page, page_size=page_size, items=[_task_set_to_out(item) for item in rows.scalars().all()])
|
||||
|
||||
|
||||
async def task_set_detail(db: AsyncSession, *, current_user: User, task_set_id: str) -> ShotTaskSetDetailOut:
|
||||
task_set = await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user)
|
||||
return _task_set_to_detail_out(task_set)
|
||||
|
||||
|
||||
async def _next_segment_index(db: AsyncSession, task_set_id: str) -> int:
|
||||
result = await db.execute(
|
||||
select(func.max(ShotReplicateSegment.segment_index)).where(
|
||||
ShotReplicateSegment.task_set_id == task_set_id,
|
||||
ShotReplicateSegment.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return int(result.scalar() or 0) + 1
|
||||
|
||||
|
||||
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
|
||||
task_set_result = await db.execute(select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.id == task_set_id).with_for_update().limit(1))
|
||||
task_set = task_set_result.scalar_one_or_none()
|
||||
if not task_set:
|
||||
return
|
||||
|
||||
result = await db.execute(
|
||||
select(ShotReplicateSegment).where(
|
||||
ShotReplicateSegment.task_set_id == task_set_id,
|
||||
ShotReplicateSegment.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
segments = list(result.scalars().all())
|
||||
total = len(segments)
|
||||
completed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.COMPLETED.value])
|
||||
failed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.FAILED.value])
|
||||
|
||||
task_set.segment_count = total
|
||||
task_set.completed_segment_count = completed
|
||||
task_set.failed_segment_count = failed
|
||||
|
||||
if total <= 0:
|
||||
task_set.split_status = ShotSplitStatusEnum.NONE.value
|
||||
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
||||
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
|
||||
return
|
||||
|
||||
old_status = task_set.status
|
||||
old_split_status = task_set.split_status
|
||||
|
||||
if completed == total:
|
||||
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
|
||||
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
|
||||
elif failed == total:
|
||||
task_set.split_status = ShotSplitStatusEnum.FAILED.value
|
||||
task_set.status = ShotTaskSetStatusEnum.FAILED.value
|
||||
elif failed > 0:
|
||||
task_set.split_status = ShotSplitStatusEnum.FAILED.value
|
||||
task_set.status = ShotTaskSetStatusEnum.PARTIAL_FAILED.value
|
||||
else:
|
||||
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
||||
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
||||
|
||||
if old_status != task_set.status or old_split_status != task_set.split_status:
|
||||
log_module_event_file(
|
||||
module=MODULE,
|
||||
event_type="SHOT_SPLIT_STATUS_CHANGED",
|
||||
project_id=task_set.id,
|
||||
user_id=task_set.user_id,
|
||||
message="拆镜总任务集拆分状态变更",
|
||||
detail={
|
||||
"task_set_id": task_set.id,
|
||||
"from_status": old_status,
|
||||
"to_status": task_set.status,
|
||||
"from_split_status": old_split_status,
|
||||
"to_split_status": task_set.split_status,
|
||||
"segment_count": total,
|
||||
"completed_segment_count": completed,
|
||||
"failed_segment_count": failed,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def create_segments_by_ai(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
task_set_id: str,
|
||||
req: ShotSplitByAIRequest,
|
||||
) -> ShotSplitByAIOut:
|
||||
task_set = await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user, for_update=True)
|
||||
if task_set.analysis_status != ShotAnalysisStatusEnum.COMPLETED.value:
|
||||
raise HTTPException(status_code=400, detail="原视频分析未完成，不能按 AI 建议拆镜")
|
||||
|
||||
suggestions = _normalize_suggestions(task_set.ai_suggestion_json)
|
||||
if not suggestions:
|
||||
raise HTTPException(status_code=400, detail="当前没有可用 AI 建议拆镜方案，请使用自定义拆镜")
|
||||
|
||||
if req.selected_indices:
|
||||
selected_set = {int(x) for x in req.selected_indices}
|
||||
suggestions = [item for item in suggestions if int(item["index"]) in selected_set]
|
||||
if not suggestions:
|
||||
raise HTTPException(status_code=400, detail="selected_indices 没有匹配到可用 AI 建议")
|
||||
|
||||
old_result = await db.execute(
|
||||
select(ShotReplicateSegment).where(
|
||||
ShotReplicateSegment.task_set_id == task_set.id,
|
||||
ShotReplicateSegment.source_mode == ShotSegmentSourceModeEnum.AI_SUGGESTION.value,
|
||||
ShotReplicateSegment.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
old_segments = list(old_result.scalars().all())
|
||||
if old_segments and not req.replace_existing:
|
||||
raise HTTPException(status_code=409, detail="已存在 AI 建议拆镜片段，如需重拆请传 replace_existing=true")
|
||||
if old_segments and req.replace_existing:
|
||||
now = _now()
|
||||
for segment in old_segments:
|
||||
segment.deleted_at = now
|
||||
|
||||
created: list[ShotReplicateSegment] = []
|
||||
next_index = await _next_segment_index(db, task_set.id)
|
||||
for item in suggestions:
|
||||
start, end, duration = validate_split_range(
|
||||
start_second=item["start_second"],
|
||||
end_second=item["end_second"],
|
||||
video_duration_seconds=task_set.video_duration_seconds,
|
||||
)
|
||||
segment = ShotReplicateSegment(
|
||||
id=generate_id(),
|
||||
task_set_id=task_set.id,
|
||||
user_id=task_set.user_id,
|
||||
segment_index=next_index,
|
||||
source_mode=ShotSegmentSourceModeEnum.AI_SUGGESTION.value,
|
||||
start_second=start,
|
||||
end_second=end,
|
||||
duration_seconds=duration,
|
||||
time_node=build_time_node(start, end),
|
||||
split_status=ShotSplitStatusEnum.PENDING.value,
|
||||
analysis_status=ShotSegmentAnalysisStatusEnum.NOT_REQUIRED.value,
|
||||
replicate_status=ShotSegmentReplicateStatusEnum.NOT_STARTED.value,
|
||||
original_video_content=task_set.original_video_content,
|
||||
original_video_category=task_set.original_video_category,
|
||||
original_video_audience=task_set.original_video_audience,
|
||||
segment_content=item.get("content"),
|
||||
segment_category=item.get("category"),
|
||||
segment_audience=item.get("audience"),
|
||||
ai_suggestion_json=item.get("raw") or item,
|
||||
split_enqueued_at=_now(),
|
||||
split_celery_task_id=f"shot-split:{uuid.uuid4().hex}",
|
||||
)
|
||||
db.add(segment)
|
||||
created.append(segment)
|
||||
next_index += 1
|
||||
|
||||
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
||||
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
||||
await db.flush()
|
||||
await refresh_task_set_split_summary(db, task_set.id)
|
||||
await db.flush()
|
||||
|
||||
log_module_event_file(
|
||||
module=MODULE,
|
||||
event_type="SHOT_SPLIT_BY_AI_SUBMITTED",
|
||||
project_id=task_set.id,
|
||||
user_id=task_set.user_id,
|
||||
message="按 AI 建议创建拆镜片段",
|
||||
detail={
|
||||
"task_set_id": task_set.id,
|
||||
"selected_indices": req.selected_indices,
|
||||
"replace_existing": req.replace_existing,
|
||||
"created_segment_count": len(created),
|
||||
"segment_ids": [segment.id for segment in created],
|
||||
},
|
||||
)
|
||||
|
||||
return ShotSplitByAIOut(
|
||||
task_set_id=task_set.id,
|
||||
status=task_set.status,
|
||||
split_status=task_set.split_status,
|
||||
created_segment_count=len(created),
|
||||
segments=[_segment_to_out(segment) for segment in created],
|
||||
)
|
||||
|
||||
|
||||
async def create_custom_segment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
task_set_id: str,
|
||||
req: ShotSplitCustomRequest,
|
||||
) -> ShotSplitCustomOut:
|
||||
task_set = await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user, for_update=True)
|
||||
start, end, duration = validate_split_range(
|
||||
start_second=req.start_second,
|
||||
end_second=req.end_second,
|
||||
video_duration_seconds=task_set.video_duration_seconds,
|
||||
)
|
||||
next_index = await _next_segment_index(db, task_set.id)
|
||||
segment = ShotReplicateSegment(
|
||||
id=generate_id(),
|
||||
task_set_id=task_set.id,
|
||||
user_id=task_set.user_id,
|
||||
segment_index=next_index,
|
||||
source_mode=ShotSegmentSourceModeEnum.CUSTOM.value,
|
||||
start_second=start,
|
||||
end_second=end,
|
||||
duration_seconds=duration,
|
||||
time_node=build_time_node(start, end),
|
||||
split_status=ShotSplitStatusEnum.PENDING.value,
|
||||
analysis_status=ShotSegmentAnalysisStatusEnum.PENDING.value,
|
||||
replicate_status=ShotSegmentReplicateStatusEnum.NOT_STARTED.value,
|
||||
split_enqueued_at=_now(),
|
||||
split_celery_task_id=f"shot-split:{uuid.uuid4().hex}",
|
||||
)
|
||||
db.add(segment)
|
||||
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
||||
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
||||
await db.flush()
|
||||
await refresh_task_set_split_summary(db, task_set.id)
|
||||
await db.flush()
|
||||
log_module_event_file(
|
||||
module=MODULE,
|
||||
event_type="SHOT_SPLIT_CUSTOM_SUBMITTED",
|
||||
project_id=task_set.id,
|
||||
step_id=segment.id,
|
||||
user_id=task_set.user_id,
|
||||
message="按用户自定义时间创建拆镜片段",
|
||||
detail={
|
||||
"task_set_id": task_set.id,
|
||||
"segment_id": segment.id,
|
||||
"start_second": start,
|
||||
"end_second": end,
|
||||
"duration_seconds": duration,
|
||||
"time_node": segment.time_node,
|
||||
},
|
||||
)
|
||||
return ShotSplitCustomOut(task_set_id=task_set.id, segment=_segment_to_out(segment))
|
||||
|
||||
|
||||
async def enqueue_segment_split(segment_id: str, *, countdown: int | None = None, recover: bool = False) -> None:
|
||||
if not celery_app:
|
||||
return
|
||||
from app.tasks.shot_replicate_tasks import split_one_segment
|
||||
|
||||
split_one_segment.apply_async(
|
||||
args=[segment_id],
|
||||
queue="gen_result_download",
|
||||
countdown=countdown,
|
||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
|
||||
)
|
||||
|
||||
|
||||
async def list_segments(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
task_set_id: str,
|
||||
source_mode: str | None = None,
|
||||
split_status: str | None = None,
|
||||
analysis_status: str | None = None,
|
||||
replicate_status: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> ShotSegmentListOut:
|
||||
await get_task_set_for_user(db, task_set_id=task_set_id, user=current_user)
|
||||
query = select(ShotReplicateSegment).where(
|
||||
ShotReplicateSegment.task_set_id == task_set_id,
|
||||
ShotReplicateSegment.deleted_at.is_(None),
|
||||
)
|
||||
if not current_user.is_admin:
|
||||
query = query.where(ShotReplicateSegment.user_id == current_user.id)
|
||||
if source_mode:
|
||||
query = query.where(ShotReplicateSegment.source_mode == source_mode)
|
||||
if split_status:
|
||||
query = query.where(ShotReplicateSegment.split_status == split_status)
|
||||
if analysis_status:
|
||||
query = query.where(ShotReplicateSegment.analysis_status == analysis_status)
|
||||
if replicate_status:
|
||||
query = query.where(ShotReplicateSegment.replicate_status == replicate_status)
|
||||
|
||||
total_result = await db.execute(select(func.count()).select_from(query.subquery()))
|
||||
total = int(total_result.scalar() or 0)
|
||||
rows = await db.execute(
|
||||
query.order_by(ShotReplicateSegment.segment_index.asc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
)
|
||||
return ShotSegmentListOut(total=total, page=page, page_size=page_size, items=[_segment_to_out(item) for item in rows.scalars().all()])
|
||||
|
||||
|
||||
async def segment_detail(db: AsyncSession, *, current_user: User, segment_id: str) -> ShotSegmentDetailOut:
|
||||
segment = await get_segment_for_user(db, segment_id=segment_id, user=current_user)
|
||||
return _segment_to_detail_out(segment)
|
||||
@@ -0,0 +1,521 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.models.model_config import ModelConfig
|
||||
from app.models.token_usage import TokenUsage
|
||||
from app.services.upload_video_asset_service import resolve_upload_video_path
|
||||
from app.services.resource_signed_url_service import build_resource_signed_url
|
||||
from app.utils.id_gen import generate_id
|
||||
|
||||
AnalysisMode = Literal["full_breakdown", "summary_only"]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ShotVideoAnalysisResult:
|
||||
result: dict[str, Any]
|
||||
raw_response: dict[str, Any]
|
||||
usage: dict[str, Any]
|
||||
|
||||
|
||||
def _timeout_seconds() -> int:
|
||||
return int(getattr(settings, "SHOT_ANALYSIS_TIMEOUT_SECONDS", 180) or 180)
|
||||
|
||||
|
||||
def _video_fps() -> float:
|
||||
return float(getattr(settings, "SHOT_ANALYSIS_VIDEO_FPS", 1.0) or 1.0)
|
||||
|
||||
|
||||
def _split_min_seconds() -> float:
|
||||
return float(getattr(settings, "SHOT_SPLIT_MIN_SECONDS", 1) or 1)
|
||||
|
||||
|
||||
def _split_max_seconds() -> float:
|
||||
return float(getattr(settings, "SHOT_SPLIT_MAX_SECONDS", 120) or 120)
|
||||
|
||||
|
||||
def _resolve_local_file_path(file_url: str) -> str:
|
||||
if file_url.startswith("/uploads/") or file_url.startswith("uploads/"):
|
||||
return str(resolve_upload_video_path(file_url))
|
||||
return file_url
|
||||
|
||||
|
||||
def build_file_url_or_data_uri(file_url: str, fallback_mime: str = "video/mp4") -> str:
|
||||
if file_url.startswith(("http://", "https://", "data:")):
|
||||
return file_url
|
||||
file_url_sign = build_resource_signed_url(resource_url=file_url, expire_seconds=86400)
|
||||
return f"{settings.BASE_URL}{file_url_sign}"
|
||||
|
||||
# file_path = _resolve_local_file_path(file_url)
|
||||
# path = Path(file_path)
|
||||
# if not path.exists():
|
||||
# raise FileNotFoundError(f"视频文件不存在: {file_path}")
|
||||
#
|
||||
# max_mb = float(getattr(settings, "SHOT_ANALYSIS_MAX_LOCAL_VIDEO_MB", 45) or 45)
|
||||
# size_mb = path.stat().st_size / 1024 / 1024
|
||||
# if size_mb > max_mb:
|
||||
# raise ValueError(f"本地视频文件过大: {size_mb:.2f} MB，当前限制 {max_mb:g} MB")
|
||||
#
|
||||
# mime = mimetypes.guess_type(str(path))[0] or fallback_mime
|
||||
# with open(path, "rb") as f:
|
||||
# b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
# return f"data:{mime};base64,{b64}"
|
||||
|
||||
|
||||
def build_user_message(user_text: str, video_url: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
real_url = build_file_url_or_data_uri(video_url)
|
||||
content_parts = [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": real_url,
|
||||
"fps": _video_fps(),
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": user_text},
|
||||
]
|
||||
log_content_parts = [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": video_url,
|
||||
"fps": _video_fps(),
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": user_text},
|
||||
]
|
||||
return {"role": "user", "content": content_parts}, {"role": "user", "content": log_content_parts}
|
||||
|
||||
|
||||
def build_video_analysis_system_prompt(*, mode: AnalysisMode) -> str:
|
||||
if mode == "summary_only":
|
||||
return """
|
||||
你是专业的短视频内容分析师、广告素材拆解师。
|
||||
|
||||
你的任务：
|
||||
1. 根据用户提供的视频附件，分析这个视频片段的整体内容。
|
||||
2. 判断视频分类。
|
||||
3. 判断视频可能面向的受众人群。
|
||||
4. 必须输出严格 JSON 对象。
|
||||
5. 不输出 Markdown。
|
||||
6. 不输出解释文字。
|
||||
7. 不返回 null，未知内容填“无”。
|
||||
|
||||
顶级字段只能包含：
|
||||
- 原视频内容
|
||||
- 原视频分类
|
||||
- 原视频受众人群
|
||||
- 拆镜内容剖析
|
||||
|
||||
summary_only 模式下“拆镜内容剖析”必须返回空数组。
|
||||
|
||||
安全规则：
|
||||
1. 不要识别视频中人物身份。
|
||||
2. 不要猜测真实姓名、联系方式、账号身份。
|
||||
3. 如果视频是游戏录屏，只分析画面内容、玩法内容、玩家情绪表达、受众，不要编造不存在的剧情。
|
||||
4. 如果视频包含广告内容，可以分析广告品类、目标用户、转化意图，但不要编造品牌信息。
|
||||
""".strip()
|
||||
|
||||
return f"""
|
||||
你是专业的短视频内容分析师、广告素材拆解师、视频分镜分析师。
|
||||
|
||||
你的任务：
|
||||
1. 根据用户提供的视频附件，分析原视频整体内容。
|
||||
2. 判断原视频分类。
|
||||
3. 判断原视频可能面向的受众人群。
|
||||
4. 对视频进行拆镜内容剖析。
|
||||
5. 必须输出严格 JSON 对象。
|
||||
6. 不输出 Markdown。
|
||||
7. 不输出解释文字。
|
||||
8. 不返回 null，未知内容填“无”。
|
||||
|
||||
顶级字段只能包含：
|
||||
- 原视频内容
|
||||
- 原视频分类
|
||||
- 原视频受众人群
|
||||
- 拆镜内容剖析
|
||||
|
||||
拆镜内容剖析必须是数组。
|
||||
|
||||
每个拆镜片段必须包含：
|
||||
- 拆镜开始秒
|
||||
- 拆镜结束秒
|
||||
- 拆镜时间节点
|
||||
- 对应时间节点内的内容
|
||||
- 分类
|
||||
- 受众人群
|
||||
|
||||
拆镜时间规则：
|
||||
1. 拆镜开始秒必须是数字，例如 0、15、26。
|
||||
2. 拆镜结束秒必须是数字，例如 15、26、31。
|
||||
3. 拆镜时间节点必须由拆镜开始秒和拆镜结束秒组成，例如“0-15秒”。
|
||||
4. 禁止输出“-15秒”这种缺少开始秒的时间节点。
|
||||
5. 禁止输出“15-秒”这种缺少结束秒的时间节点。
|
||||
6. 每个拆镜片段时长不能低于 {_split_min_seconds():g} 秒。
|
||||
7. 每个拆镜片段时长不能高于 {_split_max_seconds():g} 秒。
|
||||
8. 如果某段内容不足 {_split_min_seconds():g} 秒，不要单独拆出来。
|
||||
9. 如果单个连续内容超过 {_split_max_seconds():g} 秒，需要按语义变化继续拆分。
|
||||
10. 如果没有明显镜头变化、场景变化、人物动作变化、剧情变化、字幕重点变化或语义变化，不要强行剖析。
|
||||
11. 如果无法可靠拆镜，则“拆镜内容剖析”返回空数组。
|
||||
12. 拆镜时间必须从 0 秒或视频中实际可识别的开始时间开始，不允许出现负数。
|
||||
13. 拆镜结束秒必须大于拆镜开始秒。
|
||||
14. 拆镜片段必须按时间顺序排列。
|
||||
|
||||
安全规则：
|
||||
1. 不要识别视频中人物身份。
|
||||
2. 不要猜测真实姓名、联系方式、账号身份。
|
||||
3. 如果视频是游戏录屏，只分析画面内容、玩法内容、玩家情绪表达、受众，不要编造不存在的剧情。
|
||||
4. 如果视频包含广告内容，可以分析广告品类、目标用户、转化意图，但不要编造品牌信息。
|
||||
""".strip()
|
||||
|
||||
|
||||
def build_video_analysis_user_text(*, mode: AnalysisMode) -> str:
|
||||
if mode == "summary_only":
|
||||
payload = {
|
||||
"任务": "请根据上传的视频片段附件，返回这个视频片段的内容分析 JSON。",
|
||||
"输出JSON格式": {
|
||||
"原视频内容": "概括这个视频片段整体内容，描述主要画面、主体、场景、动作、剧情或信息点",
|
||||
"原视频分类": "判断视频类型，例如：游戏视频、产品广告视频、剧情视频、口播讲解视频、教程视频、生活记录视频等",
|
||||
"原视频受众人群": "判断该片段更适合的人群",
|
||||
"拆镜内容剖析": [],
|
||||
},
|
||||
"返回要求": ["只返回 JSON 对象", "不要返回 Markdown", "不要返回解释文字", "不要返回代码块", "不要返回 null，未知填无"],
|
||||
}
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
payload = {
|
||||
"任务": "请根据上传的视频附件，返回原视频内容分析和拆镜内容剖析 JSON。",
|
||||
"输出JSON格式": {
|
||||
"原视频内容": "概括原视频整体内容，描述主要画面、人物/主体、场景、动作、剧情或信息点",
|
||||
"原视频分类": "判断视频类型，例如：游戏视频、产品广告视频、剧情视频、口播讲解视频、教程视频、生活记录视频、直播切片视频、图文快闪视频等",
|
||||
"原视频受众人群": "判断该视频更适合的人群，例如：游戏玩家、年轻娱乐用户、潜在购买用户、同城社交用户等",
|
||||
"拆镜内容剖析": [
|
||||
{
|
||||
"拆镜开始秒": 0,
|
||||
"拆镜结束秒": 15,
|
||||
"拆镜时间节点": "0-15秒",
|
||||
"对应时间节点内的内容": "描述这个时间片段内发生了什么",
|
||||
"分类": "判断这个片段的内容分类，例如：开场吸引、冲突铺垫、玩法展示、卖点展示、情绪爆发、行动引导、结果展示等",
|
||||
"受众人群": "判断这个片段主要吸引的人群",
|
||||
}
|
||||
],
|
||||
},
|
||||
"拆镜规则": [
|
||||
f"每个拆镜片段时长必须大于等于 {_split_min_seconds():g} 秒",
|
||||
f"每个拆镜片段时长必须小于等于 {_split_max_seconds():g} 秒",
|
||||
"拆镜开始秒必须是数字",
|
||||
"拆镜结束秒必须是数字",
|
||||
"拆镜开始秒不能是负数",
|
||||
"拆镜结束秒必须大于拆镜开始秒",
|
||||
"拆镜时间节点必须等于：拆镜开始秒-拆镜结束秒秒",
|
||||
"禁止输出“-15秒”",
|
||||
"禁止输出“15-秒”",
|
||||
"如果无法判断拆镜节点，拆镜内容剖析返回空数组",
|
||||
],
|
||||
"返回要求": ["只返回 JSON 对象", "不要返回 Markdown", "不要返回解释文字", "不要返回代码块", "不要返回 null，未知填无"],
|
||||
}
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def strip_json_code_fence(text: str) -> str:
|
||||
text = text.strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?", "", text, flags=re.IGNORECASE).strip()
|
||||
text = re.sub(r"```$", "", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def parse_model_json(content: str) -> dict[str, Any]:
|
||||
cleaned = strip_json_code_fence(content)
|
||||
data = json.loads(cleaned)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"模型返回 JSON 不是对象类型: {type(data).__name__}")
|
||||
return data
|
||||
|
||||
|
||||
def get_message_content_or_raise(data: dict[str, Any]) -> str:
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
raise RuntimeError(f"模型响应没有 choices: {json.dumps(data, ensure_ascii=False)}")
|
||||
choice = choices[0]
|
||||
finish_reason = choice.get("finish_reason")
|
||||
if finish_reason == "length":
|
||||
usage = data.get("usage", {})
|
||||
raise RuntimeError(f"模型输出被长度限制截断，finish_reason={finish_reason}, usage={json.dumps(usage, ensure_ascii=False)}")
|
||||
message = choice.get("message") or {}
|
||||
content = message.get("content", "")
|
||||
if not content:
|
||||
raise RuntimeError(f"模型响应 content 为空: {json.dumps(data, ensure_ascii=False)}")
|
||||
return content.strip()
|
||||
|
||||
|
||||
def fill_none_with_wu(value: Any) -> Any:
|
||||
if value is None:
|
||||
return "无"
|
||||
if isinstance(value, str):
|
||||
return value if value.strip() else "无"
|
||||
if isinstance(value, list):
|
||||
return [fill_none_with_wu(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: fill_none_with_wu(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def first_present(data: dict[str, Any], keys: list[str]) -> Any:
|
||||
for key in keys:
|
||||
if key in data:
|
||||
return data.get(key)
|
||||
return None
|
||||
|
||||
|
||||
def parse_number(value: Any) -> float | None:
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
m = re.search(r"-?\d+(?:\.\d+)?", text)
|
||||
return float(m.group(0)) if m else None
|
||||
|
||||
|
||||
def parse_time_value_to_seconds(value: str) -> float | None:
|
||||
value = str(value or "").strip()
|
||||
if not value:
|
||||
return None
|
||||
m = re.match(r"^(\d+(?:\.\d+)?)\s*(?:秒|s)?$", value, flags=re.IGNORECASE)
|
||||
if m:
|
||||
return float(m.group(1))
|
||||
parts = value.split(":")
|
||||
if len(parts) in (2, 3) and all(re.match(r"^\d+(?:\.\d+)?$", p.strip()) for p in parts):
|
||||
nums = [float(p.strip()) for p in parts]
|
||||
if len(nums) == 2:
|
||||
minute, second = nums
|
||||
return minute * 60 + second
|
||||
hour, minute, second = nums
|
||||
return hour * 3600 + minute * 60 + second
|
||||
return None
|
||||
|
||||
|
||||
def parse_time_node_to_range(time_node: str) -> tuple[float, float] | None:
|
||||
text = str(time_node or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
text = text.replace("—", "-").replace("–", "-").replace("－", "-")
|
||||
text = text.replace("到", "-").replace("至", "-").replace("~", "-").replace("～", "-")
|
||||
text = text.replace("第", "").replace("时间段", "").replace("：", ":")
|
||||
m = re.search(r"(\d{1,2}:\d{1,2}(?::\d{1,2})?)\s*-\s*(\d{1,2}:\d{1,2}(?::\d{1,2})?)", text)
|
||||
if m:
|
||||
start = parse_time_value_to_seconds(m.group(1))
|
||||
end = parse_time_value_to_seconds(m.group(2))
|
||||
if start is not None and end is not None and end > start:
|
||||
return start, end
|
||||
m = re.search(r"(\d+(?:\.\d+)?)\s*(?:秒|s)?\s*-\s*(\d+(?:\.\d+)?)\s*(?:秒|s)?", text, flags=re.IGNORECASE)
|
||||
if m:
|
||||
start = float(m.group(1))
|
||||
end = float(m.group(2))
|
||||
if end > start:
|
||||
return start, end
|
||||
m = re.search(r"^\s*-\s*(\d+(?:\.\d+)?)\s*(?:秒|s)?\s*$", text, flags=re.IGNORECASE)
|
||||
if m:
|
||||
end = float(m.group(1))
|
||||
if end > 0:
|
||||
return 0.0, end
|
||||
return None
|
||||
|
||||
|
||||
def format_second(value: float) -> int | float:
|
||||
checked = float(value)
|
||||
return int(checked) if checked.is_integer() else round(checked, 2)
|
||||
|
||||
|
||||
def normalize_time_node_by_range(start: float, end: float) -> str:
|
||||
return f"{format_second(start)}-{format_second(end)}秒"
|
||||
|
||||
|
||||
def ensure_result_schema(result: dict[str, Any]) -> dict[str, Any]:
|
||||
final_result = {
|
||||
"原视频内容": result.get("原视频内容", "无"),
|
||||
"原视频分类": result.get("原视频分类", "无"),
|
||||
"原视频受众人群": result.get("原视频受众人群", "无"),
|
||||
"拆镜内容剖析": result.get("拆镜内容剖析", []),
|
||||
}
|
||||
for key in ("原视频内容", "原视频分类", "原视频受众人群"):
|
||||
if not isinstance(final_result[key], str):
|
||||
final_result[key] = json.dumps(final_result[key], ensure_ascii=False)
|
||||
if not isinstance(final_result["拆镜内容剖析"], list):
|
||||
final_result["拆镜内容剖析"] = []
|
||||
return final_result
|
||||
|
||||
|
||||
def filter_and_normalize_breakdown(result: dict[str, Any], *, mode: AnalysisMode = "full_breakdown") -> dict[str, Any]:
|
||||
if mode == "summary_only":
|
||||
result["拆镜内容剖析"] = []
|
||||
return result
|
||||
|
||||
breakdown = result.get("拆镜内容剖析")
|
||||
if not isinstance(breakdown, list):
|
||||
result["拆镜内容剖析"] = []
|
||||
return result
|
||||
|
||||
normalized_items: list[dict[str, Any]] = []
|
||||
for item in breakdown:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
raw_start = first_present(item, ["拆镜开始秒", "开始秒", "起始秒", "开始时间", "起始时间", "start", "start_second", "start_seconds"])
|
||||
raw_end = first_present(item, ["拆镜结束秒", "结束秒", "结束时间", "end", "end_second", "end_seconds"])
|
||||
start = parse_number(raw_start)
|
||||
end = parse_number(raw_end)
|
||||
time_node = str(first_present(item, ["拆镜时间节点", "时间节点", "时间段", "镜头时间", "time_node", "time_range"]) or "").strip()
|
||||
if start is None or end is None:
|
||||
parsed_range = parse_time_node_to_range(time_node)
|
||||
if parsed_range is None:
|
||||
continue
|
||||
start, end = parsed_range
|
||||
if start is None or end is None or start < 0 or end < 0 or end <= start:
|
||||
continue
|
||||
duration = end - start
|
||||
if duration < _split_min_seconds() or duration > _split_max_seconds():
|
||||
continue
|
||||
content = first_present(item, ["对应时间节点内的内容", "内容", "画面内容", "片段内容", "镜头内容", "content"]) or "无"
|
||||
category = first_present(item, ["分类", "片段分类", "内容分类", "镜头分类", "category"]) or "无"
|
||||
audience = first_present(item, ["受众人群", "目标受众", "片段受众", "镜头受众", "audience"]) or "无"
|
||||
normalized_items.append({
|
||||
"拆镜开始秒": format_second(start),
|
||||
"拆镜结束秒": format_second(end),
|
||||
"拆镜时间节点": normalize_time_node_by_range(start, end),
|
||||
"对应时间节点内的内容": str(content or "无"),
|
||||
"分类": str(category or "无"),
|
||||
"受众人群": str(audience or "无"),
|
||||
})
|
||||
normalized_items.sort(key=lambda x: float(x.get("拆镜开始秒", 0)))
|
||||
result["拆镜内容剖析"] = normalized_items
|
||||
return result
|
||||
|
||||
|
||||
async def _select_model_config(db: AsyncSession) -> ModelConfig | None:
|
||||
result = await db.execute(
|
||||
select(ModelConfig)
|
||||
.where(ModelConfig.is_active == True)
|
||||
.order_by(ModelConfig.priority.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _int_usage(value: Any) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
async def analyze_video_for_shot_split(
|
||||
db: AsyncSession,
|
||||
video_url: str,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
mode: AnalysisMode = "full_breakdown",
|
||||
) -> ShotVideoAnalysisResult:
|
||||
"""调用模型完成拆镜/片段分析。
|
||||
|
||||
模型配置统一从 model_configs 表选择当前启用且 priority 最高的配置；
|
||||
不再读取 SHOT_ANALYSIS_API_BASE / SHOT_ANALYSIS_API_KEY / SHOT_ANALYSIS_MODEL_NAME，
|
||||
也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。
|
||||
"""
|
||||
config = await _select_model_config(db)
|
||||
if not config:
|
||||
raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型")
|
||||
if not str(config.api_key or "").strip():
|
||||
raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}")
|
||||
if not str(config.api_base or "").strip():
|
||||
raise RuntimeError(f"拆镜分析模型 API Base 为空: model_config_id={config.id}")
|
||||
if not str(config.model_name or "").strip():
|
||||
raise RuntimeError(f"拆镜分析模型名称为空: model_config_id={config.id}")
|
||||
|
||||
system_prompt = build_video_analysis_system_prompt(mode=mode)
|
||||
user_text = build_video_analysis_user_text(mode=mode)
|
||||
user_message, log_user_message = build_user_message(user_text, video_url)
|
||||
|
||||
request_data: dict[str, Any] = {
|
||||
"model": config.model_name,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
user_message,
|
||||
],
|
||||
"max_tokens": int(getattr(settings, "SHOT_ANALYSIS_MAX_TOKENS", 5000) or getattr(config, "max_tokens", 5000) or 5000),
|
||||
"temperature": float(getattr(settings, "SHOT_ANALYSIS_TEMPERATURE", 0.1) or getattr(config, "temperature", 0.1) or 0.1),
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
log_request_data: dict[str, Any] = {
|
||||
**request_data,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
log_user_message,
|
||||
],
|
||||
"model_config_id": config.id,
|
||||
"model_config_name": config.name,
|
||||
"provider": config.provider,
|
||||
"analysis_mode": mode,
|
||||
}
|
||||
|
||||
url = f"{str(config.api_base).rstrip('/')}/chat/completions"
|
||||
async with httpx.AsyncClient(timeout=_timeout_seconds()) as client:
|
||||
response = await client.post(
|
||||
url,
|
||||
headers={"Authorization": f"Bearer {config.api_key}", "Content-Type": "application/json"},
|
||||
json=request_data,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"视频拆镜分析 API 请求失败: HTTP {response.status_code}: {response.text}")
|
||||
|
||||
raw = response.json()
|
||||
content = get_message_content_or_raise(raw)
|
||||
result = parse_model_json(content)
|
||||
result = fill_none_with_wu(result)
|
||||
result = ensure_result_schema(result)
|
||||
result = filter_and_normalize_breakdown(result, mode=mode)
|
||||
|
||||
usage = raw.get("usage") or {}
|
||||
token_usage = {
|
||||
"input_tokens": _int_usage(usage.get("prompt_tokens") or usage.get("input_tokens")),
|
||||
"output_tokens": _int_usage(usage.get("completion_tokens") or usage.get("output_tokens")),
|
||||
"total_tokens": _int_usage(usage.get("total_tokens")),
|
||||
"finish_reason": ((raw.get("choices") or [{}])[0] or {}).get("finish_reason"),
|
||||
"model_config_id": config.id,
|
||||
"model_config_name": config.name,
|
||||
"model": config.model_name,
|
||||
"provider": config.provider,
|
||||
"video_fps": _video_fps(),
|
||||
"split_min_seconds": _split_min_seconds(),
|
||||
"split_max_seconds": _split_max_seconds(),
|
||||
"analysis_mode": mode,
|
||||
"log_request": log_request_data,
|
||||
}
|
||||
if not token_usage["total_tokens"]:
|
||||
token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"]
|
||||
|
||||
db.add(
|
||||
TokenUsage(
|
||||
id=generate_id(),
|
||||
model_config_id=config.id,
|
||||
user_id=user_id,
|
||||
input_tokens=token_usage["input_tokens"],
|
||||
output_tokens=token_usage["output_tokens"],
|
||||
total_tokens=token_usage["total_tokens"],
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
return ShotVideoAnalysisResult(result=result, raw_response=raw, usage=token_usage)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
from app.services.upload_video_asset_service import (
|
||||
build_upload_url_from_path,
|
||||
ensure_shot_segment_dir,
|
||||
get_ffmpeg_bin,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ShotSplitResult:
|
||||
url: str
|
||||
path: str
|
||||
file_size_bytes: int
|
||||
|
||||
|
||||
def _date_dir_from_segment_id(segment_id: str) -> str:
|
||||
# 由调用方更适合按 created_at 传入；这里兜底按当前日期。
|
||||
from datetime import datetime
|
||||
|
||||
return datetime.now().strftime("%Y/%m/%d")
|
||||
|
||||
|
||||
def _safe_unlink(path: Path) -> None:
|
||||
try:
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def split_video_segment(
|
||||
*,
|
||||
source_path: str | Path,
|
||||
segment_id: str,
|
||||
start_second: float,
|
||||
end_second: float,
|
||||
date_dir: str | None = None,
|
||||
) -> ShotSplitResult:
|
||||
"""使用 ffmpeg 拆出单个视频片段，输出到 storage/uploads/shot_segments。"""
|
||||
source_path = Path(source_path)
|
||||
|
||||
if not source_path.exists():
|
||||
raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}")
|
||||
|
||||
start = max(float(start_second), 0.0)
|
||||
end = max(float(end_second), 0.0)
|
||||
duration = end - start
|
||||
|
||||
if duration <= 0:
|
||||
raise RuntimeError(
|
||||
f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}"
|
||||
)
|
||||
|
||||
date_dir = date_dir or _date_dir_from_segment_id(segment_id)
|
||||
output_dir = ensure_shot_segment_dir(date_dir)
|
||||
output_path = output_dir / f"{segment_id}.mp4"
|
||||
|
||||
# 注意：
|
||||
# 不能用 xxx.mp4.part，因为 ffmpeg 会按最后一个扩展名 .part 判断输出格式，导致：
|
||||
# Unable to choose an output format
|
||||
# 这里改为 xxx.part.mp4，让 ffmpeg 能识别 mp4 容器。
|
||||
part_path = output_dir / f"{segment_id}.part.mp4"
|
||||
|
||||
_safe_unlink(part_path)
|
||||
|
||||
timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120)
|
||||
|
||||
cmd = [
|
||||
get_ffmpeg_bin(),
|
||||
"-y",
|
||||
|
||||
# 先 seek 到起始秒，再按 duration 切割，避免 -to 在不同 ffmpeg 参数位置下语义不一致。
|
||||
"-ss",
|
||||
f"{start:.3f}",
|
||||
"-i",
|
||||
str(source_path),
|
||||
"-t",
|
||||
f"{duration:.3f}",
|
||||
|
||||
# 只取主视频流，音频可选，避免 map 0 把字幕/数据流带进去导致 mp4 封装失败。
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a:0?",
|
||||
|
||||
# 当前是拆镜片段，重编码更稳，避免关键帧不准导致片段首尾异常。
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"23",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"128k",
|
||||
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
|
||||
# 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。
|
||||
"-f",
|
||||
"mp4",
|
||||
|
||||
str(part_path),
|
||||
]
|
||||
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
_safe_unlink(part_path)
|
||||
raise RuntimeError(
|
||||
f"ffmpeg 拆镜超时：timeout={timeout}s, start={start:.3f}, end={end:.3f}"
|
||||
) from exc
|
||||
|
||||
if completed.returncode != 0:
|
||||
_safe_unlink(part_path)
|
||||
raise RuntimeError(
|
||||
f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}"
|
||||
)
|
||||
|
||||
if not part_path.exists() or part_path.stat().st_size <= 0:
|
||||
_safe_unlink(part_path)
|
||||
raise RuntimeError("ffmpeg 拆镜失败：输出文件为空")
|
||||
|
||||
os.replace(part_path, output_path)
|
||||
|
||||
return ShotSplitResult(
|
||||
url=build_upload_url_from_path(output_path),
|
||||
path=str(output_path),
|
||||
file_size_bytes=output_path.stat().st_size,
|
||||
)
|
||||
|
||||
|
||||
async def split_video_segment_async(
|
||||
*,
|
||||
source_path: str | Path,
|
||||
segment_id: str,
|
||||
start_second: float,
|
||||
end_second: float,
|
||||
date_dir: str | None = None,
|
||||
) -> ShotSplitResult:
|
||||
"""异步拆镜入口。
|
||||
|
||||
ffmpeg 本身是同步阻塞命令，不能直接在 Celery 进程内唯一 event loop 中执行。
|
||||
这里通过 asyncio.to_thread 跑同步拆镜函数，避免阻塞 asyncpg / Redis / HTTP 等异步任务。
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
split_video_segment,
|
||||
source_path=source_path,
|
||||
segment_id=segment_id,
|
||||
start_second=start_second,
|
||||
end_second=end_second,
|
||||
date_dir=date_dir,
|
||||
)
|
||||
@@ -0,0 +1,211 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.config import settings
|
||||
|
||||
VIDEO_EXTENSIONS = {".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv"}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class UploadVideoAsset:
|
||||
url: str
|
||||
path: Path
|
||||
duration_seconds: float
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def _abs_path(value: str | Path) -> Path:
|
||||
path = Path(value)
|
||||
if not path.is_absolute():
|
||||
path = _project_root() / path
|
||||
return path.resolve()
|
||||
|
||||
|
||||
def upload_root() -> Path:
|
||||
return _abs_path(settings.UPLOAD_LOCAL_PATH)
|
||||
|
||||
|
||||
def shot_segment_root() -> Path:
|
||||
return _abs_path(getattr(settings, "SHOT_SEGMENT_LOCAL_PATH", "./storage/uploads/shot_segments"))
|
||||
|
||||
|
||||
def _strip_base_url(url: str) -> str:
|
||||
text = str(url or "").strip()
|
||||
if not text:
|
||||
return text
|
||||
|
||||
base_url = str(getattr(settings, "BASE_URL", "") or "").strip().rstrip("/")
|
||||
if base_url and text.startswith(base_url + "/"):
|
||||
return text[len(base_url):]
|
||||
|
||||
parsed = urlparse(text)
|
||||
if parsed.scheme in ("http", "https"):
|
||||
# 只接受本系统 BASE_URL 下的上传资源；外部 URL 不允许 ffmpeg 本地切片。
|
||||
raise HTTPException(status_code=400, detail="拆镜源视频必须来自本系统上传接口，不能传外部 http/https URL")
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _safe_relative_from_upload_url(url: str) -> str:
|
||||
value = _strip_base_url(url)
|
||||
value = value.split("?", 1)[0].split("#", 1)[0]
|
||||
|
||||
if value.startswith("/uploads/"):
|
||||
rel = value.replace("/uploads/", "", 1)
|
||||
elif value.startswith("uploads/"):
|
||||
rel = value.replace("uploads/", "", 1)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="拆镜源视频链接必须是 /uploads/ 下的上传资源")
|
||||
|
||||
rel = rel.lstrip("/")
|
||||
if not rel or ".." in Path(rel).parts:
|
||||
raise HTTPException(status_code=400, detail="上传视频路径非法")
|
||||
return rel
|
||||
|
||||
|
||||
def resolve_upload_video_path(video_url: str) -> Path:
|
||||
rel = _safe_relative_from_upload_url(video_url)
|
||||
root = upload_root()
|
||||
path = (root / rel).resolve()
|
||||
|
||||
try:
|
||||
path.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="上传视频路径越界") from exc
|
||||
|
||||
if path.suffix.lower() not in VIDEO_EXTENSIONS:
|
||||
raise HTTPException(status_code=400, detail="上传资源不是支持的视频格式")
|
||||
if not path.exists() or not path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"上传视频文件不存在: {video_url}")
|
||||
return path
|
||||
|
||||
|
||||
def build_upload_url_from_path(path: str | Path) -> str:
|
||||
root = upload_root()
|
||||
checked_path = _abs_path(path)
|
||||
try:
|
||||
rel = checked_path.relative_to(root).as_posix()
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=500, detail="生成上传资源 URL 失败：路径不在 uploads 目录下") from exc
|
||||
return f"/uploads/{rel}"
|
||||
|
||||
|
||||
def get_ffmpeg_bin() -> str:
|
||||
return str(getattr(settings, "FFMPEG_BIN", "") or "ffmpeg")
|
||||
|
||||
|
||||
def get_ffprobe_bin() -> str:
|
||||
configured = str(getattr(settings, "FFPROBE_BIN", "") or "").strip()
|
||||
if configured:
|
||||
return configured
|
||||
ffmpeg_bin = get_ffmpeg_bin()
|
||||
if ffmpeg_bin.endswith("ffmpeg.exe"):
|
||||
return ffmpeg_bin[:-10] + "ffprobe.exe"
|
||||
if ffmpeg_bin.endswith("ffmpeg"):
|
||||
return ffmpeg_bin[:-6] + "ffprobe"
|
||||
return "ffprobe"
|
||||
|
||||
|
||||
def probe_video_duration_seconds(video_path: str | Path) -> float:
|
||||
path = _abs_path(video_path)
|
||||
timeout = int(getattr(settings, "SHOT_FFPROBE_TIMEOUT_SECONDS", 20) or 20)
|
||||
cmd = [
|
||||
get_ffprobe_bin(),
|
||||
"-v", "error",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "json",
|
||||
str(path),
|
||||
]
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=f"ffprobe 获取视频时长失败: {exc}") from exc
|
||||
|
||||
if completed.returncode != 0:
|
||||
raise HTTPException(status_code=400, detail=f"ffprobe 获取视频时长失败: {completed.stderr.strip()}")
|
||||
|
||||
try:
|
||||
data = json.loads(completed.stdout or "{}")
|
||||
duration = float((data.get("format") or {}).get("duration") or 0)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail="ffprobe 返回的视频时长无法解析") from exc
|
||||
|
||||
if duration <= 0:
|
||||
raise HTTPException(status_code=400, detail="视频时长无效")
|
||||
return round(duration, 3)
|
||||
|
||||
|
||||
def validate_upload_video_asset(video_url: str, frontend_duration_seconds: float | None = None) -> UploadVideoAsset:
|
||||
path = resolve_upload_video_path(video_url)
|
||||
real_duration = probe_video_duration_seconds(path)
|
||||
|
||||
if frontend_duration_seconds is not None and frontend_duration_seconds > 0:
|
||||
tolerance = float(getattr(settings, "SHOT_DURATION_TOLERANCE_SECONDS", 1.0) or 1.0)
|
||||
# 超出误差时以后端 ffprobe 为准，不拒绝，避免前端浮点或浏览器 metadata 偏差导致创建失败。
|
||||
if abs(float(frontend_duration_seconds) - real_duration) <= tolerance:
|
||||
real_duration = round(float(frontend_duration_seconds), 3)
|
||||
|
||||
return UploadVideoAsset(url=_strip_base_url(video_url), path=path, duration_seconds=real_duration)
|
||||
|
||||
|
||||
def validate_split_range(*, start_second: float, end_second: float, video_duration_seconds: float) -> tuple[float, float, float]:
|
||||
start = round(float(start_second), 3)
|
||||
end = round(float(end_second), 3)
|
||||
|
||||
if start < 0:
|
||||
raise HTTPException(status_code=400, detail="开始秒不能小于0")
|
||||
if end <= start:
|
||||
raise HTTPException(status_code=400, detail="结束秒必须大于开始秒")
|
||||
|
||||
tolerance = float(getattr(settings, "SHOT_SPLIT_END_TOLERANCE_SECONDS", 0.5) or 0.5)
|
||||
if end > float(video_duration_seconds) + tolerance:
|
||||
raise HTTPException(status_code=400, detail="结束秒不能超过视频总时长")
|
||||
|
||||
duration = round(end - start, 3)
|
||||
min_seconds = float(getattr(settings, "SHOT_SPLIT_MIN_SECONDS", 1) or 1)
|
||||
max_seconds = float(getattr(settings, "SHOT_SPLIT_MAX_SECONDS", 120) or 120)
|
||||
if duration < min_seconds:
|
||||
raise HTTPException(status_code=400, detail=f"拆镜片段不能低于 {min_seconds:g} 秒")
|
||||
if duration > max_seconds:
|
||||
raise HTTPException(status_code=400, detail=f"拆镜片段不能超过 {max_seconds:g} 秒")
|
||||
return start, end, duration
|
||||
|
||||
|
||||
def format_second(value: float) -> int | float:
|
||||
checked = float(value)
|
||||
if checked.is_integer():
|
||||
return int(checked)
|
||||
return round(checked, 2)
|
||||
|
||||
|
||||
def build_time_node(start_second: float, end_second: float) -> str:
|
||||
return f"{format_second(start_second)}-{format_second(end_second)}秒"
|
||||
|
||||
|
||||
def ensure_shot_segment_dir(date_dir: str) -> Path:
|
||||
root = shot_segment_root()
|
||||
output_dir = (root / date_dir).resolve()
|
||||
try:
|
||||
output_dir.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("拆镜输出目录越界") from exc
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
return output_dir
|
||||
Reference in New Issue
Block a user