340 lines
13 KiB
Python
340 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from decimal import Decimal
|
|
from typing import Any, Iterable, Mapping
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
from app.services.model_pricing.usage_normalizer import (
|
|
extract_image_output_items,
|
|
parse_size,
|
|
safe_bool,
|
|
safe_float,
|
|
safe_int,
|
|
safe_json_dict,
|
|
sanitize_output_items,
|
|
)
|
|
|
|
|
|
MEDIA_TYPES = {"image", "video", "audio"}
|
|
|
|
|
|
def _safe_json(value: Any) -> Any:
|
|
if value in (None, ""):
|
|
return None
|
|
if isinstance(value, (dict, list)):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
return json.loads(value)
|
|
except Exception:
|
|
return value
|
|
return value
|
|
|
|
|
|
def _normalize_url(value: str | None) -> tuple[str | None, str | None]:
|
|
if not value:
|
|
return None, None
|
|
text = str(value).strip()
|
|
try:
|
|
parts = urlsplit(text)
|
|
if parts.scheme and parts.netloc:
|
|
normalized = urlunsplit((parts.scheme.lower(), parts.netloc.lower(), parts.path, "", ""))
|
|
else:
|
|
normalized = text.split("?", 1)[0].split("#", 1)[0]
|
|
except Exception:
|
|
normalized = text.split("?", 1)[0].split("#", 1)[0]
|
|
normalized = normalized[:1024]
|
|
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
|
return normalized, digest
|
|
|
|
|
|
def _guess_media_type(item: Mapping[str, Any]) -> str | None:
|
|
value = str(item.get("type") or item.get("media_type") or item.get("resource_type") or "").lower().strip()
|
|
if value in MEDIA_TYPES:
|
|
return value
|
|
url = str(item.get("url") or item.get("path") or item.get("display_url") or "").lower()
|
|
if re.search(r"\.(png|jpe?g|webp|gif|bmp)(?:\?|$)", url):
|
|
return "image"
|
|
if re.search(r"\.(mp4|mov|m4v|webm|avi|mkv)(?:\?|$)", url):
|
|
return "video"
|
|
if re.search(r"\.(mp3|wav|aac|m4a|flac|ogg)(?:\?|$)", url):
|
|
return "audio"
|
|
return None
|
|
|
|
|
|
def _walk_reference_items(value: Any) -> Iterable[Mapping[str, Any]]:
|
|
parsed = _safe_json(value)
|
|
if isinstance(parsed, list):
|
|
for item in parsed:
|
|
yield from _walk_reference_items(item)
|
|
return
|
|
if isinstance(parsed, dict):
|
|
if _guess_media_type(parsed) or any(k in parsed for k in ("url", "path", "resource_id", "private_asset_id")):
|
|
yield parsed
|
|
return
|
|
for child in parsed.values():
|
|
if isinstance(child, (dict, list, str)):
|
|
yield from _walk_reference_items(child)
|
|
|
|
|
|
def _billable_input(
|
|
raw: Mapping[str, Any],
|
|
media_type: str,
|
|
*,
|
|
allow_provider_input: bool,
|
|
) -> bool:
|
|
# 是否作为供应商直接输入由服务端调用链决定，不能信任客户端附件字段。
|
|
if not allow_provider_input:
|
|
return False
|
|
if "billable_input" in raw:
|
|
return safe_bool(raw.get("billable_input"), True)
|
|
role = str(raw.get("role") or raw.get("label") or raw.get("reference_role") or "").lower()
|
|
if role in {"cover", "preview", "display_only", "generated_output", "output"}:
|
|
return False
|
|
return media_type in MEDIA_TYPES
|
|
|
|
|
|
def build_attachment_snapshot(
|
|
media_references: Any,
|
|
*,
|
|
allow_provider_input: bool = True,
|
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
items: list[dict[str, Any]] = []
|
|
image_count = video_count = audio_count = 0
|
|
provider_input_image_count = 0
|
|
provider_input_video_count = 0
|
|
provider_input_audio_count = 0
|
|
video_duration = Decimal("0")
|
|
audio_duration = Decimal("0")
|
|
seen: set[str] = set()
|
|
|
|
for raw in _walk_reference_items(media_references):
|
|
media_type = _guess_media_type(raw)
|
|
if media_type not in MEDIA_TYPES:
|
|
continue
|
|
raw_url = raw.get("url") or raw.get("path") or raw.get("display_url") or raw.get("preview_url")
|
|
safe_url, url_hash = _normalize_url(str(raw_url) if raw_url else None)
|
|
identity = str(
|
|
raw.get("resource_id")
|
|
or raw.get("upload_resource_id")
|
|
or raw.get("private_asset_id")
|
|
or url_hash
|
|
or f"{media_type}:{len(items)}"
|
|
)
|
|
dedupe_key = f"{media_type}:{identity}"
|
|
if dedupe_key in seen:
|
|
continue
|
|
seen.add(dedupe_key)
|
|
|
|
duration = max(0.0, safe_float(raw.get("duration"), safe_float(raw.get("duration_seconds"))))
|
|
billable = _billable_input(
|
|
raw,
|
|
media_type,
|
|
allow_provider_input=allow_provider_input,
|
|
)
|
|
item = {
|
|
"type": media_type,
|
|
"role": raw.get("role") or raw.get("label") or raw.get("reference_role"),
|
|
"source": raw.get("source"),
|
|
"billable_input": billable,
|
|
"resource_id": raw.get("resource_id") or raw.get("upload_resource_id"),
|
|
"private_asset_id": raw.get("private_asset_id"),
|
|
"name": raw.get("name") or raw.get("filename"),
|
|
"duration_seconds": duration or None,
|
|
"file_size": safe_int(raw.get("file_size"), safe_int(raw.get("size"))) or None,
|
|
"safe_url": safe_url,
|
|
"url_sha256": url_hash,
|
|
}
|
|
items.append({k: v for k, v in item.items() if v is not None})
|
|
if media_type == "image":
|
|
image_count += 1
|
|
provider_input_image_count += int(billable)
|
|
elif media_type == "video":
|
|
video_count += 1
|
|
provider_input_video_count += int(billable)
|
|
video_duration += Decimal(str(duration))
|
|
else:
|
|
audio_count += 1
|
|
provider_input_audio_count += int(billable)
|
|
audio_duration += Decimal(str(duration))
|
|
|
|
counts = {
|
|
"attachment_image_count": image_count,
|
|
"attachment_video_count": video_count,
|
|
"attachment_audio_count": audio_count,
|
|
"attachment_total_count": image_count + video_count + audio_count,
|
|
"attachment_video_duration_seconds": video_duration,
|
|
"attachment_audio_duration_seconds": audio_duration,
|
|
"provider_input_image_count": provider_input_image_count,
|
|
"provider_input_video_count": provider_input_video_count,
|
|
"provider_input_audio_count": provider_input_audio_count,
|
|
}
|
|
snapshot = {
|
|
"schema_version": 1,
|
|
"items": items,
|
|
"counts": {
|
|
"image": image_count,
|
|
"video": video_count,
|
|
"audio": audio_count,
|
|
"total": image_count + video_count + audio_count,
|
|
"provider_input_image": provider_input_image_count,
|
|
"provider_input_video": provider_input_video_count,
|
|
"provider_input_audio": provider_input_audio_count,
|
|
},
|
|
"durations": {
|
|
"video_seconds": str(video_duration),
|
|
"audio_seconds": str(audio_duration),
|
|
},
|
|
}
|
|
return snapshot, counts
|
|
|
|
|
|
def parse_dimensions(*values: Any, resolution: str | None = None, aspect_ratio: str | None = None) -> tuple[int, int]:
|
|
"""仅解析明确像素；resolution/aspect_ratio 不再映射为猜测尺寸。"""
|
|
del resolution, aspect_ratio
|
|
for value in values:
|
|
width, height = parse_size(value)
|
|
if width > 0 and height > 0:
|
|
return width, height
|
|
return 0, 0
|
|
|
|
|
|
def _engine_snapshot(owner: Any) -> dict[str, Any]:
|
|
return safe_json_dict(getattr(owner, "engine_snapshot_json", None))
|
|
|
|
|
|
def build_generation_snapshot(
|
|
owner: Any,
|
|
*,
|
|
provider_response: Any = None,
|
|
stage: str | None = None,
|
|
) -> tuple[dict[str, Any], dict[str, int], dict[str, Any]]:
|
|
"""构建请求/Provider/资源快照。
|
|
|
|
图片只读取同步接口明确的 data 输出条目；不会递归扫描 provider response 中的通用 URL。
|
|
"""
|
|
gen_type = str(getattr(owner, "gen_type", None) or getattr(owner, "media_type", None) or "").lower().strip()
|
|
response = safe_json_dict(provider_response if provider_response is not None else getattr(owner, "provider_response_json", None))
|
|
engine_snapshot = _engine_snapshot(owner)
|
|
aspect_ratio = getattr(owner, "aspect_ratio", None) or getattr(owner, "image_proportion", None)
|
|
resolution = getattr(owner, "resolution", None)
|
|
|
|
width, height = parse_dimensions(
|
|
response.get("size"),
|
|
getattr(owner, "image_px", None),
|
|
engine_snapshot.get("selected_px"),
|
|
engine_snapshot.get("image_px"),
|
|
)
|
|
dimension_source = "unavailable"
|
|
if parse_dimensions(response.get("size")) != (0, 0):
|
|
dimension_source = "provider_response"
|
|
elif parse_dimensions(getattr(owner, "image_px", None)) != (0, 0):
|
|
dimension_source = "request_explicit"
|
|
elif parse_dimensions(engine_snapshot.get("selected_px"), engine_snapshot.get("image_px")) != (0, 0):
|
|
dimension_source = "engine_snapshot"
|
|
|
|
output_items: list[dict[str, Any]] = []
|
|
generated_image_count = 0
|
|
generated_video_count = 0
|
|
if gen_type == "image":
|
|
output_items = extract_image_output_items(response)
|
|
if width <= 0 or height <= 0:
|
|
first_sized = next(
|
|
(item for item in output_items if safe_int(item.get("width")) > 0 and safe_int(item.get("height")) > 0),
|
|
None,
|
|
)
|
|
if first_sized:
|
|
width = safe_int(first_sized.get("width"))
|
|
height = safe_int(first_sized.get("height"))
|
|
dimension_source = "provider_response"
|
|
if width > 0 and height > 0:
|
|
for item in output_items:
|
|
if not item.get("width") or not item.get("height"):
|
|
item.update(
|
|
{
|
|
"width": width,
|
|
"height": height,
|
|
"pixels": width * height,
|
|
"size_source": dimension_source,
|
|
}
|
|
)
|
|
output_items = sanitize_output_items(output_items)
|
|
generated_image_count = len(output_items)
|
|
if generated_image_count == 0 and stage == "resource_download_completed" and getattr(owner, "image_url", None):
|
|
generated_image_count = 1
|
|
output_items = sanitize_output_items(
|
|
[{"index": 0, "url": getattr(owner, "image_url"), "size_source": "resource_snapshot"}]
|
|
)
|
|
elif gen_type == "video":
|
|
video_url = response.get("video_url") or response.get("url")
|
|
if stage == "resource_download_completed":
|
|
video_url = getattr(owner, "video_url", None) or video_url
|
|
generated_video_count = 1 if video_url else 0
|
|
if video_url:
|
|
output_items = sanitize_output_items([{"index": 0, "url": video_url, "type": "video"}])
|
|
|
|
pricing_meta = response.get("pricing_meta") if isinstance(response.get("pricing_meta"), Mapping) else {}
|
|
requested_output_count = max(
|
|
1,
|
|
safe_int(
|
|
pricing_meta.get("requested_output_count"),
|
|
safe_int(getattr(owner, "output_count", None), safe_int(getattr(owner, "count", None), 1)),
|
|
),
|
|
)
|
|
output_duration = max(0.0, safe_float(getattr(owner, "duration", None)))
|
|
fps = max(0.0, safe_float(getattr(owner, "fps", None), safe_float(getattr(owner, "frame_rate", None))))
|
|
generate_audio = safe_bool(getattr(owner, "generate_audio", None), safe_bool(response.get("generate_audio")))
|
|
inference_mode = str(
|
|
getattr(owner, "inference_mode", None)
|
|
or getattr(owner, "service_tier", None)
|
|
or response.get("service_tier")
|
|
or "online"
|
|
).lower()
|
|
|
|
counts = {
|
|
"requested_output_count": requested_output_count,
|
|
"generated_image_count": generated_image_count,
|
|
"generated_video_count": generated_video_count,
|
|
"generated_total_count": generated_image_count + generated_video_count,
|
|
}
|
|
snapshot = {
|
|
"schema_version": 1,
|
|
"stage": stage or "unknown",
|
|
"gen_type": gen_type,
|
|
"requested_output_count": requested_output_count,
|
|
"generated_image_count": generated_image_count,
|
|
"generated_video_count": generated_video_count,
|
|
"generated_total_count": generated_image_count + generated_video_count,
|
|
"output_items": output_items,
|
|
"duration_seconds": output_duration or None,
|
|
"resolution": resolution,
|
|
"aspect_ratio": aspect_ratio,
|
|
"width": width or None,
|
|
"height": height or None,
|
|
"dimension_source": dimension_source,
|
|
"fps": fps or None,
|
|
"generate_audio": generate_audio,
|
|
"inference_mode": inference_mode,
|
|
}
|
|
usage = {
|
|
"requested_output_count": requested_output_count,
|
|
"generated_image_count": generated_image_count,
|
|
"generated_video_count": generated_video_count,
|
|
"successful_output_count": generated_image_count if gen_type == "image" else generated_video_count,
|
|
"output_items": output_items,
|
|
"output_width": width,
|
|
"output_height": height,
|
|
"dimension_source": dimension_source,
|
|
"output_video_duration_seconds": output_duration,
|
|
"resolution": str(resolution or "").lower(),
|
|
"aspect_ratio": str(aspect_ratio or ""),
|
|
"fps": fps,
|
|
"generate_audio": generate_audio,
|
|
"inference_mode": inference_mode,
|
|
"usage_stage": stage or "unknown",
|
|
}
|
|
return snapshot, counts, usage
|