from __future__ import annotations import json import logging import os import re import traceback from datetime import datetime from typing import Any from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from app.services.log_config import LOG_DATE_FORMAT, LOG_DIR, is_enabled logger = logging.getLogger("video_gen") MAX_LOG_FIELD_LENGTH = 20000 MAX_TRACEBACK_LENGTH = 12000 LOG_BASE_DIR = os.path.dirname(LOG_DIR) OPERATION_LOG_ROOT = os.path.join(LOG_BASE_DIR, "OperationLogs") MODULE_GENERATION_LOG_ROOT = os.path.join(LOG_BASE_DIR, "ModuleGeneration") AI_MODEL_LOG_ROOT = LOG_DIR SENSITIVE_KEY_PATTERNS = ( "secret", "token", "authorization", "cookie", "credential", "signature", "accesskey", "access_key", "api_key", "apikey", "security-token", "x-tos-security-token", ) def _safe_name(value: str | None, default: str = "unknown") -> str: text = str(value or default).strip() or default text = re.sub(r"[^a-zA-Z0-9_.-]+", "_", text) return text[:120] or default def _mask_string(value: str) -> str: if len(value) <= 8: return "***" return f"{value[:4]}***{value[-4:]}" def _is_sensitive_key(key: str) -> bool: lower = str(key).replace("-", "_").lower() return lower == "sign" or any(pattern in lower for pattern in SENSITIVE_KEY_PATTERNS) def _sanitize_url(value: str) -> str: try: parts = urlsplit(value) if not parts.scheme or not parts.netloc: return value query = [] for k, v in parse_qsl(parts.query, keep_blank_values=True): query.append((k, _mask_string(v) if _is_sensitive_key(k) else v)) return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment)) except Exception: return value def sanitize_log_value(value: Any) -> Any: if value is None: return None if isinstance(value, str): text = _sanitize_url(value) if value.startswith(("http://", "https://")) else value if len(text) > MAX_LOG_FIELD_LENGTH: return text[:MAX_LOG_FIELD_LENGTH] + f"..." return text if isinstance(value, dict): output: dict[str, Any] = {} for k, v in value.items(): key = str(k) output[key] = "***" if _is_sensitive_key(key) else sanitize_log_value(v) return output if isinstance(value, list): return [sanitize_log_value(v) for v in value] return value def build_exception_detail(exc: BaseException | None, extra: dict[str, Any] | None = None) -> dict[str, Any]: detail: dict[str, Any] = dict(extra or {}) if exc is not None: tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) if len(tb) > MAX_TRACEBACK_LENGTH: tb = tb[:MAX_TRACEBACK_LENGTH] + f"..." detail.update( { "exception_type": type(exc).__name__, "exception_message": str(exc), "traceback": tb, } ) return detail def _append_json_log(root_dir: str, domain: str | None, entry: dict[str, Any]) -> None: if not is_enabled(): return try: target_dir = root_dir if domain is None else os.path.join(root_dir, _safe_name(domain, "default")) os.makedirs(target_dir, exist_ok=True) today = datetime.now().strftime(LOG_DATE_FORMAT) with open(os.path.join(target_dir, f"{today}.log"), "a", encoding="utf-8") as f: f.write(json.dumps(sanitize_log_value(entry), ensure_ascii=False, default=str) + "\n") except Exception as exc: logger.warning("operation log write failed: root_dir=%s domain=%s error=%s", root_dir, domain, exc, exc_info=True) def _append_operation_log(domain: str, entry: dict[str, Any]) -> None: _append_json_log(OPERATION_LOG_ROOT, domain, entry) def _base_entry( *, log_type: str, domain: str, event_type: str, module: str | None = None, event_status: str = "success", source: str | None = None, trace_id: str | None = None, request_id: str | None = None, user_id: str | None = None, project_id: str | None = None, session_id: str | None = None, group_id: str | None = None, asset_id: str | None = None, task_id: str | None = None, step_id: str | None = None, remote_action: str | None = None, remote_request_id: str | None = None, message: str | None = None, detail: dict[str, Any] | None = None, error: str | None = None, ) -> dict[str, Any]: return { "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "log_type": log_type, "domain": domain, "module": module or domain, "event_type": event_type, "event_status": event_status, "source": source, "trace_id": trace_id, "request_id": request_id, "user_id": user_id, "project_id": project_id, "session_id": session_id, "group_id": group_id, "asset_id": asset_id, "task_id": task_id, "step_id": step_id, "remote_action": remote_action, "remote_request_id": remote_request_id, "message": message, "detail": detail or {}, "error": error, } def log_operation_event( *, domain: str, event_type: str, module: str | None = None, event_status: str = "success", source: str | None = None, trace_id: str | None = None, request_id: str | None = None, user_id: str | None = None, project_id: str | None = None, session_id: str | None = None, group_id: str | None = None, asset_id: str | None = None, task_id: str | None = None, step_id: str | None = None, remote_action: str | None = None, remote_request_id: str | None = None, message: str | None = None, detail: dict[str, Any] | None = None, error: str | None = None, ) -> None: _append_operation_log( domain, _base_entry( log_type="operation_event", domain=domain, module=module, event_type=event_type, event_status=event_status, source=source, trace_id=trace_id, request_id=request_id, user_id=user_id, project_id=project_id, session_id=session_id, group_id=group_id, asset_id=asset_id, task_id=task_id, step_id=step_id, remote_action=remote_action, remote_request_id=remote_request_id, message=message, detail=detail, error=error, ), ) def log_module_generation_event( *, module: str, event_type: str, event_status: str = "success", source: str | None = None, trace_id: str | None = None, request_id: str | None = None, user_id: str | None = None, project_id: str | None = None, task_id: str | None = None, step_id: str | None = None, remote_action: str | None = None, remote_request_id: str | None = None, message: str | None = None, detail: dict[str, Any] | None = None, error: str | None = None, ) -> None: """Write module business logs to log/ModuleGeneration/{module}/YYYY-MM-DD.log.""" _append_json_log( MODULE_GENERATION_LOG_ROOT, module, _base_entry( log_type="module_generation_event", domain="module_generation", module=module, event_type=event_type, event_status=event_status, source=source, trace_id=trace_id, request_id=request_id, user_id=user_id, project_id=project_id, task_id=task_id, step_id=step_id, remote_action=remote_action, remote_request_id=remote_request_id, message=message, detail=detail, error=error, ), ) def log_ai_model_event( *, event_type: str, module: str | None = None, event_status: str = "success", source: str | None = None, trace_id: str | None = None, request_id: str | None = None, user_id: str | None = None, project_id: str | None = None, task_id: str | None = None, step_id: str | None = None, remote_action: str | None = None, remote_request_id: str | None = None, model_config_id: str | None = None, model_config_name: str | None = None, model_name: str | None = None, provider: str | None = None, api_base: str | None = None, http_status: int | None = None, request: dict[str, Any] | None = None, response: dict[str, Any] | list[Any] | str | None = None, token_usage: dict[str, Any] | None = None, message: str | None = None, detail: dict[str, Any] | None = None, error: str | None = None, ) -> None: """Write AI model call logs to existing log/AiModel/YYYY-MM-DD.log.""" final_detail = dict(detail or {}) if request is not None: final_detail["request"] = request if response is not None: final_detail["response"] = response if token_usage is not None: final_detail["token_usage"] = token_usage entry = _base_entry( log_type="ai_model_event", domain="ai_model", module=module or "ai_model", event_type=event_type, event_status=event_status, source=source, trace_id=trace_id, request_id=request_id, user_id=user_id, project_id=project_id, task_id=task_id, step_id=step_id, remote_action=remote_action, remote_request_id=remote_request_id, message=message, detail=final_detail, error=error, ) entry.update( { "model_name": model_config_name, "model_id": model_name, "model_config_id": model_config_id, "provider": provider, "api_base": api_base, "http_status": http_status, } ) _append_json_log(AI_MODEL_LOG_ROOT, None, entry) def log_operation_error(*, domain: str, event_type: str, exc: BaseException | None = None, detail: dict[str, Any] | None = None, **kwargs: Any) -> None: kwargs.setdefault("event_status", "failed") kwargs["detail"] = build_exception_detail(exc, detail) kwargs.setdefault("error", str(exc) if exc is not None else None) log_operation_event(domain=domain, event_type=event_type, **kwargs) def log_remote_api_event( *, domain: str, remote_action: str, event_type: str, event_status: str, request: dict[str, Any] | None = None, response: dict[str, Any] | None = None, remote_request_id: str | None = None, remote_code: str | None = None, remote_message: str | None = None, **kwargs: Any, ) -> None: detail = dict(kwargs.pop("detail", {}) or {}) if request is not None: detail["request"] = request if response is not None: detail["response"] = response if remote_code is not None: detail["remote_code"] = remote_code if remote_message is not None: detail["remote_message"] = remote_message log_operation_event( domain=domain, event_type=event_type, event_status=event_status, remote_action=remote_action, remote_request_id=remote_request_id, detail=detail, error=remote_message if event_status == "failed" else None, **kwargs, )