import re from fastapi import Request, Response from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from starlette.responses import JSONResponse from app.config import settings BLOCKED_UA_PATTERNS = [ re.compile(r"(?i)(curl|wget|python-requests|scrapy|httpx|go-http-client|java/)"), ] class AntiCrawlerMiddleware(BaseHTTPMiddleware): async def dispatch( self, request: Request, call_next: RequestResponseEndpoint ) -> Response: # Skip health/docs if request.url.path in ("/health", "/api-docs", "/openapi.json", "/api-redoc"): return await call_next(request) # Skip callback endpoints if request.url.path.startswith("/api/callbacks/"): return await call_next(request) user_agent = request.headers.get("user-agent", "") # Block empty User-Agent if not user_agent: return JSONResponse(status_code=403, content={"detail": "Forbidden"}) # Block known bot User-Agents (unless they have an API key) api_key = request.headers.get("x-api-key") if not api_key: for pattern in BLOCKED_UA_PATTERNS: if pattern.search(user_agent): return JSONResponse( status_code=403, content={"detail": "Forbidden"} ) return await call_next(request)