import json import logging from fastapi import APIRouter from axon.config import settings from axon.models import ExtractRequest, ExtractResponse, ExtractResult from axon.services.cache import get_cache, make_cache_key from axon.services.scraper import extract_url router = APIRouter() logger = logging.getLogger(__name__) async def _extract_one(url: str, include_images: bool) -> ExtractResult: cache = await get_cache() cache_key = make_cache_key("extract", url=url, include_images=include_images) cached = await cache.get(cache_key) if cached: return ExtractResult(**json.loads(cached)) try: content, title, error = await extract_url( url, include_images=include_images, timeout=settings.extract_timeout, ) if error: return ExtractResult( url=url, title=title, success=False, error=error, ) result = ExtractResult( url=url, title=title, content=content or "", success=True, error=None, ) await cache.set(cache_key, result.model_dump_json(), ttl=3600) return result except Exception as exc: logger.debug("Extract failed for %s: %s", url, exc) return ExtractResult(url=url, success=False, error=str(exc)) @router.post("/extract", response_model=ExtractResponse) async def extract(request: ExtractRequest) -> ExtractResponse: import asyncio results = await asyncio.gather( *[_extract_one(url, request.include_images) for url in request.urls] ) return ExtractResponse(results=list(results))