diff --git a/axon/routers/extract.py b/axon/routers/extract.py new file mode 100644 index 0000000..ee7d646 --- /dev/null +++ b/axon/routers/extract.py @@ -0,0 +1,58 @@ +import json +import logging + +from fastapi import APIRouter + +from axon.config import settings +from axon.models import ExtractRequest, ExtractResponse, ExtractResult +from axon.services.cache import get_cache, make_cache_key +from axon.services.scraper import extract_url + +router = APIRouter() +logger = logging.getLogger(__name__) + + +async def _extract_one(url: str, include_images: bool) -> ExtractResult: + cache = await get_cache() + cache_key = make_cache_key("extract", url=url, include_images=include_images) + + cached = await cache.get(cache_key) + if cached: + return ExtractResult(**json.loads(cached)) + + try: + content, title, error = await extract_url( + url, + include_images=include_images, + timeout=settings.extract_timeout, + ) + if error: + return ExtractResult( + url=url, + title=title, + success=False, + error=error, + ) + + result = ExtractResult( + url=url, + title=title, + content=content or "", + success=True, + error=None, + ) + await cache.set(cache_key, result.model_dump_json(), ttl=3600) + return result + except Exception as exc: + logger.debug("Extract failed for %s: %s", url, exc) + return ExtractResult(url=url, success=False, error=str(exc)) + + +@router.post("/extract", response_model=ExtractResponse) +async def extract(request: ExtractRequest) -> ExtractResponse: + import asyncio + + results = await asyncio.gather( + *[_extract_one(url, request.include_images) for url in request.urls] + ) + return ExtractResponse(results=list(results))