arsam
d748376603
Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.
59 lines
1.7 KiB
Python
59 lines
1.7 KiB
Python
import json
|
|
import logging
|
|
|
|
from fastapi import APIRouter
|
|
|
|
from axon.config import settings
|
|
from axon.models import ExtractRequest, ExtractResponse, ExtractResult
|
|
from axon.services.cache import get_cache, make_cache_key
|
|
from axon.services.scraper import extract_url
|
|
|
|
router = APIRouter()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def _extract_one(url: str, include_images: bool) -> ExtractResult:
|
|
cache = await get_cache()
|
|
cache_key = make_cache_key("extract", url=url, include_images=include_images)
|
|
|
|
cached = await cache.get(cache_key)
|
|
if cached:
|
|
return ExtractResult(**json.loads(cached))
|
|
|
|
try:
|
|
content, title, error = await extract_url(
|
|
url,
|
|
include_images=include_images,
|
|
timeout=settings.extract_timeout,
|
|
)
|
|
if error:
|
|
return ExtractResult(
|
|
url=url,
|
|
title=title,
|
|
success=False,
|
|
error=error,
|
|
)
|
|
|
|
result = ExtractResult(
|
|
url=url,
|
|
title=title,
|
|
content=content or "",
|
|
success=True,
|
|
error=None,
|
|
)
|
|
await cache.set(cache_key, result.model_dump_json(), ttl=3600)
|
|
return result
|
|
except Exception as exc:
|
|
logger.debug("Extract failed for %s: %s", url, exc)
|
|
return ExtractResult(url=url, success=False, error=str(exc))
|
|
|
|
|
|
@router.post("/extract", response_model=ExtractResponse)
|
|
async def extract(request: ExtractRequest) -> ExtractResponse:
|
|
import asyncio
|
|
|
|
results = await asyncio.gather(
|
|
*[_extract_one(url, request.include_images) for url in request.urls]
|
|
)
|
|
return ExtractResponse(results=list(results))
|