arsam
ef62de2410
Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.
78 lines
2.5 KiB
Python
78 lines
2.5 KiB
Python
import time
|
|
|
|
from fastapi import APIRouter
|
|
|
|
from axon.config import settings
|
|
from axon.models import CrawlRequest, CrawlResponse, CrawlResult
|
|
from axon.services.scraper import (
|
|
discover_links,
|
|
extract_html,
|
|
fetch_html,
|
|
normalize_url,
|
|
same_site,
|
|
)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def _matches_instructions(url: str, title: str | None, instructions: str | None) -> bool:
|
|
if not instructions:
|
|
return True
|
|
terms = [term for term in instructions.lower().split() if len(term) > 2]
|
|
haystack = f"{url} {title or ''}".lower()
|
|
return any(term in haystack for term in terms)
|
|
|
|
|
|
@router.post("/crawl", response_model=CrawlResponse)
|
|
async def crawl(request: CrawlRequest) -> CrawlResponse:
|
|
start = time.monotonic()
|
|
start_url = normalize_url(request.url)
|
|
max_pages = min(request.max_pages, settings.crawl_max_pages)
|
|
max_depth = min(request.max_depth, settings.crawl_max_depth)
|
|
|
|
queue: list[tuple[str, int]] = [(start_url, 0)]
|
|
queued = {start_url}
|
|
visited: set[str] = set()
|
|
results: list[CrawlResult] = []
|
|
|
|
while queue and len(results) < max_pages:
|
|
url, depth = queue.pop(0)
|
|
if url in visited:
|
|
continue
|
|
visited.add(url)
|
|
|
|
html_text = await fetch_html(url, timeout=settings.extract_timeout)
|
|
if html_text is None:
|
|
results.append(CrawlResult(url=url, depth=depth, success=False, error="Fetch failed or unsafe URL"))
|
|
continue
|
|
|
|
content, title = await extract_html(html_text)
|
|
links = await discover_links(url, html_text, limit=100)
|
|
|
|
if _matches_instructions(url, title, request.instructions):
|
|
results.append(
|
|
CrawlResult(
|
|
url=url,
|
|
title=title,
|
|
content=(content or "") if request.include_raw_content else (content or "")[:500],
|
|
links=links,
|
|
depth=depth,
|
|
success=bool(content),
|
|
error=None if content else "No extractable content found",
|
|
)
|
|
)
|
|
|
|
if depth < max_depth:
|
|
for link in links:
|
|
if len(queued) >= max_pages * 4:
|
|
break
|
|
if same_site(link, start_url) and link not in queued and link not in visited:
|
|
queued.add(link)
|
|
queue.append((link, depth + 1))
|
|
|
|
return CrawlResponse(
|
|
start_url=start_url,
|
|
results=results[:max_pages],
|
|
response_time=round(time.monotonic() - start, 3),
|
|
)
|