Import source: axon/routers/crawl

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

axon/routers/crawl.py

@@ -0,0 +1,77 @@
+import time
+
+from fastapi import APIRouter
+
+from axon.config import settings
+from axon.models import CrawlRequest, CrawlResponse, CrawlResult
+from axon.services.scraper import (
+    discover_links,
+    extract_html,
+    fetch_html,
+    normalize_url,
+    same_site,
+)
+
+router = APIRouter()
+
+
+def _matches_instructions(url: str, title: str | None, instructions: str | None) -> bool:
+    if not instructions:
+        return True
+    terms = [term for term in instructions.lower().split() if len(term) > 2]
+    haystack = f"{url} {title or ''}".lower()
+    return any(term in haystack for term in terms)
+
+
+@router.post("/crawl", response_model=CrawlResponse)
+async def crawl(request: CrawlRequest) -> CrawlResponse:
+    start = time.monotonic()
+    start_url = normalize_url(request.url)
+    max_pages = min(request.max_pages, settings.crawl_max_pages)
+    max_depth = min(request.max_depth, settings.crawl_max_depth)
+
+    queue: list[tuple[str, int]] = [(start_url, 0)]
+    queued = {start_url}
+    visited: set[str] = set()
+    results: list[CrawlResult] = []
+
+    while queue and len(results) < max_pages:
+        url, depth = queue.pop(0)
+        if url in visited:
+            continue
+        visited.add(url)
+
+        html_text = await fetch_html(url, timeout=settings.extract_timeout)
+        if html_text is None:
+            results.append(CrawlResult(url=url, depth=depth, success=False, error="Fetch failed or unsafe URL"))
+            continue
+
+        content, title = await extract_html(html_text)
+        links = await discover_links(url, html_text, limit=100)
+
+        if _matches_instructions(url, title, request.instructions):
+            results.append(
+                CrawlResult(
+                    url=url,
+                    title=title,
+                    content=(content or "") if request.include_raw_content else (content or "")[:500],
+                    links=links,
+                    depth=depth,
+                    success=bool(content),
+                    error=None if content else "No extractable content found",
+                )
+            )
+
+        if depth < max_depth:
+            for link in links:
+                if len(queued) >= max_pages * 4:
+                    break
+                if same_site(link, start_url) and link not in queued and link not in visited:
+                    queued.add(link)
+                    queue.append((link, depth + 1))
+
+    return CrawlResponse(
+        start_url=start_url,
+        results=results[:max_pages],
+        response_time=round(time.monotonic() - start, 3),
+    )