Import source: axon/routers/map

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

axon/routers/map.py

@@ -0,0 +1,58 @@
+import time
+
+from fastapi import APIRouter
+
+from axon.config import settings
+from axon.models import MapRequest, MapResponse
+from axon.services.scraper import discover_links, fetch_html, normalize_url, same_site
+
+router = APIRouter()
+
+
+def _matches_instructions(url: str, instructions: str | None) -> bool:
+    if not instructions:
+        return True
+    terms = [term for term in instructions.lower().split() if len(term) > 2]
+    return any(term in url.lower() for term in terms)
+
+
+@router.post("/map", response_model=MapResponse)
+async def map_site(request: MapRequest) -> MapResponse:
+    start = time.monotonic()
+    start_url = normalize_url(request.url)
+    max_pages = min(request.max_pages, settings.crawl_max_pages)
+    max_depth = min(request.max_depth, settings.crawl_max_depth)
+
+    queue: list[tuple[str, int]] = [(start_url, 0)]
+    queued = {start_url}
+    visited: set[str] = set()
+    urls: list[str] = []
+
+    while queue and len(visited) < max_pages:
+        url, depth = queue.pop(0)
+        if url in visited:
+            continue
+        visited.add(url)
+
+        if _matches_instructions(url, request.instructions):
+            urls.append(url)
+
+        if depth >= max_depth:
+            continue
+
+        html_text = await fetch_html(url, timeout=settings.extract_timeout)
+        if html_text is None:
+            continue
+
+        for link in await discover_links(url, html_text, limit=100):
+            if len(queued) >= max_pages * 4:
+                break
+            if same_site(link, start_url) and link not in queued and link not in visited:
+                queued.add(link)
+                queue.append((link, depth + 1))
+
+    return MapResponse(
+        start_url=start_url,
+        urls=urls[:max_pages],
+        response_time=round(time.monotonic() - start, 3),
+    )