import time from fastapi import APIRouter from axon.config import settings from axon.models import MapRequest, MapResponse from axon.services.scraper import discover_links, fetch_html, normalize_url, same_site router = APIRouter() def _matches_instructions(url: str, instructions: str | None) -> bool: if not instructions: return True terms = [term for term in instructions.lower().split() if len(term) > 2] return any(term in url.lower() for term in terms) @router.post("/map", response_model=MapResponse) async def map_site(request: MapRequest) -> MapResponse: start = time.monotonic() start_url = normalize_url(request.url) max_pages = min(request.max_pages, settings.crawl_max_pages) max_depth = min(request.max_depth, settings.crawl_max_depth) queue: list[tuple[str, int]] = [(start_url, 0)] queued = {start_url} visited: set[str] = set() urls: list[str] = [] while queue and len(visited) < max_pages: url, depth = queue.pop(0) if url in visited: continue visited.add(url) if _matches_instructions(url, request.instructions): urls.append(url) if depth >= max_depth: continue html_text = await fetch_html(url, timeout=settings.extract_timeout) if html_text is None: continue for link in await discover_links(url, html_text, limit=100): if len(queued) >= max_pages * 4: break if same_site(link, start_url) and link not in queued and link not in visited: queued.add(link) queue.append((link, depth + 1)) return MapResponse( start_url=start_url, urls=urls[:max_pages], response_time=round(time.monotonic() - start, 3), )