arsam
82096c51e3
Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.
59 lines
1.8 KiB
Python
59 lines
1.8 KiB
Python
import time
|
|
|
|
from fastapi import APIRouter
|
|
|
|
from axon.config import settings
|
|
from axon.models import MapRequest, MapResponse
|
|
from axon.services.scraper import discover_links, fetch_html, normalize_url, same_site
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def _matches_instructions(url: str, instructions: str | None) -> bool:
|
|
if not instructions:
|
|
return True
|
|
terms = [term for term in instructions.lower().split() if len(term) > 2]
|
|
return any(term in url.lower() for term in terms)
|
|
|
|
|
|
@router.post("/map", response_model=MapResponse)
|
|
async def map_site(request: MapRequest) -> MapResponse:
|
|
start = time.monotonic()
|
|
start_url = normalize_url(request.url)
|
|
max_pages = min(request.max_pages, settings.crawl_max_pages)
|
|
max_depth = min(request.max_depth, settings.crawl_max_depth)
|
|
|
|
queue: list[tuple[str, int]] = [(start_url, 0)]
|
|
queued = {start_url}
|
|
visited: set[str] = set()
|
|
urls: list[str] = []
|
|
|
|
while queue and len(visited) < max_pages:
|
|
url, depth = queue.pop(0)
|
|
if url in visited:
|
|
continue
|
|
visited.add(url)
|
|
|
|
if _matches_instructions(url, request.instructions):
|
|
urls.append(url)
|
|
|
|
if depth >= max_depth:
|
|
continue
|
|
|
|
html_text = await fetch_html(url, timeout=settings.extract_timeout)
|
|
if html_text is None:
|
|
continue
|
|
|
|
for link in await discover_links(url, html_text, limit=100):
|
|
if len(queued) >= max_pages * 4:
|
|
break
|
|
if same_site(link, start_url) and link not in queued and link not in visited:
|
|
queued.add(link)
|
|
queue.append((link, depth + 1))
|
|
|
|
return MapResponse(
|
|
start_url=start_url,
|
|
urls=urls[:max_pages],
|
|
response_time=round(time.monotonic() - start, 3),
|
|
)
|