reactor-crw/crawler.go

package reactor_crw

import (
	"fmt"
	"io"
	"strconv"

	"reactor-crw/parser"
)

// Transport is an interface that wraps all network related operations performed
// by crawler.
type Transport interface {
	FetchData(url string) (io.ReadCloser, error)
}

// HtmlCrawler allows crawling HTML pages using provided transport.Transport and
// parser.Parser. The crawler doesn't do anything with the content itself it only
// gathers the collection of sources links.
type HtmlCrawler struct {
	// Transport performs all network request.
	Transport Transport

	// Parser contains parser.Parser implementation for HTML.
	Parser parser.Parser

	// MultiPage allows applying the crawler for multiple pages depending on its
	// value. If its value set to true the crawler will try to calculate the
	// number of available pages using the source page.
	MultiPage bool
}

// Fetch retrieves content sources from the page using the path value. Depending
// on HtmlCrawler.MultiPage it may fetch content links from multiple pages.
func (c *HtmlCrawler) Fetch(path string, search []string) (parser.QueryResult, error) {
	if c.MultiPage {
		return c.multiPage(path, search)
	}

	collectedData := make(parser.QueryResult)

	err := c.fetch(path, search, collectedData)
	if err != nil {
		return nil, err
	}

	return collectedData, nil
}

// fetchMultiPage will fetch content sources from multiple pages. It'll try to
// retrieve the number of pages and consequently crawl all pages.
func (c *HtmlCrawler) multiPage(path string, search []string) (parser.QueryResult, error) {
	maxPage, err := c.resolveMaxPage(path)
	if err != nil {
		return nil, err
	}

	collectedData := make(parser.QueryResult)

	for p := 1; p <= maxPage; p++ {
		err = c.fetch(fmt.Sprintf("%s/%d", path, p), search, collectedData)
		if err != nil {
			return nil, err
		}
	}

	return collectedData, nil
}

func (c *HtmlCrawler) fetch(path string, search []string, qr parser.QueryResult) error {
	body, err := c.Transport.FetchData(path)
	if err != nil {
		return err
	}

	defer func(b io.ReadCloser) {
		_ = b.Close()
	}(body)

	err = c.Parser.FindAttrMap(body, buildQuery(search), qr)
	if err != nil {
		return fmt.Errorf("cannot apply crawler: %w", err)
	}

	return nil
}

func (c *HtmlCrawler) resolveMaxPage(path string) (int, error) {
	const htmlPagination = ".pagination_expanded .current"

	body, err := c.Transport.FetchData(path)
	if err != nil {
		return 0, err
	}
	defer func(b io.ReadCloser) {
		_ = b.Close()
	}(body)

	pa, err := c.Parser.FindContent(body, htmlPagination)
	if err != nil {
		return 0, err
	}

	intPa, err := strconv.Atoi(pa)
	if err != nil {
		return 0, fmt.Errorf("cannot prosess pagination: %w", err)
	}

	return intPa, nil
}

// query represents a search query that will be applied against the parser.
// Each query has a query itself to find corresponding elements, its attr,
// content of which should be returned, contentType marks a type of content
// and used for fetching specific types only.
type query struct {
	contentType string
	query       string
	attr        string
}

var queries = []query{
	{"image", ".post_content .image > img", "src"},
	{"image", ".post_content .image > a", "href"},
	{"gif", ".post_content .video_gif_source", "href"},
	{"mp4", ".post_content .video_gif source[type='video/mp4']", "src"},
	{"webm", ".post_content .video_gif source[type='video/webm']", "src"},
}

// buildQuery builds a parser.QueryAttrMap according to provided search list.
// The resulting parser.QueryAttrMap will contain only those queries that meet
// required content types.
//
// Example: buildQuery([]string{"image", "mp4"}).
func buildQuery(search []string) parser.QueryAttrMap {
	qa := parser.QueryAttrMap{}

	searchMap := make(map[string]struct{}, len(search))
	for _, s := range search {
		searchMap[s] = struct{}{}
	}

	for _, sq := range queries {
		if _, ok := searchMap[sq.contentType]; ok {
			qa[sq.query] = sq.attr
		}
	}

	return qa
}