Import source: crawler

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

crawler.go

@@ -0,0 +1,150 @@
+package reactor_crw
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+
+	"reactor-crw/parser"
+)
+
+// Transport is an interface that wraps all network related operations performed
+// by crawler.
+type Transport interface {
+	FetchData(url string) (io.ReadCloser, error)
+}
+
+// HtmlCrawler allows crawling HTML pages using provided transport.Transport and
+// parser.Parser. The crawler doesn't do anything with the content itself it only
+// gathers the collection of sources links.
+type HtmlCrawler struct {
+	// Transport performs all network request.
+	Transport Transport
+
+	// Parser contains parser.Parser implementation for HTML.
+	Parser parser.Parser
+
+	// MultiPage allows applying the crawler for multiple pages depending on its
+	// value. If its value set to true the crawler will try to calculate the
+	// number of available pages using the source page.
+	MultiPage bool
+}
+
+// Fetch retrieves content sources from the page using the path value. Depending
+// on HtmlCrawler.MultiPage it may fetch content links from multiple pages.
+func (c *HtmlCrawler) Fetch(path string, search []string) (parser.QueryResult, error) {
+	if c.MultiPage {
+		return c.multiPage(path, search)
+	}
+
+	collectedData := make(parser.QueryResult)
+
+	err := c.fetch(path, search, collectedData)
+	if err != nil {
+		return nil, err
+	}
+
+	return collectedData, nil
+}
+
+// fetchMultiPage will fetch content sources from multiple pages. It'll try to
+// retrieve the number of pages and consequently crawl all pages.
+func (c *HtmlCrawler) multiPage(path string, search []string) (parser.QueryResult, error) {
+	maxPage, err := c.resolveMaxPage(path)
+	if err != nil {
+		return nil, err
+	}
+
+	collectedData := make(parser.QueryResult)
+
+	for p := 1; p <= maxPage; p++ {
+		err = c.fetch(fmt.Sprintf("%s/%d", path, p), search, collectedData)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return collectedData, nil
+}
+
+func (c *HtmlCrawler) fetch(path string, search []string, qr parser.QueryResult) error {
+	body, err := c.Transport.FetchData(path)
+	if err != nil {
+		return err
+	}
+
+	defer func(b io.ReadCloser) {
+		_ = b.Close()
+	}(body)
+
+	err = c.Parser.FindAttrMap(body, buildQuery(search), qr)
+	if err != nil {
+		return fmt.Errorf("cannot apply crawler: %w", err)
+	}
+
+	return nil
+}
+
+func (c *HtmlCrawler) resolveMaxPage(path string) (int, error) {
+	const htmlPagination = ".pagination_expanded .current"
+
+	body, err := c.Transport.FetchData(path)
+	if err != nil {
+		return 0, err
+	}
+	defer func(b io.ReadCloser) {
+		_ = b.Close()
+	}(body)
+
+	pa, err := c.Parser.FindContent(body, htmlPagination)
+	if err != nil {
+		return 0, err
+	}
+
+	intPa, err := strconv.Atoi(pa)
+	if err != nil {
+		return 0, fmt.Errorf("cannot prosess pagination: %w", err)
+	}
+
+	return intPa, nil
+}
+
+// query represents a search query that will be applied against the parser.
+// Each query has a query itself to find corresponding elements, its attr,
+// content of which should be returned, contentType marks a type of content
+// and used for fetching specific types only.
+type query struct {
+	contentType string
+	query       string
+	attr        string
+}
+
+var queries = []query{
+	{"image", ".post_content .image > img", "src"},
+	{"image", ".post_content .image > a", "href"},
+	{"gif", ".post_content .video_gif_source", "href"},
+	{"mp4", ".post_content .video_gif source[type='video/mp4']", "src"},
+	{"webm", ".post_content .video_gif source[type='video/webm']", "src"},
+}
+
+// buildQuery builds a parser.QueryAttrMap according to provided search list.
+// The resulting parser.QueryAttrMap will contain only those queries that meet
+// required content types.
+//
+// Example: buildQuery([]string{"image", "mp4"}).
+func buildQuery(search []string) parser.QueryAttrMap {
+	qa := parser.QueryAttrMap{}
+
+	searchMap := make(map[string]struct{}, len(search))
+	for _, s := range search {
+		searchMap[s] = struct{}{}
+	}
+
+	for _, sq := range queries {
+		if _, ok := searchMap[sq.contentType]; ok {
+			qa[sq.query] = sq.attr
+		}
+	}
+
+	return qa
+}

crawler_test.go

@@ -0,0 +1,131 @@
+//go:build unit
+// +build unit
+
+package reactor_crw
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"reactor-crw/parser"
+)
+
+type transportMock struct {
+	mock.Mock
+}
+
+func (m *transportMock) FetchData(url string) (io.ReadCloser, error) {
+	args := m.Called(url)
+	return args.Get(0).(io.ReadCloser), args.Error(1)
+}
+
+type parserMock struct {
+	mock.Mock
+}
+
+func (m *parserMock) FindContent(r io.Reader, q string) (string, error) {
+	args := m.Called(r, q)
+	return args.String(0), args.Error(1)
+}
+
+func (m *parserMock) FindAttrMap(r io.Reader, qa parser.QueryAttrMap, qr parser.QueryResult) error {
+	args := m.Called(r, qa, qr)
+	return args.Error(0)
+}
+
+func TestHtmlCrawler_Fetch(t *testing.T) {
+	path := "https://test.com/test/path"
+
+	trp := &transportMock{}
+	prs := &parserMock{}
+
+	t.Log("Given the need to crawl html page.")
+	{
+		t.Log("When multiple pages requested")
+		{
+			c := &HtmlCrawler{trp, prs, true}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, nil).Once()
+
+			prs.On("FindContent", rc, ".pagination_expanded .current").Return("2", nil)
+			trp.On("FetchData", path+"/1").Return(rc, nil).Once()
+			trp.On("FetchData", path+"/2").Return(rc, nil).Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Run(func(args mock.Arguments) {
+					qr := args.Get(2).(parser.QueryResult)
+					qr["link_1"] = struct{}{}
+				}).
+				Return(nil).
+				Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Run(func(args mock.Arguments) {
+					qr := args.Get(2).(parser.QueryResult)
+					qr["link_2"] = struct{}{}
+				}).
+				Return(nil).
+				Once()
+
+			res, err := c.Fetch(path, []string{"image"})
+			require.NoErrorf(t, err, "Wasn't expected an error during crawl")
+			require.Equal(t, parser.QueryResult{"link_1": struct{}{}, "link_2": struct{}{}}, res)
+		}
+
+		t.Log("When parser returned an error")
+		{
+			expectedErr := errors.New("error")
+			c := &HtmlCrawler{trp, prs, false}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, nil).Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Return(expectedErr).
+				Once()
+
+			_, err := c.Fetch(path, []string{"image"})
+			assert.ErrorIs(t, err, expectedErr)
+		}
+
+		t.Log("When transport returned an error")
+		{
+			expectedErr := errors.New("error")
+			c := &HtmlCrawler{trp, prs, false}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, expectedErr).Once()
+
+			_, err := c.Fetch(path, []string{"image"})
+			assert.ErrorIs(t, err, expectedErr)
+		}
+
+		t.Log("When single page requested")
+		{
+			c := &HtmlCrawler{trp, prs, false}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, nil).Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Run(func(args mock.Arguments) {
+					qr := args.Get(2).(parser.QueryResult)
+					qr["link_1"] = struct{}{}
+				}).
+				Return(nil).
+				Once()
+
+			res, err := c.Fetch(path, nil)
+			require.NoErrorf(t, err, "Wasn't expected an error during crawl")
+			require.Equal(t, parser.QueryResult{"link_1": struct{}{}}, res)
+		}
+	}
+}