diff --git a/crawler.go b/crawler.go new file mode 100644 index 0000000..395349f --- /dev/null +++ b/crawler.go @@ -0,0 +1,150 @@ +package reactor_crw + +import ( + "fmt" + "io" + "strconv" + + "reactor-crw/parser" +) + +// Transport is an interface that wraps all network related operations performed +// by crawler. +type Transport interface { + FetchData(url string) (io.ReadCloser, error) +} + +// HtmlCrawler allows crawling HTML pages using provided transport.Transport and +// parser.Parser. The crawler doesn't do anything with the content itself it only +// gathers the collection of sources links. +type HtmlCrawler struct { + // Transport performs all network request. + Transport Transport + + // Parser contains parser.Parser implementation for HTML. + Parser parser.Parser + + // MultiPage allows applying the crawler for multiple pages depending on its + // value. If its value set to true the crawler will try to calculate the + // number of available pages using the source page. + MultiPage bool +} + +// Fetch retrieves content sources from the page using the path value. Depending +// on HtmlCrawler.MultiPage it may fetch content links from multiple pages. +func (c *HtmlCrawler) Fetch(path string, search []string) (parser.QueryResult, error) { + if c.MultiPage { + return c.multiPage(path, search) + } + + collectedData := make(parser.QueryResult) + + err := c.fetch(path, search, collectedData) + if err != nil { + return nil, err + } + + return collectedData, nil +} + +// fetchMultiPage will fetch content sources from multiple pages. It'll try to +// retrieve the number of pages and consequently crawl all pages. +func (c *HtmlCrawler) multiPage(path string, search []string) (parser.QueryResult, error) { + maxPage, err := c.resolveMaxPage(path) + if err != nil { + return nil, err + } + + collectedData := make(parser.QueryResult) + + for p := 1; p <= maxPage; p++ { + err = c.fetch(fmt.Sprintf("%s/%d", path, p), search, collectedData) + if err != nil { + return nil, err + } + } + + return collectedData, nil +} + +func (c *HtmlCrawler) fetch(path string, search []string, qr parser.QueryResult) error { + body, err := c.Transport.FetchData(path) + if err != nil { + return err + } + + defer func(b io.ReadCloser) { + _ = b.Close() + }(body) + + err = c.Parser.FindAttrMap(body, buildQuery(search), qr) + if err != nil { + return fmt.Errorf("cannot apply crawler: %w", err) + } + + return nil +} + +func (c *HtmlCrawler) resolveMaxPage(path string) (int, error) { + const htmlPagination = ".pagination_expanded .current" + + body, err := c.Transport.FetchData(path) + if err != nil { + return 0, err + } + defer func(b io.ReadCloser) { + _ = b.Close() + }(body) + + pa, err := c.Parser.FindContent(body, htmlPagination) + if err != nil { + return 0, err + } + + intPa, err := strconv.Atoi(pa) + if err != nil { + return 0, fmt.Errorf("cannot prosess pagination: %w", err) + } + + return intPa, nil +} + +// query represents a search query that will be applied against the parser. +// Each query has a query itself to find corresponding elements, its attr, +// content of which should be returned, contentType marks a type of content +// and used for fetching specific types only. +type query struct { + contentType string + query string + attr string +} + +var queries = []query{ + {"image", ".post_content .image > img", "src"}, + {"image", ".post_content .image > a", "href"}, + {"gif", ".post_content .video_gif_source", "href"}, + {"mp4", ".post_content .video_gif source[type='video/mp4']", "src"}, + {"webm", ".post_content .video_gif source[type='video/webm']", "src"}, +} + +// buildQuery builds a parser.QueryAttrMap according to provided search list. +// The resulting parser.QueryAttrMap will contain only those queries that meet +// required content types. +// +// Example: buildQuery([]string{"image", "mp4"}). +func buildQuery(search []string) parser.QueryAttrMap { + qa := parser.QueryAttrMap{} + + searchMap := make(map[string]struct{}, len(search)) + for _, s := range search { + searchMap[s] = struct{}{} + } + + for _, sq := range queries { + if _, ok := searchMap[sq.contentType]; ok { + qa[sq.query] = sq.attr + } + } + + return qa +} diff --git a/crawler_test.go b/crawler_test.go new file mode 100644 index 0000000..a493be3 --- /dev/null +++ b/crawler_test.go @@ -0,0 +1,131 @@ +//go:build unit +// +build unit + +package reactor_crw + +import ( + "errors" + "io" + "io/ioutil" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "reactor-crw/parser" +) + +type transportMock struct { + mock.Mock +} + +func (m *transportMock) FetchData(url string) (io.ReadCloser, error) { + args := m.Called(url) + return args.Get(0).(io.ReadCloser), args.Error(1) +} + +type parserMock struct { + mock.Mock +} + +func (m *parserMock) FindContent(r io.Reader, q string) (string, error) { + args := m.Called(r, q) + return args.String(0), args.Error(1) +} + +func (m *parserMock) FindAttrMap(r io.Reader, qa parser.QueryAttrMap, qr parser.QueryResult) error { + args := m.Called(r, qa, qr) + return args.Error(0) +} + +func TestHtmlCrawler_Fetch(t *testing.T) { + path := "https://test.com/test/path" + + trp := &transportMock{} + prs := &parserMock{} + + t.Log("Given the need to crawl html page.") + { + t.Log("When multiple pages requested") + { + c := &HtmlCrawler{trp, prs, true} + + rc := ioutil.NopCloser(strings.NewReader("")) + trp.On("FetchData", path).Return(rc, nil).Once() + + prs.On("FindContent", rc, ".pagination_expanded .current").Return("2", nil) + trp.On("FetchData", path+"/1").Return(rc, nil).Once() + trp.On("FetchData", path+"/2").Return(rc, nil).Once() + + prs.On("FindAttrMap", rc, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + qr := args.Get(2).(parser.QueryResult) + qr["link_1"] = struct{}{} + }). + Return(nil). + Once() + + prs.On("FindAttrMap", rc, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + qr := args.Get(2).(parser.QueryResult) + qr["link_2"] = struct{}{} + }). + Return(nil). + Once() + + res, err := c.Fetch(path, []string{"image"}) + require.NoErrorf(t, err, "Wasn't expected an error during crawl") + require.Equal(t, parser.QueryResult{"link_1": struct{}{}, "link_2": struct{}{}}, res) + } + + t.Log("When parser returned an error") + { + expectedErr := errors.New("error") + c := &HtmlCrawler{trp, prs, false} + + rc := ioutil.NopCloser(strings.NewReader("")) + trp.On("FetchData", path).Return(rc, nil).Once() + + prs.On("FindAttrMap", rc, mock.Anything, mock.Anything). + Return(expectedErr). + Once() + + _, err := c.Fetch(path, []string{"image"}) + assert.ErrorIs(t, err, expectedErr) + } + + t.Log("When transport returned an error") + { + expectedErr := errors.New("error") + c := &HtmlCrawler{trp, prs, false} + + rc := ioutil.NopCloser(strings.NewReader("")) + trp.On("FetchData", path).Return(rc, expectedErr).Once() + + _, err := c.Fetch(path, []string{"image"}) + assert.ErrorIs(t, err, expectedErr) + } + + t.Log("When single page requested") + { + c := &HtmlCrawler{trp, prs, false} + + rc := ioutil.NopCloser(strings.NewReader("")) + trp.On("FetchData", path).Return(rc, nil).Once() + + prs.On("FindAttrMap", rc, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + qr := args.Get(2).(parser.QueryResult) + qr["link_1"] = struct{}{} + }). + Return(nil). + Once() + + res, err := c.Fetch(path, nil) + require.NoErrorf(t, err, "Wasn't expected an error during crawl") + require.Equal(t, parser.QueryResult{"link_1": struct{}{}}, res) + } + } +}