Import documentation: README.MD

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

Makefile

@@ -0,0 +1,28 @@
+BINARY_NAME=reactor-crw
+BIN_DIR=bin
+BUILD_PATH=${BIN_DIR}/${BINARY_NAME}
+VERSION=0.0.1
+
+.PHONY: build clean test test-coverage dep vet
+
+build:
+	@mkdir -p $(BIN_DIR)
+	GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w" -o ${BUILD_PATH}_${VERSION}_macOS_64bit cmd/main.go && upx --best --lzma ${BUILD_PATH}_${VERSION}_macOS_64bit
+	GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o ${BUILD_PATH}_${VERSION}_Linux_64bit cmd/main.go && upx --best --lzma ${BUILD_PATH}_${VERSION}_Linux_64bit
+	GOOS=windows GOARCH=amd64 go build -ldflags "-s -w" -o ${BUILD_PATH}_${VERSION}_Windows_64bit.exe cmd/main.go && upx --best --lzma ${BUILD_PATH}_${VERSION}_Windows_64bit.exe
+
+clean:
+	@go clean
+	@rm -rf ${BIN_DIR}
+
+test:
+	go test ./... --tags unit
+
+test-coverage:
+	go test ./... --tags unit -coverprofile=coverage.out
+
+dep:
+	go mod download
+
+vet:
+	go vet

README.MD

@@ -0,0 +1,56 @@
+Reactor Crawler
+==========
+[![reactor-crw](https://github.com/reactor-joy/reactor-crw/actions/workflows/go.yml/badge.svg)](https://github.com/reactor-joy/reactor-crw/actions/workflows/go.yml)
+[![Go Report Card](https://goreportcard.com/badge/github.com/reactor-joy/reactor-crw)](https://goreportcard.com/report/github.com/reactor-joy/reactor-crw)
+
+Simple CLI content crawler for [Joyreactor](http://joyreactor.cc). He'll find all media content on the 
+page you've provided and save it. If there will be any kind of pagination... he'll go through all pages
+as well unless you'll tell him to not.
+
+<p>
+    <img src="https://i.imgur.com/DjD6cW0.gif" width="800" alt="reactor_crawler_example">
+</p>
+
+## Quick start
+
+Here's the quickest way to download something and test the crawler:
+* Download a build according to your OS.
+* Pick some URL from [Joyreactor](http://joyreactor.cc).
+* Run the crawler `$ reactor-crw -p "http://joyreactor.cc/tag/digital+art"`
+
+## What else
+
+There's a list of optional flags that adds a little more control over the crawler.
+
+```
+$ reactor-crw --help
+
+Allows to quickly download all content by its direct url or entire tag or fandom from joyreactor.cc.
+Example: reactor-crw -d "." -p "http://joyreactor.cc/tag/someTag/all" -w 2 -c "cookie-string"
+
+Usage:
+  reactor-crw [flags]
+
+Flags:
+  -c, --cookie string        User's cookie. Some content may be unavailable without it
+  -d, --destination string   Save path for content. Default value is a user's home folder
+                             (example C:\Users\username for Windows) (default "/home/avpretty")
+  -h, --help                 help for reactor-crw
+  -p, --path string          Provide a full page URL
+  -s, --search string        A comma separated list of content types that should be downloaded.
+                             Possible values: image,gif,webm,mp4. Example: -s "image,webm" (default "image,gif")
+  -o, --single-page          Crawl only one page
+  -w, --workers int          Amount of workers (default 1)
+```
+
+From all flags only `-p --path` is required. All other flags can be omitted and default values will be used.
+
+Here's another example:
+
+```
+$ reactor-crw -p "http://joyreactor.cc/post/000000" -d "." -s "mp4" -o -c "cookies from joyreactor"
+```
+This one will download only `mp4` content from the post and will save it to the current directory.
+`-o` means that only the current page will be parsed, and the user's cookie `-s` will be used by the crawler.
+
+**Note**: some content may be parsed only with user's cookie.

cmd/main.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"reactor-crw"
+	"reactor-crw/handler/fs"
+	"reactor-crw/parser"
+
+	"github.com/vbauerster/mpb/v7"
+	"github.com/vbauerster/mpb/v7/decor"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	search     string
+	path       string
+	savePath   string
+	cookie     string
+	maxWorkers int
+	singlePage bool
+
+	crawlerCmd = &cobra.Command{
+		Use:   "reactor-crw",
+		Short: "Grab your favorite content from joyreactor.cc",
+		Long: "Allows to quickly download all content by its direct url or entire" +
+			" tag or fandom from joyreactor.cc.\nExample: reactor-crw -d \".\" " +
+			"-p \"http://joyreactor.cc/tag/someTag/all\" -w 2 -c \"cookie-string\"",
+		Run: run,
+	}
+)
+
+func init() {
+	hd, _ := os.UserHomeDir()
+
+	crawlerCmd.Flags().StringVarP(&search, "search", "s", "image,gif", "A comma separated list of content types that should be downloaded.\nPossible values: image,gif,webm,mp4. Example: -s \"image,webm\"")
+	crawlerCmd.Flags().StringVarP(&path, "path", "p", "", "Provide a full page URL")
+	crawlerCmd.Flags().StringVarP(&savePath, "destination", "d", hd, "Save path for content. Default value is a user's home folder \n(example C:\\Users\\username for Windows)")
+	crawlerCmd.Flags().StringVarP(&cookie, "cookie", "c", "", "User's cookie. Some content may be unavailable without it")
+	crawlerCmd.Flags().IntVarP(&maxWorkers, "workers", "w", 1, "Amount of workers")
+	crawlerCmd.Flags().BoolVarP(&singlePage, "single-page", "o", false, "Crawl only one page")
+
+	_ = crawlerCmd.MarkFlagRequired("path")
+}
+
+func run(_ *cobra.Command, _ []string) {
+	start := time.Now()
+	t := reactor_crw.NewHttpTransport(&http.Client{}, reactor_crw.Headers{"Cookie": cookie})
+
+	pathUrl, err := url.Parse(path)
+	if err != nil {
+		log.Fatalf("invalid path provided: %s", path)
+	}
+
+	pr, err := fs.NewPathResolver(savePath)
+	if err != nil {
+		log.Fatalf("cannot process provided destination: %s", savePath)
+	}
+
+	ch, _ := fs.NewFileSaver(pr, t, strings.Replace(pathUrl.Path, "/", "_", -1))
+	c := reactor_crw.NewClient(
+		&reactor_crw.HtmlCrawler{
+			Transport: t,
+			Parser:    &parser.Html{},
+			MultiPage: !singlePage,
+		},
+		maxWorkers,
+		ch,
+	)
+
+	go func() {
+		err = c.Run(path, search)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	progress(c.TotalSources, c.Progress, c.Errors)
+
+	fmt.Printf("\n>>> Done in %s\n", time.Since(start).String())
+}
+
+func progress(total, task <-chan int, err <-chan error) {
+	fmt.Print("\n>>> Trying to count the amount of links. Please wait...\n\n")
+
+	t := <-total
+	if t == 0 {
+		fmt.Print(">>> No links were found. Stopping...\n")
+		return
+	}
+	fmt.Printf(">>> %d links were found. Start downloading\n\n", t)
+
+	prg := mpb.New(mpb.WithWidth(64))
+
+	name := ">>> Progress:"
+	bar := prg.AddBar(int64(t),
+		mpb.PrependDecorators(
+			decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}),
+			decor.CountersNoUnit("%d/%d", decor.WCSyncWidth),
+		),
+		mpb.AppendDecorators(decor.Percentage(decor.WC{W: 5})),
+	)
+
+	errBuf := strings.Builder{}
+
+	go func() {
+		for {
+			select {
+			case <-task:
+				bar.Increment()
+			case e := <-err:
+				if e != nil {
+					errBuf.WriteString(e.Error() + "\n")
+				}
+			}
+		}
+	}()
+
+	prg.Wait()
+
+	if errBuf.Len() != 0 {
+		fmt.Printf(">>> Errors during crawler work:\n%s", errBuf.String())
+	}
+}
+
+func main() {
+	err := crawlerCmd.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

crawler.go

@@ -0,0 +1,150 @@
+package reactor_crw
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+
+	"reactor-crw/parser"
+)
+
+// Transport is an interface that wraps all network related operations performed
+// by crawler.
+type Transport interface {
+	FetchData(url string) (io.ReadCloser, error)
+}
+
+// HtmlCrawler allows crawling HTML pages using provided transport.Transport and
+// parser.Parser. The crawler doesn't do anything with the content itself it only
+// gathers the collection of sources links.
+type HtmlCrawler struct {
+	// Transport performs all network request.
+	Transport Transport
+
+	// Parser contains parser.Parser implementation for HTML.
+	Parser parser.Parser
+
+	// MultiPage allows applying the crawler for multiple pages depending on its
+	// value. If its value set to true the crawler will try to calculate the
+	// number of available pages using the source page.
+	MultiPage bool
+}
+
+// Fetch retrieves content sources from the page using the path value. Depending
+// on HtmlCrawler.MultiPage it may fetch content links from multiple pages.
+func (c *HtmlCrawler) Fetch(path string, search []string) (parser.QueryResult, error) {
+	if c.MultiPage {
+		return c.multiPage(path, search)
+	}
+
+	collectedData := make(parser.QueryResult)
+
+	err := c.fetch(path, search, collectedData)
+	if err != nil {
+		return nil, err
+	}
+
+	return collectedData, nil
+}
+
+// fetchMultiPage will fetch content sources from multiple pages. It'll try to
+// retrieve the number of pages and consequently crawl all pages.
+func (c *HtmlCrawler) multiPage(path string, search []string) (parser.QueryResult, error) {
+	maxPage, err := c.resolveMaxPage(path)
+	if err != nil {
+		return nil, err
+	}
+
+	collectedData := make(parser.QueryResult)
+
+	for p := 1; p <= maxPage; p++ {
+		err = c.fetch(fmt.Sprintf("%s/%d", path, p), search, collectedData)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return collectedData, nil
+}
+
+func (c *HtmlCrawler) fetch(path string, search []string, qr parser.QueryResult) error {
+	body, err := c.Transport.FetchData(path)
+	if err != nil {
+		return err
+	}
+
+	defer func(b io.ReadCloser) {
+		_ = b.Close()
+	}(body)
+
+	err = c.Parser.FindAttrMap(body, buildQuery(search), qr)
+	if err != nil {
+		return fmt.Errorf("cannot apply crawler: %w", err)
+	}
+
+	return nil
+}
+
+func (c *HtmlCrawler) resolveMaxPage(path string) (int, error) {
+	const htmlPagination = ".pagination_expanded .current"
+
+	body, err := c.Transport.FetchData(path)
+	if err != nil {
+		return 0, err
+	}
+	defer func(b io.ReadCloser) {
+		_ = b.Close()
+	}(body)
+
+	pa, err := c.Parser.FindContent(body, htmlPagination)
+	if err != nil {
+		return 0, err
+	}
+
+	intPa, err := strconv.Atoi(pa)
+	if err != nil {
+		return 0, fmt.Errorf("cannot prosess pagination: %w", err)
+	}
+
+	return intPa, nil
+}
+
+// query represents a search query that will be applied against the parser.
+// Each query has a query itself to find corresponding elements, its attr,
+// content of which should be returned, contentType marks a type of content
+// and used for fetching specific types only.
+type query struct {
+	contentType string
+	query       string
+	attr        string
+}
+
+var queries = []query{
+	{"image", ".post_content .image > img", "src"},
+	{"image", ".post_content .image > a", "href"},
+	{"gif", ".post_content .video_gif_source", "href"},
+	{"mp4", ".post_content .video_gif source[type='video/mp4']", "src"},
+	{"webm", ".post_content .video_gif source[type='video/webm']", "src"},
+}
+
+// buildQuery builds a parser.QueryAttrMap according to provided search list.
+// The resulting parser.QueryAttrMap will contain only those queries that meet
+// required content types.
+//
+// Example: buildQuery([]string{"image", "mp4"}).
+func buildQuery(search []string) parser.QueryAttrMap {
+	qa := parser.QueryAttrMap{}
+
+	searchMap := make(map[string]struct{}, len(search))
+	for _, s := range search {
+		searchMap[s] = struct{}{}
+	}
+
+	for _, sq := range queries {
+		if _, ok := searchMap[sq.contentType]; ok {
+			qa[sq.query] = sq.attr
+		}
+	}
+
+	return qa
+}

crawler_test.go

@@ -0,0 +1,131 @@
+//go:build unit
+// +build unit
+
+package reactor_crw
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"reactor-crw/parser"
+)
+
+type transportMock struct {
+	mock.Mock
+}
+
+func (m *transportMock) FetchData(url string) (io.ReadCloser, error) {
+	args := m.Called(url)
+	return args.Get(0).(io.ReadCloser), args.Error(1)
+}
+
+type parserMock struct {
+	mock.Mock
+}
+
+func (m *parserMock) FindContent(r io.Reader, q string) (string, error) {
+	args := m.Called(r, q)
+	return args.String(0), args.Error(1)
+}
+
+func (m *parserMock) FindAttrMap(r io.Reader, qa parser.QueryAttrMap, qr parser.QueryResult) error {
+	args := m.Called(r, qa, qr)
+	return args.Error(0)
+}
+
+func TestHtmlCrawler_Fetch(t *testing.T) {
+	path := "https://test.com/test/path"
+
+	trp := &transportMock{}
+	prs := &parserMock{}
+
+	t.Log("Given the need to crawl html page.")
+	{
+		t.Log("When multiple pages requested")
+		{
+			c := &HtmlCrawler{trp, prs, true}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, nil).Once()
+
+			prs.On("FindContent", rc, ".pagination_expanded .current").Return("2", nil)
+			trp.On("FetchData", path+"/1").Return(rc, nil).Once()
+			trp.On("FetchData", path+"/2").Return(rc, nil).Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Run(func(args mock.Arguments) {
+					qr := args.Get(2).(parser.QueryResult)
+					qr["link_1"] = struct{}{}
+				}).
+				Return(nil).
+				Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Run(func(args mock.Arguments) {
+					qr := args.Get(2).(parser.QueryResult)
+					qr["link_2"] = struct{}{}
+				}).
+				Return(nil).
+				Once()
+
+			res, err := c.Fetch(path, []string{"image"})
+			require.NoErrorf(t, err, "Wasn't expected an error during crawl")
+			require.Equal(t, parser.QueryResult{"link_1": struct{}{}, "link_2": struct{}{}}, res)
+		}
+
+		t.Log("When parser returned an error")
+		{
+			expectedErr := errors.New("error")
+			c := &HtmlCrawler{trp, prs, false}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, nil).Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Return(expectedErr).
+				Once()
+
+			_, err := c.Fetch(path, []string{"image"})
+			assert.ErrorIs(t, err, expectedErr)
+		}
+
+		t.Log("When transport returned an error")
+		{
+			expectedErr := errors.New("error")
+			c := &HtmlCrawler{trp, prs, false}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, expectedErr).Once()
+
+			_, err := c.Fetch(path, []string{"image"})
+			assert.ErrorIs(t, err, expectedErr)
+		}
+
+		t.Log("When single page requested")
+		{
+			c := &HtmlCrawler{trp, prs, false}
+
+			rc := ioutil.NopCloser(strings.NewReader(""))
+			trp.On("FetchData", path).Return(rc, nil).Once()
+
+			prs.On("FindAttrMap", rc, mock.Anything, mock.Anything).
+				Run(func(args mock.Arguments) {
+					qr := args.Get(2).(parser.QueryResult)
+					qr["link_1"] = struct{}{}
+				}).
+				Return(nil).
+				Once()
+
+			res, err := c.Fetch(path, nil)
+			require.NoErrorf(t, err, "Wasn't expected an error during crawl")
+			require.Equal(t, parser.QueryResult{"link_1": struct{}{}}, res)
+		}
+	}
+}

handler/content.go

@@ -0,0 +1,8 @@
+package handler
+
+// ContentHandler defines an interface for handling content sources that are
+// represented as URLs. When the handler will finish its job it should notify
+// the progress. Each error should be sent to the errors channel.
+type ContentHandler interface {
+	Process(url string, progress chan<- int, errors chan<- error)
+}

handler/fs/fs.go

@@ -0,0 +1,84 @@
+package fs
+
+import (
+	"io"
+	"path"
+	"reactor-crw"
+)
+
+// PathResolver defines a simple interface to resolve path for content
+// before saving it.
+type pathResolver interface {
+	// CreateFolder creates a new folder in the "current" dir. The current
+	// directory is specified in Config.SavePath provided by a user. If dir
+	// with such name already exists it'll skip creation step.
+	CreateFolder(name string) error
+
+	// CreateFile creates a new file for downloaded content in the directory
+	// created by the CreateFolder function and returns the corresponding
+	// io.WriteCloser. If a file with such a name already exists then it'll
+	// be overwritten.
+	CreateFile(name string) (io.WriteCloser, error)
+
+	// Remove removes a file or folder by its name. If provided path does not
+	// exist it'll be skipped.
+	Remove(name string)
+}
+
+// FileSaver defines ContentHandler implementation that will download content
+// and save it to the host's file system. It resolves the corresponding file
+// path with PathResolver.
+type FileSaver struct {
+	pr pathResolver
+	t  reactor_crw.Transport
+}
+
+// NewFileSaver creates a new FileSaver instance along with a new folder that
+// will act as a context for a new instance. All files will be processed within
+// this new folder.
+func NewFileSaver(pr pathResolver, t reactor_crw.Transport, baseFolder string) (*FileSaver, error) {
+	err := pr.CreateFolder(baseFolder)
+	if err != nil {
+		return nil, err
+	}
+
+	return &FileSaver{
+		pr: pr,
+		t:  t,
+	}, nil
+}
+
+// Process downloads content by the corresponding URL by making an HTTP request
+// and saves the result to the file system. In case of error during saving the
+// content the corresponding file will be deleted from the file system.
+func (f *FileSaver) Process(url string, progress chan<- int, e chan<- error) {
+	defer func() {
+		progress <- 1
+	}()
+
+	data, err := f.t.FetchData(url)
+	if err != nil {
+		e <- err
+		return
+	}
+
+	defer func(b io.ReadCloser) {
+		_ = b.Close()
+	}(data)
+
+	file, err := f.pr.CreateFile(path.Base(url))
+	if err != nil {
+		e <- err
+		return
+	}
+
+	defer func(f io.WriteCloser) {
+		_ = f.Close()
+	}(file)
+
+	_, err = io.Copy(file, data)
+	if err != nil {
+		f.pr.Remove(path.Base(url))
+		e <- err
+	}
+}

handler/fs/fs_test.go

@@ -0,0 +1,130 @@
+//go:build unit
+// +build unit
+
+package fs_test
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"reactor-crw/handler/fs"
+	"testing"
+
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type pathResolverMock struct {
+	mock.Mock
+}
+
+func (m *pathResolverMock) CreateFolder(name string) error {
+	args := m.Called(name)
+	return args.Error(0)
+}
+
+func (m *pathResolverMock) CreateFile(name string) (io.WriteCloser, error) {
+	args := m.Called(name)
+	return args.Get(0).(io.WriteCloser), args.Error(1)
+}
+
+func (m *pathResolverMock) Remove(name string) {
+	_ = m.Called(name)
+}
+
+type transportMock struct {
+	mock.Mock
+}
+
+func (m *transportMock) FetchData(url string) (io.ReadCloser, error) {
+	args := m.Called(url)
+	return args.Get(0).(io.ReadCloser), args.Error(1)
+}
+
+func TestNewFileSaver(t *testing.T) {
+	pr := pathResolverMock{}
+	trp := transportMock{}
+
+	t.Log("Given the need to create a file saver handler.")
+	{
+		t.Log("When baseFolder is incorrect.")
+		{
+			pr.On("CreateFolder", "baseFolder").Return(errors.New("error"))
+			_, err := fs.NewFileSaver(&pr, &trp, "baseFolder")
+			require.Error(t, err, "Expected an error during create folder")
+			pr.ExpectedCalls = []*mock.Call{}
+		}
+
+		t.Log("When all data correct.")
+		{
+			pr.On("CreateFolder", "baseFolder").Return(nil)
+			_, err := fs.NewFileSaver(&pr, &trp, "baseFolder")
+			require.NoError(t, err, "Wasn't expected an error on new creating file saver")
+		}
+	}
+}
+
+func TestFileSaver_Process(t *testing.T) {
+	tmlFile, _ := ioutil.TempFile(os.TempDir(), "file-title.txt")
+
+	pr := pathResolverMock{}
+	pr.On("CreateFolder", "baseFolder").Return(nil)
+	pr.On("Remove", mock.Anything)
+
+	trp := transportMock{}
+
+	p := make(chan int, 1)
+	e := make(chan error, 1)
+
+	fileSaver, _ := fs.NewFileSaver(&pr, &trp, "baseFolder")
+
+	t.Log("Given the need to process data.")
+	{
+		t.Log("When transport returns an error.")
+		{
+			trp.On("FetchData", "file-title.txt").Return(tmlFile, errors.New("error")).Once()
+
+			fileSaver.Process("file-title.txt", p, e)
+			require.Error(t, <-e, "Expected an error during create file")
+			<-p
+		}
+
+		t.Log("When path resolver returns an error.")
+		{
+			trp.On("FetchData", "file-title.txt").Return(tmlFile, nil).Once()
+			pr.On("CreateFile", "file-title.txt").Return(tmlFile, errors.New("error")).Once()
+
+			fileSaver.Process("file-title.txt", p, e)
+			require.Error(t, <-e, "Expected an error during create file")
+			<-p
+		}
+
+		t.Log("When cannot save the data.")
+		{
+			trp.On("FetchData", "file-title.txt").Return(tmlFile, nil).Once()
+			pr.On("CreateFile", "file-title.txt").Return(tmlFile, nil).Once()
+			pr.On("Remove", mock.Anything)
+
+			fileSaver.Process("file-title.txt", p, e)
+			require.Error(t, <-e, "Expected an error file copying")
+			<-p
+		}
+
+		t.Log("When all data correct.")
+		{
+			tmlFile, _ = ioutil.TempFile(os.TempDir(), "new-file-title.txt")
+
+			trp.On("FetchData", "new-file-title.txt").Return(tmlFile, nil).Once()
+			pr.On("CreateFile", "new-file-title.txt").Return(tmlFile, nil).Once()
+
+			fileSaver.Process("new-file-title.txt", p, e)
+			select {
+			case err := <-e:
+				require.NoError(t, err, "Wasn't expected an error on new file process")
+			case <-p:
+				return
+			}
+		}
+	}
+}

handler/fs/path.go

@@ -0,0 +1,94 @@
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+var (
+	// ErrNoCurrentDir returned on creating the content file before the current
+	// directory was created.
+	ErrNoCurrentDir = errors.New("current dir was not created")
+
+	// ErrInvalidDest returned when provided save path is not a valid directory.
+	ErrInvalidDest = errors.New("invalid destination path")
+)
+
+type PathResolver struct {
+	// Dest contains a base directory for all folders that will be created.
+	// It should be a valid absolute or relative path and must already exist.
+	Dest string
+
+	absDest     string
+	currentDest string
+}
+
+// NewPathResolver returns new *PathResolver along with resolving provided dest
+// value. If an incorrect dest value is provided an error will be returned.
+func NewPathResolver(dest string) (*PathResolver, error) {
+	absDest, err := filepath.Abs(dest)
+	if err != nil {
+		return nil, ErrInvalidDest
+	}
+
+	stat, err := os.Stat(absDest)
+	if err != nil || !stat.IsDir() {
+		return nil, ErrInvalidDest
+	}
+
+	return &PathResolver{
+		Dest:    dest,
+		absDest: absDest,
+	}, nil
+}
+
+// CreateFolder creates a new dir with the provided name within PathResolver.Dest dir.
+// Created dir will be marked as current dir. If dir already exists the creation
+// step will be skipped.
+func (p *PathResolver) CreateFolder(name string) error {
+	err := os.Chdir(p.absDest)
+	if err != nil {
+		return fmt.Errorf("cannot resolve path for folder %s: %w", name, err)
+	}
+
+	p.currentDest = path.Join(p.absDest, name)
+
+	err = os.Mkdir(name, fs.ModePerm)
+	if err != nil {
+		if os.IsExist(err) {
+			return nil
+		}
+		return fmt.Errorf("cannot create folder %s: %w", p.currentDest, err)
+	}
+
+	return nil
+}
+
+// CreateFile creates a new file with the corresponding name and returns its
+// io.WriteCloser. If FSResolver.currentDest wasn't created before, then an error
+// will be returned.
+func (p *PathResolver) CreateFile(name string) (io.WriteCloser, error) {
+	if p.currentDest == "" {
+		return nil, ErrNoCurrentDir
+	}
+
+	filePath := path.Join(p.currentDest, name)
+
+	f, err := os.Create(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create file %s: %w", filePath, err)
+	}
+
+	return f, nil
+}
+
+// Remove removes file or dir by its name. If the provided path wasn't found the
+// deletion step will be skipped.
+func (p *PathResolver) Remove(name string) {
+	_ = os.Remove(name)
+}

handler/fs/path_test.go

@@ -0,0 +1,82 @@
+//go:build unit
+// +build unit
+
+package fs_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"reactor-crw/handler/fs"
+)
+
+func TestNewPathResolver(t *testing.T) {
+	paths := []struct {
+		name        string
+		msg         string
+		errExpected bool
+	}{
+		{" ", "Expected an error during resolving empty path", true},
+		{"invalid_path", "Expected an error during resolving non-existent dir", true},
+		{os.TempDir(), "Wasn't expected an error", false},
+	}
+
+	for _, path := range paths {
+		_, err := fs.NewPathResolver(path.name)
+		if path.errExpected {
+			require.Errorf(t, err, path.msg)
+		}
+
+		if !path.errExpected {
+			require.NoError(t, err, path.msg)
+		}
+	}
+}
+
+func TestPathResolver_CreateFolder(t *testing.T) {
+	p, _ := fs.NewPathResolver(os.TempDir())
+
+	folders := []struct {
+		name        string
+		errExpected bool
+	}{
+		{name: "test"},
+		{name: "_123_test"},
+		{name: "109483___"},
+	}
+
+	for _, folder := range folders {
+		err := p.CreateFolder(folder.name)
+		if !folder.errExpected {
+			require.NoError(t, err, "Wasn't expected an error during creating %s", folder.name)
+		}
+
+		if folder.errExpected {
+			require.Error(t, err, "Expected an error during creating %s", folder.name)
+		}
+
+		p.Remove(folder.name)
+	}
+
+	p = &fs.PathResolver{}
+	err := p.CreateFolder("test")
+	require.Error(t, err, "Expected an error during creating folder")
+}
+
+func TestPathResolver_CreateFile(t *testing.T) {
+	p, _ := fs.NewPathResolver(os.TempDir())
+	_, err := p.CreateFile("filename")
+	require.Error(t, err, "Expected an error during file. Folder wasn't created yet.")
+
+	_ = p.CreateFolder("test")
+
+	_, err = p.CreateFile("filename")
+	require.NoError(t, err, "Wasn't expected an error during creating filename")
+
+	filePath := "./test/filename"
+	_, err = os.Stat(filePath)
+	require.NoError(t, err, "File %s wasn't created", filePath)
+
+	p.Remove(filePath)
+}

parser/html.go

@@ -0,0 +1,48 @@
+package parser
+
+import (
+	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	"io"
+	"net/url"
+)
+
+// Html represents a HTML parser implementation.
+type Html struct{}
+
+// FindContent searches for the first element that satisfies the query and returns
+// its text node as content.
+func (h *Html) FindContent(r io.Reader, query string) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return "", fmt.Errorf("cannot parse document: %w", err)
+	}
+
+	return doc.Find(query).First().Text(), nil
+}
+
+// FindAttrMap parses HTML documents with multiple queries and retrieves the
+// corresponding attributes of found elements. All queries and related attributes
+// are stores within QueryAttrMap. All results will be stored in QueryResult.
+//
+// Example: p.FindAttrMap(body, QueryAttrMap{"div": "class"}, QueryResult{})
+func (h *Html) FindAttrMap(r io.Reader, q QueryAttrMap, res QueryResult) error {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return fmt.Errorf("cannot parse document: %w", err)
+	}
+
+	for query, attr := range q {
+		doc.Find(query).Each(func(_ int, s *goquery.Selection) {
+			val, ok := s.Attr(attr)
+			if !ok || val == "javascript:" {
+				return
+			}
+
+			val, _ = url.QueryUnescape(val)
+			res[val] = struct{}{}
+		})
+	}
+
+	return nil
+}

parser/html_test.go

@@ -0,0 +1,62 @@
+//go:build unit
+// +build unit
+
+package parser_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"reactor-crw/parser"
+)
+
+func TestHtml_FindOne(t *testing.T) {
+	p := parser.Html{}
+
+	r := strings.NewReader(`
+		<body>
+			<h1>heading</h1>
+			<a class="test-class" href="href-test" target="_blank">First Link</a>
+			<a class="class" href="href-test" target="_blank">Second Link</a>
+			<a href="href-test" target="_blank">Third Link</a>
+		</body>
+	`)
+
+	res, err := p.FindContent(r, "body .test-class")
+	require.NoErrorf(t, err, "Wasn't expected an error on html parse")
+
+	require.Equal(t, "First Link", res)
+}
+
+func TestHtml_FindAttrMap(t *testing.T) {
+	p := parser.Html{}
+
+	r := strings.NewReader(`
+		<body>
+			<img src="image-src">
+			<a class="test-class" href="href-test" target="_blank">First Link</a>
+			<div class="div-class"></div>
+		</body>
+	`)
+
+	queryAttrMap := parser.QueryAttrMap{
+		".test-class": "href",
+		"div":         "class",
+		"img":         "src",
+		"body":        "src",
+	}
+	res := make(parser.QueryResult)
+
+	err := p.FindAttrMap(r, queryAttrMap, res)
+	require.NoErrorf(t, err, "Wasn't expected an error on html parse")
+	require.Equal(
+		t,
+		parser.QueryResult{
+			"image-src": struct{}{},
+			"href-test": struct{}{},
+			"div-class": struct{}{},
+		},
+		res,
+	)
+}

parser/parser.go

@@ -0,0 +1,23 @@
+package parser
+
+import "io"
+
+// QueryAttrMap stores a mapping of parser query and an attribute which data
+// should be retrieved. Example: QueryAttrMap{"div": "class"}. Here div elements
+// should be found and the class value of each will be collected.
+type QueryAttrMap map[string]string
+
+// QueryResult stores all results parsed using the QueryAttrMap value. For preventing
+// data duplicates the values are stored in a map structure.
+type QueryResult map[string]struct{}
+
+// Parser describes a generic set of parser functions.
+type Parser interface {
+	// FindContent searches for only one element and returns its text content.
+	FindContent(r io.Reader, query string) (string, error)
+
+	// FindAttrMap searches by multiple queries and retrieves corresponding attributes
+	// of found elements. All queries and related attributes stores within QueryAttrMap.
+	// All results will be stored in QueryResult.
+	FindAttrMap(io.Reader, QueryAttrMap, QueryResult) error
+}

transport.go

@@ -0,0 +1,68 @@
+package reactor_crw
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// Headers defines a simple wrapper for client headers.
+type Headers map[string]string
+
+// HttpTransport allows making a network request over HTTP protocol. It is a
+// wrapper over HTTP.Client with a set of default params required by the crawler.
+type HttpTransport struct {
+	headers Headers
+	client  *http.Client
+}
+
+// NewHttpTransport creates a new *HttpTransport with provided client and custom
+// headers. Custom headers can override the default ones.
+func NewHttpTransport(c *http.Client, h Headers) *HttpTransport {
+	var defaultHeaders = Headers{
+		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0",
+		"Referer":    "http://joyreactor.cc/",
+		"DNT":        "1",
+	}
+
+	t := &HttpTransport{
+		headers: defaultHeaders,
+		client:  c,
+	}
+
+	for k, v := range h {
+		t.headers[k] = v
+	}
+
+	return t
+}
+
+// FetchData makes an HTTP request using provided URL and returns the response
+// body as io.ReadCloser interface. Each request will be prepared with provided
+// headers list. The end client is responsible for closing the response body.
+func (t *HttpTransport) FetchData(url string) (io.ReadCloser, error) {
+	req, err := t.prepareRequest(http.MethodGet, url)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := t.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot make request to %s: %w", url, err)
+	}
+
+	return res.Body, nil
+}
+
+func (t *HttpTransport) prepareRequest(method, url string) (*http.Request, error) {
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot prepare request to %s: %w", url, err)
+	}
+
+	for k, v := range t.headers {
+		req.Header.Add(k, v)
+	}
+
+	return req, nil
+}

transport_test.go

@@ -0,0 +1,58 @@
+//go:build unit
+// +build unit
+
+package reactor_crw_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"reactor-crw"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHttpTransport_FetchData(t *testing.T) {
+	t.Log("Given the need to fetch the data.")
+	{
+		t.Log("When request params are valid.")
+		{
+			expected := "response"
+
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, "test-val", r.Header.Get("test-key"))
+				_, _ = fmt.Fprintf(w, expected)
+			}))
+			defer srv.Close()
+
+			httpTransport := reactor_crw.NewHttpTransport(
+				http.DefaultClient,
+				reactor_crw.Headers{"test-key": "test-val"},
+			)
+
+			data, err := httpTransport.FetchData(srv.URL)
+			require.NoErrorf(t, err, "Wasn't expected an error during http call")
+
+			response, _ := ioutil.ReadAll(data)
+			require.Equal(t, expected, string(response))
+		}
+
+		t.Log("When request cannot be prepared.")
+		{
+			httpTransport := reactor_crw.NewHttpTransport(http.DefaultClient, nil)
+
+			_, err := httpTransport.FetchData("\u007F")
+			require.Error(t, err, "Expected an error during request")
+		}
+
+		t.Log("When endpoint is invalid.")
+		{
+			httpTransport := reactor_crw.NewHttpTransport(http.DefaultClient, nil)
+
+			_, err := httpTransport.FetchData(" ")
+			require.Error(t, err, "Expected an error during request")
+		}
+	}
+}