Import source: transport

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

transport.go

@@ -0,0 +1,68 @@
+package reactor_crw
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// Headers defines a simple wrapper for client headers.
+type Headers map[string]string
+
+// HttpTransport allows making a network request over HTTP protocol. It is a
+// wrapper over HTTP.Client with a set of default params required by the crawler.
+type HttpTransport struct {
+	headers Headers
+	client  *http.Client
+}
+
+// NewHttpTransport creates a new *HttpTransport with provided client and custom
+// headers. Custom headers can override the default ones.
+func NewHttpTransport(c *http.Client, h Headers) *HttpTransport {
+	var defaultHeaders = Headers{
+		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0",
+		"Referer":    "http://joyreactor.cc/",
+		"DNT":        "1",
+	}
+
+	t := &HttpTransport{
+		headers: defaultHeaders,
+		client:  c,
+	}
+
+	for k, v := range h {
+		t.headers[k] = v
+	}
+
+	return t
+}
+
+// FetchData makes an HTTP request using provided URL and returns the response
+// body as io.ReadCloser interface. Each request will be prepared with provided
+// headers list. The end client is responsible for closing the response body.
+func (t *HttpTransport) FetchData(url string) (io.ReadCloser, error) {
+	req, err := t.prepareRequest(http.MethodGet, url)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := t.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot make request to %s: %w", url, err)
+	}
+
+	return res.Body, nil
+}
+
+func (t *HttpTransport) prepareRequest(method, url string) (*http.Request, error) {
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot prepare request to %s: %w", url, err)
+	}
+
+	for k, v := range t.headers {
+		req.Header.Add(k, v)
+	}
+
+	return req, nil
+}

transport_test.go

@@ -0,0 +1,58 @@
+//go:build unit
+// +build unit
+
+package reactor_crw_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"reactor-crw"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHttpTransport_FetchData(t *testing.T) {
+	t.Log("Given the need to fetch the data.")
+	{
+		t.Log("When request params are valid.")
+		{
+			expected := "response"
+
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, "test-val", r.Header.Get("test-key"))
+				_, _ = fmt.Fprintf(w, expected)
+			}))
+			defer srv.Close()
+
+			httpTransport := reactor_crw.NewHttpTransport(
+				http.DefaultClient,
+				reactor_crw.Headers{"test-key": "test-val"},
+			)
+
+			data, err := httpTransport.FetchData(srv.URL)
+			require.NoErrorf(t, err, "Wasn't expected an error during http call")
+
+			response, _ := ioutil.ReadAll(data)
+			require.Equal(t, expected, string(response))
+		}
+
+		t.Log("When request cannot be prepared.")
+		{
+			httpTransport := reactor_crw.NewHttpTransport(http.DefaultClient, nil)
+
+			_, err := httpTransport.FetchData("\u007F")
+			require.Error(t, err, "Expected an error during request")
+		}
+
+		t.Log("When endpoint is invalid.")
+		{
+			httpTransport := reactor_crw.NewHttpTransport(http.DefaultClient, nil)
+
+			_, err := httpTransport.FetchData(" ")
+			require.Error(t, err, "Expected an error during request")
+		}
+	}
+}