Import source: transport

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

transport.go

@@ -0,0 +1,68 @@
+package reactor_crw
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// Headers defines a simple wrapper for client headers.
+type Headers map[string]string
+
+// HttpTransport allows making a network request over HTTP protocol. It is a
+// wrapper over HTTP.Client with a set of default params required by the crawler.
+type HttpTransport struct {
+	headers Headers
+	client  *http.Client
+}
+
+// NewHttpTransport creates a new *HttpTransport with provided client and custom
+// headers. Custom headers can override the default ones.
+func NewHttpTransport(c *http.Client, h Headers) *HttpTransport {
+	var defaultHeaders = Headers{
+		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0",
+		"Referer":    "http://joyreactor.cc/",
+		"DNT":        "1",
+	}
+
+	t := &HttpTransport{
+		headers: defaultHeaders,
+		client:  c,
+	}
+
+	for k, v := range h {
+		t.headers[k] = v
+	}
+
+	return t
+}
+
+// FetchData makes an HTTP request using provided URL and returns the response
+// body as io.ReadCloser interface. Each request will be prepared with provided
+// headers list. The end client is responsible for closing the response body.
+func (t *HttpTransport) FetchData(url string) (io.ReadCloser, error) {
+	req, err := t.prepareRequest(http.MethodGet, url)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := t.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot make request to %s: %w", url, err)
+	}
+
+	return res.Body, nil
+}
+
+func (t *HttpTransport) prepareRequest(method, url string) (*http.Request, error) {
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot prepare request to %s: %w", url, err)
+	}
+
+	for k, v := range t.headers {
+		req.Header.Add(k, v)
+	}
+
+	return req, nil
+}