arsam
f65143a3fd
Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.
69 lines
1.8 KiB
Go
69 lines
1.8 KiB
Go
package reactor_crw
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
)
|
|
|
|
// Headers defines a simple wrapper for client headers.
|
|
type Headers map[string]string
|
|
|
|
// HttpTransport allows making a network request over HTTP protocol. It is a
|
|
// wrapper over HTTP.Client with a set of default params required by the crawler.
|
|
type HttpTransport struct {
|
|
headers Headers
|
|
client *http.Client
|
|
}
|
|
|
|
// NewHttpTransport creates a new *HttpTransport with provided client and custom
|
|
// headers. Custom headers can override the default ones.
|
|
func NewHttpTransport(c *http.Client, h Headers) *HttpTransport {
|
|
var defaultHeaders = Headers{
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0",
|
|
"Referer": "http://joyreactor.cc/",
|
|
"DNT": "1",
|
|
}
|
|
|
|
t := &HttpTransport{
|
|
headers: defaultHeaders,
|
|
client: c,
|
|
}
|
|
|
|
for k, v := range h {
|
|
t.headers[k] = v
|
|
}
|
|
|
|
return t
|
|
}
|
|
|
|
// FetchData makes an HTTP request using provided URL and returns the response
|
|
// body as io.ReadCloser interface. Each request will be prepared with provided
|
|
// headers list. The end client is responsible for closing the response body.
|
|
func (t *HttpTransport) FetchData(url string) (io.ReadCloser, error) {
|
|
req, err := t.prepareRequest(http.MethodGet, url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
res, err := t.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot make request to %s: %w", url, err)
|
|
}
|
|
|
|
return res.Body, nil
|
|
}
|
|
|
|
func (t *HttpTransport) prepareRequest(method, url string) (*http.Request, error) {
|
|
req, err := http.NewRequest(method, url, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot prepare request to %s: %w", url, err)
|
|
}
|
|
|
|
for k, v := range t.headers {
|
|
req.Header.Add(k, v)
|
|
}
|
|
|
|
return req, nil
|
|
}
|