Import source: parser/html

Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality.

parser/html.go

@@ -0,0 +1,48 @@
+package parser
+
+import (
+	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	"io"
+	"net/url"
+)
+
+// Html represents a HTML parser implementation.
+type Html struct{}
+
+// FindContent searches for the first element that satisfies the query and returns
+// its text node as content.
+func (h *Html) FindContent(r io.Reader, query string) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return "", fmt.Errorf("cannot parse document: %w", err)
+	}
+
+	return doc.Find(query).First().Text(), nil
+}
+
+// FindAttrMap parses HTML documents with multiple queries and retrieves the
+// corresponding attributes of found elements. All queries and related attributes
+// are stores within QueryAttrMap. All results will be stored in QueryResult.
+//
+// Example: p.FindAttrMap(body, QueryAttrMap{"div": "class"}, QueryResult{})
+func (h *Html) FindAttrMap(r io.Reader, q QueryAttrMap, res QueryResult) error {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return fmt.Errorf("cannot parse document: %w", err)
+	}
+
+	for query, attr := range q {
+		doc.Find(query).Each(func(_ int, s *goquery.Selection) {
+			val, ok := s.Attr(attr)
+			if !ok || val == "javascript:" {
+				return
+			}
+
+			val, _ = url.QueryUnescape(val)
+			res[val] = struct{}{}
+		})
+	}
+
+	return nil
+}

parser/html_test.go

@@ -0,0 +1,62 @@
+//go:build unit
+// +build unit
+
+package parser_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"reactor-crw/parser"
+)
+
+func TestHtml_FindOne(t *testing.T) {
+	p := parser.Html{}
+
+	r := strings.NewReader(`
+		<body>
+			<h1>heading</h1>
+			<a class="test-class" href="href-test" target="_blank">First Link</a>
+			<a class="class" href="href-test" target="_blank">Second Link</a>
+			<a href="href-test" target="_blank">Third Link</a>
+		</body>
+	`)
+
+	res, err := p.FindContent(r, "body .test-class")
+	require.NoErrorf(t, err, "Wasn't expected an error on html parse")
+
+	require.Equal(t, "First Link", res)
+}
+
+func TestHtml_FindAttrMap(t *testing.T) {
+	p := parser.Html{}
+
+	r := strings.NewReader(`
+		<body>
+			<img src="image-src">
+			<a class="test-class" href="href-test" target="_blank">First Link</a>
+			<div class="div-class"></div>
+		</body>
+	`)
+
+	queryAttrMap := parser.QueryAttrMap{
+		".test-class": "href",
+		"div":         "class",
+		"img":         "src",
+		"body":        "src",
+	}
+	res := make(parser.QueryResult)
+
+	err := p.FindAttrMap(r, queryAttrMap, res)
+	require.NoErrorf(t, err, "Wasn't expected an error on html parse")
+	require.Equal(
+		t,
+		parser.QueryResult{
+			"image-src": struct{}{},
+			"href-test": struct{}{},
+			"div-class": struct{}{},
+		},
+		res,
+	)
+}