diff --git a/parser/html.go b/parser/html.go new file mode 100644 index 0000000..e578867 --- /dev/null +++ b/parser/html.go @@ -0,0 +1,48 @@ +package parser + +import ( + "fmt" + "github.com/PuerkitoBio/goquery" + "io" + "net/url" +) + +// Html represents a HTML parser implementation. +type Html struct{} + +// FindContent searches for the first element that satisfies the query and returns +// its text node as content. +func (h *Html) FindContent(r io.Reader, query string) (string, error) { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return "", fmt.Errorf("cannot parse document: %w", err) + } + + return doc.Find(query).First().Text(), nil +} + +// FindAttrMap parses HTML documents with multiple queries and retrieves the +// corresponding attributes of found elements. All queries and related attributes +// are stores within QueryAttrMap. All results will be stored in QueryResult. +// +// Example: p.FindAttrMap(body, QueryAttrMap{"div": "class"}, QueryResult{}) +func (h *Html) FindAttrMap(r io.Reader, q QueryAttrMap, res QueryResult) error { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return fmt.Errorf("cannot parse document: %w", err) + } + + for query, attr := range q { + doc.Find(query).Each(func(_ int, s *goquery.Selection) { + val, ok := s.Attr(attr) + if !ok || val == "javascript:" { + return + } + + val, _ = url.QueryUnescape(val) + res[val] = struct{}{} + }) + } + + return nil +} diff --git a/parser/html_test.go b/parser/html_test.go new file mode 100644 index 0000000..6ef2eaa --- /dev/null +++ b/parser/html_test.go @@ -0,0 +1,62 @@ +//go:build unit +// +build unit + +package parser_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" + "reactor-crw/parser" +) + +func TestHtml_FindOne(t *testing.T) { + p := parser.Html{} + + r := strings.NewReader(` + +

heading

+ First Link + Second Link + Third Link + + `) + + res, err := p.FindContent(r, "body .test-class") + require.NoErrorf(t, err, "Wasn't expected an error on html parse") + + require.Equal(t, "First Link", res) +} + +func TestHtml_FindAttrMap(t *testing.T) { + p := parser.Html{} + + r := strings.NewReader(` + + + First Link +
+ + `) + + queryAttrMap := parser.QueryAttrMap{ + ".test-class": "href", + "div": "class", + "img": "src", + "body": "src", + } + res := make(parser.QueryResult) + + err := p.FindAttrMap(r, queryAttrMap, res) + require.NoErrorf(t, err, "Wasn't expected an error on html parse") + require.Equal( + t, + parser.QueryResult{ + "image-src": struct{}{}, + "href-test": struct{}{}, + "div-class": struct{}{}, + }, + res, + ) +}