chore: rename http getter package (#1349)

plugin/http-getter/html_meta.go

@@ -0,0 +1,98 @@
+package getter
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+type HTMLMeta struct {
+	Title       string `json:"title"`
+	Description string `json:"description"`
+	Image       string `json:"image"`
+}
+
+func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {
+	if _, err := url.Parse(urlStr); err != nil {
+		return nil, err
+	}
+
+	response, err := http.Get(urlStr)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	mediatype, err := getMediatype(response)
+	if err != nil {
+		return nil, err
+	}
+	if mediatype != "text/html" {
+		return nil, fmt.Errorf("Wrong website mediatype")
+	}
+
+	htmlMeta := extractHTMLMeta(response.Body)
+	return htmlMeta, nil
+}
+
+func extractHTMLMeta(resp io.Reader) *HTMLMeta {
+	tokenizer := html.NewTokenizer(resp)
+	htmlMeta := new(HTMLMeta)
+
+	for {
+		tokenType := tokenizer.Next()
+		if tokenType == html.ErrorToken {
+			break
+		} else if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
+			token := tokenizer.Token()
+			if token.DataAtom == atom.Body {
+				break
+			}
+
+			if token.DataAtom == atom.Title {
+				tokenizer.Next()
+				token := tokenizer.Token()
+				htmlMeta.Title = token.Data
+			} else if token.DataAtom == atom.Meta {
+				description, ok := extractMetaProperty(token, "description")
+				if ok {
+					htmlMeta.Description = description
+				}
+
+				ogTitle, ok := extractMetaProperty(token, "og:title")
+				if ok {
+					htmlMeta.Title = ogTitle
+				}
+
+				ogDescription, ok := extractMetaProperty(token, "og:description")
+				if ok {
+					htmlMeta.Description = ogDescription
+				}
+
+				ogImage, ok := extractMetaProperty(token, "og:image")
+				if ok {
+					htmlMeta.Image = ogImage
+				}
+			}
+		}
+	}
+
+	return htmlMeta
+}
+
+func extractMetaProperty(token html.Token, prop string) (content string, ok bool) {
+	content, ok = "", false
+	for _, attr := range token.Attr {
+		if attr.Key == "property" && attr.Val == prop {
+			ok = true
+		}
+		if attr.Key == "content" {
+			content = attr.Val
+		}
+	}
+	return content, ok
+}

plugin/http-getter/html_meta_test.go

@@ -0,0 +1,28 @@
+package getter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHTMLMeta(t *testing.T) {
+	tests := []struct {
+		urlStr   string
+		htmlMeta HTMLMeta
+	}{
+		{
+			urlStr: "https://www.bytebase.com/blog/sql-review-tool-for-devs",
+			htmlMeta: HTMLMeta{
+				Title:       "The SQL Review Tool for Developers",
+				Description: "Reviewing SQL can be somewhat tedious, yet is essential to keep your database fleet reliable. At Bytebase, we are building a developer-first SQL review tool to empower the DevOps system.",
+				Image:       "https://www.bytebase.com/static/blog/sql-review-tool-for-devs/dev-fighting-dba.webp",
+			},
+		},
+	}
+	for _, test := range tests {
+		metadata, err := GetHTMLMeta(test.urlStr)
+		require.NoError(t, err)
+		require.Equal(t, test.htmlMeta, *metadata)
+	}
+}

plugin/http-getter/http_getter.go

@@ -0,0 +1,4 @@
+// Package getter is using to get resources from url.
+// * Get metadata for website;
+// * Get image blob to avoid CORS;
+package getter

plugin/http-getter/image.go

@@ -0,0 +1,45 @@
+package getter
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type Image struct {
+	Blob      []byte
+	Mediatype string
+}
+
+func GetImage(urlStr string) (*Image, error) {
+	if _, err := url.Parse(urlStr); err != nil {
+		return nil, err
+	}
+
+	response, err := http.Get(urlStr)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	mediatype, err := getMediatype(response)
+	if err != nil {
+		return nil, err
+	}
+	if !strings.HasPrefix(mediatype, "image/") {
+		return nil, fmt.Errorf("Wrong image mediatype")
+	}
+
+	bodyBytes, err := io.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	image := &Image{
+		Blob:      bodyBytes,
+		Mediatype: mediatype,
+	}
+	return image, nil
+}

plugin/http-getter/image_test.go

@@ -0,0 +1,21 @@
+package getter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetImage(t *testing.T) {
+	tests := []struct {
+		urlStr string
+	}{
+		{
+			urlStr: "https://star-history.com/bytebase.webp",
+		},
+	}
+	for _, test := range tests {
+		_, err := GetImage(test.urlStr)
+		require.NoError(t, err)
+	}
+}

plugin/http-getter/util.go

@@ -0,0 +1,15 @@
+package getter
+
+import (
+	"mime"
+	"net/http"
+)
+
+func getMediatype(response *http.Response) (string, error) {
+	contentType := response.Header.Get("content-type")
+	mediatype, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return "", err
+	}
+	return mediatype, nil
+}