feat: add heading tokenizer (#1723)

plugin/gomark/parser/tokenizer/token.go

@@ -1,27 +0,0 @@
-package tokenizer
-
-type TokenType = string
-
-const (
-	Underline TokenType = "_"
-	Star      TokenType = "*"
-	Newline   TokenType = "\n"
-	Hash      TokenType = "#"
-	Space     TokenType = " "
-)
-
-const (
-	Text TokenType = ""
-)
-
-type Token struct {
-	Type  TokenType
-	Value string
-}
-
-func NewToken(tp, text string) *Token {
-	return &Token{
-		Type:  tp,
-		Value: text,
-	}
-}

plugin/gomark/parser/tokenizer/tokenizer.go

@@ -1,6 +1,32 @@
 package tokenizer
 
-func tokenize(text string) []*Token {
+type TokenType = string
+
+const (
+	Underline TokenType = "_"
+	Star      TokenType = "*"
+	Hash      TokenType = "#"
+	Newline   TokenType = "\n"
+	Space     TokenType = " "
+)
+
+const (
+	Text TokenType = ""
+)
+
+type Token struct {
+	Type  TokenType
+	Value string
+}
+
+func NewToken(tp, text string) *Token {
+	return &Token{
+		Type:  tp,
+		Value: text,
+	}
+}
+
+func Tokenize(text string) []*Token {
 	tokens := []*Token{}
 	for _, c := range text {
 		switch c {
@@ -8,6 +34,8 @@ func tokenize(text string) []*Token {
 			tokens = append(tokens, NewToken(Underline, "_"))
 		case '*':
 			tokens = append(tokens, NewToken(Star, "*"))
+		case '#':
+			tokens = append(tokens, NewToken(Hash, "#"))
 		case '\n':
 			tokens = append(tokens, NewToken(Newline, "\n"))
 		case ' ':

plugin/gomark/parser/tokenizer/tokenizer_test.go

@@ -32,9 +32,44 @@ func TestTokenize(t *testing.T) {
 				},
 			},
 		},
+		{
+			text: `# hello 
+ world`,
+			tokens: []*Token{
+				{
+					Type:  Hash,
+					Value: "#",
+				},
+				{
+					Type:  Space,
+					Value: " ",
+				},
+				{
+					Type:  Text,
+					Value: "hello",
+				},
+				{
+					Type:  Space,
+					Value: " ",
+				},
+				{
+					Type:  Newline,
+					Value: "\n",
+				},
+				{
+					Type:  Space,
+					Value: " ",
+				},
+				{
+					Type:  Text,
+					Value: "world",
+				},
+			},
+		},
 	}
+
 	for _, test := range tests {
-		result := tokenize(test.text)
+		result := Tokenize(test.text)
 		require.Equal(t, test.tokens, result)
 	}
 }