YourSelfHosted/memos
1
0
Fork 0
mirror of https://github.com/usememos/memos.git synced 2025-02-12 09:20:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

chore: implement list nodes

Browse Source
This commit is contained in:
Steven 2023-12-16 08:51:29 +08:00
parent a10b3d3821
commit b00443c222
16 changed files with 283 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
plugin/gomark/ast/block.go
View File

@ -64,3 +64,26 @@ type Blockquote struct {
func (*Blockquote) Type() NodeType {
return BlockquoteNode
}
type OrderedList struct {
BaseBlock
Number string
Children []Node
}
func (*OrderedList) Type() NodeType {
return OrderedListNode
}
type UnorderedList struct {
BaseBlock
// Symbol is "*" or "-" or "+".
Symbol string
Children []Node
}
func (*UnorderedList) Type() NodeType {
return UnorderedListNode
}

2
plugin/gomark/ast/node.go
View File

@ -11,6 +11,8 @@ const (
HeadingNode
HorizontalRuleNode
BlockquoteNode
OrderedListNode
UnorderedListNode
// Inline nodes.
TextNode
BoldNode

2
plugin/gomark/parser/bold.go
View File

@ -23,7 +23,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {
return 0, false
}

2
plugin/gomark/parser/bold_italic.go
View File

@ -23,7 +23,7 @@ func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {
return 0, false
}

4
plugin/gomark/parser/heading.go
View File

@ -16,7 +16,7 @@ func NewHeadingParser() *HeadingParser {
func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) {
cursor := 0
for _, token := range tokens {
if token.Type == tokenizer.Hash {
if token.Type == tokenizer.PoundSign {
cursor++
} else {
break
@ -57,7 +57,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
level := 0
for _, token := range tokens {
if token.Type == tokenizer.Hash {
if token.Type == tokenizer.PoundSign {
level++
} else {
break

2
plugin/gomark/parser/horizontal_rule.go
View File

@ -20,7 +20,7 @@ func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {
if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {
return 0, false
}
if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {
if tokens[0].Type != tokenizer.Hyphen && tokens[0].Type != tokenizer.Underscore && tokens[0].Type != tokenizer.Asterisk {
return 0, false
}
if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {

2
plugin/gomark/parser/italic.go
View File

@ -21,7 +21,7 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
}
prefixTokens := tokens[:1]
if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {
if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underscore {
return 0, false
}
prefixTokenType := prefixTokens[0].Type

54
plugin/gomark/parser/ordered_list.go Normal file
View File

@ -0,0 +1,54 @@
package parser
import (
"errors"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type OrderedListParser struct{}
func NewOrderedListParser() *OrderedListParser {
return &OrderedListParser{}
}
func (*OrderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 4 {
return 0, false
}
if tokens[0].Type != tokenizer.Number || tokens[1].Type != tokenizer.Dot || tokens[2].Type != tokenizer.Space {
return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[3:] {
contentTokens = append(contentTokens, token)
if token.Type == tokenizer.Newline {
break
}
}
if len(contentTokens) == 0 {
return 0, false
}
return len(contentTokens) + 3, true
}
func (p *OrderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil, errors.New("not matched")
}
contentTokens := tokens[3:size]
children, err := ParseInline(contentTokens)
if err != nil {
return nil, err
}
return &ast.OrderedList{
Number: tokens[0].Value,
Children: children,
}, nil
}

58
plugin/gomark/parser/ordered_list_test.go Normal file
View File

@ -0,0 +1,58 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestOrderedListParser(t *testing.T) {
tests := []struct {
text string
node ast.Node
}{
{
text: "1.asd",
node: nil,
},
{
text: "1. Hello World",
node: &ast.OrderedList{
Number: "1",
Children: []ast.Node{
&ast.Text{
Content: "Hello World",
},
},
},
},
{
text: "1aa. Hello World",
node: nil,
},
{
text: "22. Hello *World*",
node: &ast.OrderedList{
Number: "22",
Children: []ast.Node{
&ast.Text{
Content: "Hello ",
},
&ast.Italic{
Symbol: "*",
Content: "World",
},
},
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
node, _ := NewOrderedListParser().Parse(tokens)
require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))
}
}

4
plugin/gomark/parser/tag.go
View File

@ -17,12 +17,12 @@ func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 2 {
return 0, false
}
if tokens[0].Type != tokenizer.Hash {
if tokens[0].Type != tokenizer.PoundSign {
return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[1:] {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.Hash {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.PoundSign {
break
}
contentTokens = append(contentTokens, token)

36
plugin/gomark/parser/tokenizer/tokenizer.go
View File

@ -3,9 +3,9 @@ package tokenizer
type TokenType = string
const (
Underline TokenType = "_"
Underscore TokenType = "_"
Asterisk TokenType = "*"
Hash TokenType = "#"
PoundSign TokenType = "#"
Backtick TokenType = "`"
LeftSquareBracket TokenType = "["
RightSquareBracket TokenType = "]"
@ -13,14 +13,17 @@ const (
RightParenthesis TokenType = ")"
ExclamationMark TokenType = "!"
Tilde TokenType = "~"
Dash TokenType = "-"
Hyphen TokenType = "-"
PlusSign TokenType = "+"
Dot TokenType = "."
GreaterThan TokenType = ">"
Newline TokenType = "\n"
Space TokenType = " "
)
const (
Text TokenType = ""
Number TokenType = "number"
Text TokenType = ""
)
type Token struct {
@ -40,11 +43,11 @@ func Tokenize(text string) []*Token {
for _, c := range text {
switch c {
case '_':
tokens = append(tokens, NewToken(Underline, "_"))
tokens = append(tokens, NewToken(Underscore, "_"))
case '*':
tokens = append(tokens, NewToken(Asterisk, "*"))
case '#':
tokens = append(tokens, NewToken(Hash, "#"))
tokens = append(tokens, NewToken(PoundSign, "#"))
case '`':
tokens = append(tokens, NewToken(Backtick, "`"))
case '[':
@ -60,9 +63,13 @@ func Tokenize(text string) []*Token {
case '~':
tokens = append(tokens, NewToken(Tilde, "~"))
case '-':
tokens = append(tokens, NewToken(Dash, "-"))
tokens = append(tokens, NewToken(Hyphen, "-"))
case '>':
tokens = append(tokens, NewToken(GreaterThan, ">"))
case '+':
tokens = append(tokens, NewToken(PlusSign, "+"))
case '.':
tokens = append(tokens, NewToken(Dot, "."))
case '\n':
tokens = append(tokens, NewToken(Newline, "\n"))
case ' ':
@ -72,10 +79,19 @@ func Tokenize(text string) []*Token {
if len(tokens) > 0 {
prevToken = tokens[len(tokens)-1]
}
if prevToken == nil || prevToken.Type != Text {
tokens = append(tokens, NewToken(Text, string(c)))
isNumber := c >= '0' && c <= '9'
if prevToken != nil {
if (prevToken.Type == Text && !isNumber) || (prevToken.Type == Number && isNumber) {
prevToken.Value += string(c)
continue
}
}
if isNumber {
tokens = append(tokens, NewToken(Number, string(c)))
} else {
prevToken.Value += string(c)
tokens = append(tokens, NewToken(Text, string(c)))
}
}
}

2
plugin/gomark/parser/tokenizer/tokenizer_test.go
View File

@ -41,7 +41,7 @@ func TestTokenize(t *testing.T) {
world`,
tokens: []*Token{
{
Type: Hash,
Type: PoundSign,
Value: "#",
},
{

55
plugin/gomark/parser/unordered_list.go Normal file
View File

@ -0,0 +1,55 @@
package parser
import (
"errors"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type UnorderedListParser struct{}
func NewUnorderedListParser() *UnorderedListParser {
return &UnorderedListParser{}
}
func (*UnorderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 3 {
return 0, false
}
symbolToken := tokens[0]
if (symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign) || tokens[1].Type != tokenizer.Space {
return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[2:] {
if token.Type == tokenizer.Newline {
break
}
contentTokens = append(contentTokens, token)
}
if len(contentTokens) == 0 {
return 0, false
}
return len(contentTokens) + 2, true
}
func (p *UnorderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil, errors.New("not matched")
}
symbolToken := tokens[0]
contentTokens := tokens[2:size]
children, err := ParseInline(contentTokens)
if err != nil {
return nil, err
}
return &ast.UnorderedList{
Symbol: symbolToken.Type,
Children: children,
}, nil
}

51
plugin/gomark/parser/unordered_list_test.go Normal file
View File

@ -0,0 +1,51 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestUnorderedListParser(t *testing.T) {
tests := []struct {
text string
node ast.Node
}{
{
text: "*asd",
node: nil,
},
{
text: "+ Hello World",
node: &ast.UnorderedList{
Symbol: tokenizer.PlusSign,
Children: []ast.Node{
&ast.Text{
Content: "Hello World",
},
},
},
},
{
text: "* **Hello**",
node: &ast.UnorderedList{
Symbol: tokenizer.Asterisk,
Children: []ast.Node{
&ast.Bold{
Symbol: "*",
Content: "Hello",
},
},
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
node, _ := NewUnorderedListParser().Parse(tokens)
require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))
}
}

2
plugin/gomark/render/html/html.go
View File

@ -159,7 +159,7 @@ func (r *HTMLRender) renderLink(node *ast.Link) {
func (r *HTMLRender) renderTag(node *ast.Tag) {
r.output.WriteString(`<span>`)
r.output.WriteString(`# `)
r.output.WriteString(`#`)
r.output.WriteString(node.Content)
r.output.WriteString(`</span>`)
}

4
plugin/gomark/render/html/html_test.go
View File

@ -30,6 +30,10 @@ func TestHTMLRender(t *testing.T) {
text: "**Hello** world!",
expected: `<p><strong>Hello</strong> world!</p>`,
},
{
text: "#article #memo",
expected: `<p><span>#article</span> <span>#memo</span></p>`,
},
}
for _, test := range tests {