diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.go index 42bf6f4b..7eef5823 100644 --- a/plugin/gomark/ast/block.go +++ b/plugin/gomark/ast/block.go @@ -1,6 +1,7 @@ package ast type BaseBlock struct { + Node } type LineBreak struct { @@ -50,3 +51,28 @@ var NodeTypeHeading = NewNodeType("Heading") func (*Heading) Type() NodeType { return NodeTypeHeading } + +type HorizontalRule struct { + BaseBlock + + // Symbol is "*" or "-" or "_". + Symbol string +} + +var NodeTypeHorizontalRule = NewNodeType("HorizontalRule") + +func (*HorizontalRule) Type() NodeType { + return NodeTypeHorizontalRule +} + +type Blockquote struct { + BaseBlock + + Children []Node +} + +var NodeTypeBlockquote = NewNodeType("Blockquote") + +func (*Blockquote) Type() NodeType { + return NodeTypeBlockquote +} diff --git a/plugin/gomark/ast/inline.go b/plugin/gomark/ast/inline.go index 3e4c57e5..d21fe1a1 100644 --- a/plugin/gomark/ast/inline.go +++ b/plugin/gomark/ast/inline.go @@ -1,6 +1,8 @@ package ast -type BaseInline struct{} +type BaseInline struct { + Node +} type Text struct { BaseInline @@ -28,6 +30,34 @@ func (*Bold) Type() NodeType { return NodeTypeBold } +type Italic struct { + BaseInline + + // Symbol is "*" or "_" + Symbol string + Content string +} + +var NodeTypeItalic = NewNodeType("Italic") + +func (*Italic) Type() NodeType { + return NodeTypeItalic +} + +type BoldItalic struct { + BaseInline + + // Symbol is "*" or "_" + Symbol string + Content string +} + +var NodeTypeBoldItalic = NewNodeType("BoldItalic") + +func (*BoldItalic) Type() NodeType { + return NodeTypeBoldItalic +} + type Code struct { BaseInline @@ -66,20 +96,6 @@ func (*Link) Type() NodeType { return NodeTypeLink } -type Italic struct { - BaseInline - - // Symbol is "*" or "_" - Symbol string - Content string -} - -var NodeTypeItalic = NewNodeType("Italic") - -func (*Italic) Type() NodeType { - return NodeTypeItalic -} - type Tag struct { BaseInline @@ -91,3 +107,15 @@ var NodeTypeTag = NewNodeType("Tag") func (*Tag) Type() NodeType { return NodeTypeTag } + +type Strikethrough struct { + BaseInline + + Content string +} + +var NodeTypeStrikethrough = NewNodeType("Strikethrough") + +func (*Strikethrough) Type() NodeType { + return NodeTypeStrikethrough +} diff --git a/plugin/gomark/parser/blockquote.go b/plugin/gomark/parser/blockquote.go new file mode 100644 index 00000000..fa9425c4 --- /dev/null +++ b/plugin/gomark/parser/blockquote.go @@ -0,0 +1,47 @@ +package parser + +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type BlockquoteParser struct{} + +func NewBlockquoteParser() *BlockquoteParser { + return &BlockquoteParser{} +} + +func (*BlockquoteParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 4 { + return 0, false + } + if tokens[0].Type != tokenizer.GreaterThan || tokens[1].Type != tokenizer.Space { + return 0, false + } + + contentTokens := []*tokenizer.Token{} + for _, token := range tokens[2:] { + if token.Type == tokenizer.Newline { + break + } + contentTokens = append(contentTokens, token) + } + if len(contentTokens) == 0 { + return 0, false + } + + return len(contentTokens) + 2, true +} + +func (p *BlockquoteParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil + } + + contentTokens := tokens[2:size] + children := ParseInline(contentTokens) + return &ast.Blockquote{ + Children: children, + } +} diff --git a/plugin/gomark/parser/blockquote_test.go b/plugin/gomark/parser/blockquote_test.go new file mode 100644 index 00000000..2e0f2ae0 --- /dev/null +++ b/plugin/gomark/parser/blockquote_test.go @@ -0,0 +1,47 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestBlockquoteParser(t *testing.T) { + tests := []struct { + text string + blockquote ast.Node + }{ + { + text: "> Hello world", + blockquote: &ast.Blockquote{ + Children: []ast.Node{ + &ast.Text{ + Content: "Hello world", + }, + }, + }, + }, + { + text: "> Hello\nworld", + blockquote: &ast.Blockquote{ + Children: []ast.Node{ + &ast.Text{ + Content: "Hello", + }, + }, + }, + }, + { + text: ">Hello\nworld", + blockquote: nil, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + require.Equal(t, test.blockquote, NewBlockquoteParser().Parse(tokens)) + } +} diff --git a/plugin/gomark/parser/bold.go b/plugin/gomark/parser/bold.go index 237cb885..36cf2434 100644 --- a/plugin/gomark/parser/bold.go +++ b/plugin/gomark/parser/bold.go @@ -7,10 +7,8 @@ import ( type BoldParser struct{} -var defaultBoldParser = &BoldParser{} - func NewBoldParser() InlineParser { - return defaultBoldParser + return &BoldParser{} } func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) { @@ -23,7 +21,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) { return 0, false } prefixTokenType := prefixTokens[0].Type - if prefixTokenType != tokenizer.Star && prefixTokenType != tokenizer.Underline { + if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline { return 0, false } diff --git a/plugin/gomark/parser/bold_italic.go b/plugin/gomark/parser/bold_italic.go new file mode 100644 index 00000000..a9736594 --- /dev/null +++ b/plugin/gomark/parser/bold_italic.go @@ -0,0 +1,58 @@ +package parser + +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type BoldItalicParser struct{} + +func NewBoldItalicParser() InlineParser { + return &BoldItalicParser{} +} + +func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 7 { + return 0, false + } + + prefixTokens := tokens[:3] + if prefixTokens[0].Type != prefixTokens[1].Type || prefixTokens[0].Type != prefixTokens[2].Type || prefixTokens[1].Type != prefixTokens[2].Type { + return 0, false + } + prefixTokenType := prefixTokens[0].Type + if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline { + return 0, false + } + + cursor, matched := 3, false + for ; cursor < len(tokens)-2; cursor++ { + token, nextToken, endToken := tokens[cursor], tokens[cursor+1], tokens[cursor+2] + if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline || endToken.Type == tokenizer.Newline { + return 0, false + } + if token.Type == prefixTokenType && nextToken.Type == prefixTokenType && endToken.Type == prefixTokenType { + matched = true + break + } + } + if !matched { + return 0, false + } + + return cursor + 3, true +} + +func (p *BoldItalicParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil + } + + prefixTokenType := tokens[0].Type + contentTokens := tokens[3 : size-3] + return &ast.BoldItalic{ + Symbol: prefixTokenType, + Content: tokenizer.Stringify(contentTokens), + } +} diff --git a/plugin/gomark/parser/bold_italic_test.go b/plugin/gomark/parser/bold_italic_test.go new file mode 100644 index 00000000..85579859 --- /dev/null +++ b/plugin/gomark/parser/bold_italic_test.go @@ -0,0 +1,49 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestBoldItalicParser(t *testing.T) { + tests := []struct { + text string + boldItalic ast.Node + }{ + { + text: "*Hello world!", + boldItalic: nil, + }, + { + text: "***Hello***", + boldItalic: &ast.BoldItalic{ + Symbol: "*", + Content: "Hello", + }, + }, + { + text: "*** Hello ***", + boldItalic: &ast.BoldItalic{ + Symbol: "*", + Content: " Hello ", + }, + }, + { + text: "*** Hello * *", + boldItalic: nil, + }, + { + text: "*** Hello **", + boldItalic: nil, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + require.Equal(t, test.boldItalic, NewBoldItalicParser().Parse(tokens)) + } +} diff --git a/plugin/gomark/parser/code.go b/plugin/gomark/parser/code.go index 903b1cf7..b6dff80c 100644 --- a/plugin/gomark/parser/code.go +++ b/plugin/gomark/parser/code.go @@ -7,10 +7,8 @@ import ( type CodeParser struct{} -var defaultCodeParser = &CodeParser{} - func NewCodeParser() *CodeParser { - return defaultCodeParser + return &CodeParser{} } func (*CodeParser) Match(tokens []*tokenizer.Token) (int, bool) { diff --git a/plugin/gomark/parser/code_block.go b/plugin/gomark/parser/code_block.go index 7cc8510f..8ae48176 100644 --- a/plugin/gomark/parser/code_block.go +++ b/plugin/gomark/parser/code_block.go @@ -10,10 +10,8 @@ type CodeBlockParser struct { Content string } -var defaultCodeBlockParser = &CodeBlockParser{} - func NewCodeBlockParser() *CodeBlockParser { - return defaultCodeBlockParser + return &CodeBlockParser{} } func (*CodeBlockParser) Match(tokens []*tokenizer.Token) (int, bool) { diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.go index 8becfc5f..eaefcb33 100644 --- a/plugin/gomark/parser/heading.go +++ b/plugin/gomark/parser/heading.go @@ -62,11 +62,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) ast.Node { } } contentTokens := tokens[level+1 : size] - children := ParseInline(contentTokens, []InlineParser{ - NewBoldParser(), - NewCodeParser(), - NewTextParser(), - }) + children := ParseInline(contentTokens) return &ast.Heading{ Level: level, Children: children, diff --git a/plugin/gomark/parser/horizontal_rule.go b/plugin/gomark/parser/horizontal_rule.go new file mode 100644 index 00000000..a008d443 --- /dev/null +++ b/plugin/gomark/parser/horizontal_rule.go @@ -0,0 +1,39 @@ +package parser + +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type HorizontalRuleParser struct{} + +func NewHorizontalRuleParser() *HorizontalRuleParser { + return &HorizontalRuleParser{} +} + +func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 3 { + return 0, false + } + if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type { + return 0, false + } + if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk { + return 0, false + } + if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline { + return 0, false + } + return 3, true +} + +func (p *HorizontalRuleParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil + } + + return &ast.HorizontalRule{ + Symbol: tokens[0].Type, + } +} diff --git a/plugin/gomark/parser/horizontal_rule_test.go b/plugin/gomark/parser/horizontal_rule_test.go new file mode 100644 index 00000000..f45878f1 --- /dev/null +++ b/plugin/gomark/parser/horizontal_rule_test.go @@ -0,0 +1,49 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestHorizontalRuleParser(t *testing.T) { + tests := []struct { + text string + horizontalRule ast.Node + }{ + { + text: "---", + horizontalRule: &ast.HorizontalRule{ + Symbol: "-", + }, + }, + { + text: "****", + horizontalRule: nil, + }, + { + text: "***", + horizontalRule: &ast.HorizontalRule{ + Symbol: "*", + }, + }, + { + text: "-*-", + horizontalRule: nil, + }, + { + text: "___", + horizontalRule: &ast.HorizontalRule{ + Symbol: "_", + }, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + require.Equal(t, test.horizontalRule, NewHorizontalRuleParser().Parse(tokens)) + } +} diff --git a/plugin/gomark/parser/image.go b/plugin/gomark/parser/image.go index 71398120..83be30bd 100644 --- a/plugin/gomark/parser/image.go +++ b/plugin/gomark/parser/image.go @@ -7,10 +7,8 @@ import ( type ImageParser struct{} -var defaultImageParser = &ImageParser{} - func NewImageParser() *ImageParser { - return defaultImageParser + return &ImageParser{} } func (*ImageParser) Match(tokens []*tokenizer.Token) (int, bool) { diff --git a/plugin/gomark/parser/italic.go b/plugin/gomark/parser/italic.go index 3ab01f93..e9e32fef 100644 --- a/plugin/gomark/parser/italic.go +++ b/plugin/gomark/parser/italic.go @@ -1,6 +1,9 @@ package parser -import "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) type ItalicParser struct { ContentTokens []*tokenizer.Token @@ -10,21 +13,21 @@ func NewItalicParser() *ItalicParser { return &ItalicParser{} } -func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser { +func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) { if len(tokens) < 3 { - return nil + return 0, false } prefixTokens := tokens[:1] - if prefixTokens[0].Type != tokenizer.Star && prefixTokens[0].Type != tokenizer.Underline { - return nil + if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline { + return 0, false } prefixTokenType := prefixTokens[0].Type contentTokens := []*tokenizer.Token{} matched := false for _, token := range tokens[1:] { if token.Type == tokenizer.Newline { - return nil + return 0, false } if token.Type == prefixTokenType { matched = true @@ -33,10 +36,22 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser { contentTokens = append(contentTokens, token) } if !matched || len(contentTokens) == 0 { + return 0, false + } + + return len(contentTokens) + 2, true +} + +func (p *ItalicParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { return nil } - return &ItalicParser{ - ContentTokens: contentTokens, + prefixTokenType := tokens[0].Type + contentTokens := tokens[1 : size-1] + return &ast.Italic{ + Symbol: prefixTokenType, + Content: tokenizer.Stringify(contentTokens), } } diff --git a/plugin/gomark/parser/italic_test.go b/plugin/gomark/parser/italic_test.go index 76893123..60c6ae53 100644 --- a/plugin/gomark/parser/italic_test.go +++ b/plugin/gomark/parser/italic_test.go @@ -5,13 +5,14 @@ import ( "github.com/stretchr/testify/require" + "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" ) func TestItalicParser(t *testing.T) { tests := []struct { text string - italic *ItalicParser + italic ast.Node }{ { text: "*Hello world!", @@ -19,76 +20,29 @@ func TestItalicParser(t *testing.T) { }, { text: "*Hello*", - italic: &ItalicParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Text, - Value: "Hello", - }, - }, + italic: &ast.Italic{ + Symbol: "*", + Content: "Hello", }, }, { text: "* Hello *", - italic: &ItalicParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Space, - Value: " ", - }, - { - Type: tokenizer.Text, - Value: "Hello", - }, - { - Type: tokenizer.Space, - Value: " ", - }, - }, + italic: &ast.Italic{ + Symbol: "*", + Content: " Hello ", }, }, - { - text: "** Hello * *", - italic: nil, - }, { text: "*1* Hello * *", - italic: &ItalicParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Text, - Value: "1", - }, - }, + italic: &ast.Italic{ + Symbol: "*", + Content: "1", }, }, - { - text: `* \n * Hello * *`, - italic: &ItalicParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Space, - Value: " ", - }, - { - Type: tokenizer.Text, - Value: `\n`, - }, - { - Type: tokenizer.Space, - Value: " ", - }, - }, - }, - }, - { - text: "* \n * Hello * *", - italic: nil, - }, } for _, test := range tests { tokens := tokenizer.Tokenize(test.text) - require.Equal(t, test.italic, NewItalicParser().Match(tokens)) + require.Equal(t, test.italic, NewItalicParser().Parse(tokens)) } } diff --git a/plugin/gomark/parser/line_break.go b/plugin/gomark/parser/line_break.go index bdc0957d..6017f72a 100644 --- a/plugin/gomark/parser/line_break.go +++ b/plugin/gomark/parser/line_break.go @@ -7,10 +7,8 @@ import ( type LineBreakParser struct{} -var defaultLineBreakParser = &LineBreakParser{} - func NewLineBreakParser() *LineBreakParser { - return defaultLineBreakParser + return &LineBreakParser{} } func (*LineBreakParser) Match(tokens []*tokenizer.Token) (int, bool) { diff --git a/plugin/gomark/parser/link.go b/plugin/gomark/parser/link.go index a1c8b744..3b5fd643 100644 --- a/plugin/gomark/parser/link.go +++ b/plugin/gomark/parser/link.go @@ -1,58 +1,72 @@ package parser -import "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) -type LinkParser struct { - ContentTokens []*tokenizer.Token - URL string -} +type LinkParser struct{} func NewLinkParser() *LinkParser { return &LinkParser{} } -func (*LinkParser) Match(tokens []*tokenizer.Token) *LinkParser { - if len(tokens) < 4 { - return nil +func (*LinkParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 5 { + return 0, false } if tokens[0].Type != tokenizer.LeftSquareBracket { - return nil + return 0, false } - cursor, contentTokens := 1, []*tokenizer.Token{} - for ; cursor < len(tokens)-2; cursor++ { - if tokens[cursor].Type == tokenizer.Newline { - return nil + textTokens := []*tokenizer.Token{} + for _, token := range tokens[1:] { + if token.Type == tokenizer.Newline { + return 0, false } - if tokens[cursor].Type == tokenizer.RightSquareBracket { + if token.Type == tokenizer.RightSquareBracket { break } - contentTokens = append(contentTokens, tokens[cursor]) + textTokens = append(textTokens, token) } - if tokens[cursor+1].Type != tokenizer.LeftParenthesis { - return nil + if len(textTokens)+4 >= len(tokens) { + return 0, false } - matched, url := false, "" - for _, token := range tokens[cursor+2:] { + if tokens[2+len(textTokens)].Type != tokenizer.LeftParenthesis { + return 0, false + } + urlTokens := []*tokenizer.Token{} + for _, token := range tokens[3+len(textTokens):] { if token.Type == tokenizer.Newline || token.Type == tokenizer.Space { - return nil + return 0, false } if token.Type == tokenizer.RightParenthesis { - matched = true break } - url += token.Value + urlTokens = append(urlTokens, token) } - if !matched || url == "" { + if 4+len(urlTokens)+len(textTokens) > len(tokens) { + return 0, false + } + + return 4 + len(urlTokens) + len(textTokens), true +} + +func (p *LinkParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { return nil } - if len(contentTokens) == 0 { - contentTokens = append(contentTokens, &tokenizer.Token{ - Type: tokenizer.Text, - Value: url, - }) + + textTokens := []*tokenizer.Token{} + for _, token := range tokens[1:] { + if token.Type == tokenizer.RightSquareBracket { + break + } + textTokens = append(textTokens, token) } - return &LinkParser{ - ContentTokens: contentTokens, - URL: url, + urlTokens := tokens[2+len(textTokens)+1 : size-1] + return &ast.Link{ + Text: tokenizer.Stringify(textTokens), + URL: tokenizer.Stringify(urlTokens), } } diff --git a/plugin/gomark/parser/link_test.go b/plugin/gomark/parser/link_test.go index ab4f5c7d..421ab38a 100644 --- a/plugin/gomark/parser/link_test.go +++ b/plugin/gomark/parser/link_test.go @@ -5,24 +5,20 @@ import ( "github.com/stretchr/testify/require" + "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" ) func TestLinkParser(t *testing.T) { tests := []struct { text string - link *LinkParser + link ast.Node }{ { text: "[](https://example.com)", - link: &LinkParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Text, - Value: "https://example.com", - }, - }, - URL: "https://example.com", + link: &ast.Link{ + Text: "", + URL: "https://example.com", }, }, { @@ -35,27 +31,14 @@ func TestLinkParser(t *testing.T) { }, { text: "[hello world](https://example.com)", - link: &LinkParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Text, - Value: "hello", - }, - { - Type: tokenizer.Space, - Value: " ", - }, - { - Type: tokenizer.Text, - Value: "world", - }, - }, - URL: "https://example.com", + link: &ast.Link{ + Text: "hello world", + URL: "https://example.com", }, }, } for _, test := range tests { tokens := tokenizer.Tokenize(test.text) - require.Equal(t, test.link, NewLinkParser().Match(tokens)) + require.Equal(t, test.link, NewLinkParser().Parse(tokens)) } } diff --git a/plugin/gomark/parser/paragraph.go b/plugin/gomark/parser/paragraph.go index c2403ad0..09e69047 100644 --- a/plugin/gomark/parser/paragraph.go +++ b/plugin/gomark/parser/paragraph.go @@ -9,10 +9,8 @@ type ParagraphParser struct { ContentTokens []*tokenizer.Token } -var defaultParagraphParser = &ParagraphParser{} - func NewParagraphParser() *ParagraphParser { - return defaultParagraphParser + return &ParagraphParser{} } func (*ParagraphParser) Match(tokens []*tokenizer.Token) (int, bool) { @@ -38,10 +36,7 @@ func (p *ParagraphParser) Parse(tokens []*tokenizer.Token) ast.Node { } contentTokens := tokens[:size] - children := ParseInline(contentTokens, []InlineParser{ - NewBoldParser(), - NewTextParser(), - }) + children := ParseInline(contentTokens) return &ast.Paragraph{ Children: children, } diff --git a/plugin/gomark/parser/parser.go b/plugin/gomark/parser/parser.go index c82fa50d..123eead3 100644 --- a/plugin/gomark/parser/parser.go +++ b/plugin/gomark/parser/parser.go @@ -1,6 +1,8 @@ package parser import ( + "errors" + "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" ) @@ -23,32 +25,51 @@ type BlockParser interface { BaseParser } -func Parse(tokens []*tokenizer.Token) []ast.Node { +var defaultBlockParsers = []BlockParser{ + NewCodeBlockParser(), + NewHorizontalRuleParser(), + NewHeadingParser(), + NewBlockquoteParser(), + NewParagraphParser(), + NewLineBreakParser(), +} + +func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) { nodes := []ast.Node{} - blockParsers := []BlockParser{ - NewCodeBlockParser(), - NewParagraphParser(), - NewLineBreakParser(), - } for len(tokens) > 0 { - for _, blockParser := range blockParsers { + for _, blockParser := range defaultBlockParsers { cursor, matched := blockParser.Match(tokens) if matched { node := blockParser.Parse(tokens) + if node == nil { + return nil, errors.New("parse error") + } nodes = append(nodes, node) tokens = tokens[cursor:] break } } } - return nodes + return nodes, nil } -func ParseInline(tokens []*tokenizer.Token, inlineParsers []InlineParser) []ast.Node { +var defaultInlineParsers = []InlineParser{ + NewBoldItalicParser(), + NewImageParser(), + NewLinkParser(), + NewBoldParser(), + NewItalicParser(), + NewCodeParser(), + NewTagParser(), + NewStrikethroughParser(), + NewTextParser(), +} + +func ParseInline(tokens []*tokenizer.Token) []ast.Node { nodes := []ast.Node{} var lastNode ast.Node for len(tokens) > 0 { - for _, inlineParser := range inlineParsers { + for _, inlineParser := range defaultInlineParsers { cursor, matched := inlineParser.Match(tokens) if matched { node := inlineParser.Parse(tokens) diff --git a/plugin/gomark/parser/parser_test.go b/plugin/gomark/parser/parser_test.go index dc2d6e68..62b06ab9 100644 --- a/plugin/gomark/parser/parser_test.go +++ b/plugin/gomark/parser/parser_test.go @@ -89,6 +89,8 @@ func TestParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) - require.Equal(t, test.nodes, Parse(tokens)) + nodes, err := Parse(tokens) + require.NoError(t, err) + require.Equal(t, test.nodes, nodes) } } diff --git a/plugin/gomark/parser/strikethrough.go b/plugin/gomark/parser/strikethrough.go new file mode 100644 index 00000000..fde8acc5 --- /dev/null +++ b/plugin/gomark/parser/strikethrough.go @@ -0,0 +1,49 @@ +package parser + +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type StrikethroughParser struct{} + +func NewStrikethroughParser() *StrikethroughParser { + return &StrikethroughParser{} +} + +func (*StrikethroughParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 5 { + return 0, false + } + if tokens[0].Type != tokenizer.Tilde || tokens[1].Type != tokenizer.Tilde { + return 0, false + } + + cursor, matched := 2, false + for ; cursor < len(tokens)-1; cursor++ { + token, nextToken := tokens[cursor], tokens[cursor+1] + if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline { + return 0, false + } + if token.Type == tokenizer.Tilde && nextToken.Type == tokenizer.Tilde { + matched = true + break + } + } + if !matched { + return 0, false + } + return cursor + 2, true +} + +func (p *StrikethroughParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil + } + + contentTokens := tokens[2 : size-2] + return &ast.Strikethrough{ + Content: tokenizer.Stringify(contentTokens), + } +} diff --git a/plugin/gomark/parser/strikethrough_test.go b/plugin/gomark/parser/strikethrough_test.go new file mode 100644 index 00000000..28a04bcc --- /dev/null +++ b/plugin/gomark/parser/strikethrough_test.go @@ -0,0 +1,45 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestStrikethroughParser(t *testing.T) { + tests := []struct { + text string + strikethrough ast.Node + }{ + { + text: "~~Hello world", + strikethrough: nil, + }, + { + text: "~~Hello~~", + strikethrough: &ast.Strikethrough{ + Content: "Hello", + }, + }, + { + text: "~~ Hello ~~", + strikethrough: &ast.Strikethrough{ + Content: " Hello ", + }, + }, + { + text: "~~1~~ Hello ~~~", + strikethrough: &ast.Strikethrough{ + Content: "1", + }, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + require.Equal(t, test.strikethrough, NewStrikethroughParser().Parse(tokens)) + } +} diff --git a/plugin/gomark/parser/tag.go b/plugin/gomark/parser/tag.go index c33fcd0b..80fc15ed 100644 --- a/plugin/gomark/parser/tag.go +++ b/plugin/gomark/parser/tag.go @@ -1,21 +1,22 @@ package parser -import "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +import ( + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) -type TagParser struct { - ContentTokens []*tokenizer.Token -} +type TagParser struct{} func NewTagParser() *TagParser { return &TagParser{} } -func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser { +func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) { if len(tokens) < 2 { - return nil + return 0, false } if tokens[0].Type != tokenizer.Hash { - return nil + return 0, false } contentTokens := []*tokenizer.Token{} for _, token := range tokens[1:] { @@ -25,10 +26,20 @@ func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser { contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { + return 0, false + } + + return len(contentTokens) + 1, true +} + +func (p *TagParser) Parse(tokens []*tokenizer.Token) ast.Node { + size, ok := p.Match(tokens) + if size == 0 || !ok { return nil } - return &TagParser{ - ContentTokens: contentTokens, + contentTokens := tokens[1:size] + return &ast.Tag{ + Content: tokenizer.Stringify(contentTokens), } } diff --git a/plugin/gomark/parser/tag_test.go b/plugin/gomark/parser/tag_test.go index 7066027f..54d541e7 100644 --- a/plugin/gomark/parser/tag_test.go +++ b/plugin/gomark/parser/tag_test.go @@ -5,13 +5,14 @@ import ( "github.com/stretchr/testify/require" + "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" ) func TestTagParser(t *testing.T) { tests := []struct { text string - tag *TagParser + tag ast.Node }{ { text: "*Hello world", @@ -23,30 +24,20 @@ func TestTagParser(t *testing.T) { }, { text: "#tag", - tag: &TagParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Text, - Value: "tag", - }, - }, + tag: &ast.Tag{ + Content: "tag", }, }, { - text: "#tag/subtag", - tag: &TagParser{ - ContentTokens: []*tokenizer.Token{ - { - Type: tokenizer.Text, - Value: "tag/subtag", - }, - }, + text: "#tag/subtag 123", + tag: &ast.Tag{ + Content: "tag/subtag", }, }, } for _, test := range tests { tokens := tokenizer.Tokenize(test.text) - require.Equal(t, test.tag, NewTagParser().Match(tokens)) + require.Equal(t, test.tag, NewTagParser().Parse(tokens)) } } diff --git a/plugin/gomark/parser/text.go b/plugin/gomark/parser/text.go index f930ad3d..a2523ff2 100644 --- a/plugin/gomark/parser/text.go +++ b/plugin/gomark/parser/text.go @@ -9,10 +9,8 @@ type TextParser struct { Content string } -var defaultTextParser = &TextParser{} - func NewTextParser() *TextParser { - return defaultTextParser + return &TextParser{} } func (*TextParser) Match(tokens []*tokenizer.Token) (int, bool) { diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.go index a9c9f42c..f27ecf03 100644 --- a/plugin/gomark/parser/tokenizer/tokenizer.go +++ b/plugin/gomark/parser/tokenizer/tokenizer.go @@ -4,7 +4,7 @@ type TokenType = string const ( Underline TokenType = "_" - Star TokenType = "*" + Asterisk TokenType = "*" Hash TokenType = "#" Backtick TokenType = "`" LeftSquareBracket TokenType = "[" @@ -12,6 +12,9 @@ const ( LeftParenthesis TokenType = "(" RightParenthesis TokenType = ")" ExclamationMark TokenType = "!" + Tilde TokenType = "~" + Dash TokenType = "-" + GreaterThan TokenType = ">" Newline TokenType = "\n" Space TokenType = " " ) @@ -39,7 +42,7 @@ func Tokenize(text string) []*Token { case '_': tokens = append(tokens, NewToken(Underline, "_")) case '*': - tokens = append(tokens, NewToken(Star, "*")) + tokens = append(tokens, NewToken(Asterisk, "*")) case '#': tokens = append(tokens, NewToken(Hash, "#")) case '`': @@ -54,6 +57,12 @@ func Tokenize(text string) []*Token { tokens = append(tokens, NewToken(RightParenthesis, ")")) case '!': tokens = append(tokens, NewToken(ExclamationMark, "!")) + case '~': + tokens = append(tokens, NewToken(Tilde, "~")) + case '-': + tokens = append(tokens, NewToken(Dash, "-")) + case '>': + tokens = append(tokens, NewToken(GreaterThan, ">")) case '\n': tokens = append(tokens, NewToken(Newline, "\n")) case ' ': diff --git a/plugin/gomark/parser/tokenizer/tokenizer_test.go b/plugin/gomark/parser/tokenizer/tokenizer_test.go index 8010fe4f..a85651b3 100644 --- a/plugin/gomark/parser/tokenizer/tokenizer_test.go +++ b/plugin/gomark/parser/tokenizer/tokenizer_test.go @@ -15,7 +15,7 @@ func TestTokenize(t *testing.T) { text: "*Hello world!", tokens: []*Token{ { - Type: Star, + Type: Asterisk, Value: "*", }, {