From ebe161a08a551b85ac29689d1b022fb30ce3d644 Mon Sep 17 00:00:00 2001 From: mathew murphy Date: Tue, 28 Dec 2021 12:56:01 -0600 Subject: [PATCH 1/4] Add support for Goldmark Markdown renderer --- config/config.go | 10 ++++ go.mod | 4 ++ go.sum | 9 +++- postrender.go | 117 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 135 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 2065ddf..40738e1 100644 --- a/config/config.go +++ b/config/config.go @@ -168,6 +168,9 @@ type ( // Disable password authentication if use only Oauth DisablePasswordAuth bool `ini:"disable_password_auth"` + + // Which Markdown renderer to use + Renderer string `ini:"markdown_renderer"` } // Config holds the complete configuration for running a writefreely instance @@ -245,6 +248,13 @@ func (ac AppCfg) SignupPath() string { return "/" } +func (ac AppCfg) MarkdownRenderer() string { + if strings.EqualFold(ac.Renderer, "goldmark") { + return "goldmark" + } + return "saturday" +} + // Load reads the given configuration file, then parses and returns it as a Config. func Load(fname string) (*Config, error) { if fname == "" { diff --git a/go.mod b/go.mod index 54d38bb..ba449fd 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/writefreely/writefreely require ( + git.mills.io/prologic/go-gopher v0.0.0-20210712135410-b7ebb55feece + github.com/clbanning/mxj v1.8.4 // indirect github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.13.0 github.com/go-ini/ini v1.66.4 @@ -49,6 +51,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 // indirect github.com/fatih/structs v1.1.0 // indirect + github.com/forPelevin/gomoji v1.1.3 // indirect github.com/go-fed/httpsig v0.1.1-0.20200204213531-0ef28562fabe // indirect github.com/go-test/deep v1.0.1 // indirect github.com/gofrs/uuid v3.3.0+incompatible // indirect @@ -64,6 +67,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect diff --git a/go.sum b/go.sum index d9e8885..62f1d6a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ code.as/core/socks v1.0.0 h1:SPQXNp4SbEwjOAP9VzUahLHak8SDqy5n+9cm9tpjZOs= code.as/core/socks v1.0.0/go.mod h1:BAXBy5O9s2gmw6UxLqNJcVbWY7C/UPs+801CcSsfWOY= +git.mills.io/prologic/go-gopher v0.0.0-20210712135410-b7ebb55feece h1:0esmnntqeuM1iBgHH0HOeSynsLA1l28p2K3h/WZuIfQ= +git.mills.io/prologic/go-gopher v0.0.0-20210712135410-b7ebb55feece/go.mod h1:EMXlYOIbYJQhPTtIltgaaHtCYDawV/HL0dYf8ShzAck= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= @@ -29,6 +32,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/forPelevin/gomoji v1.1.3 h1:7c3dYzVmYhpOL3bS4riXqSWJBX3BhSvH68yoNNf3FH0= +github.com/forPelevin/gomoji v1.1.3/go.mod h1:ypB7Kz3Fsp+LVR7KoT7mEFOioYBuTuAtaAT4RGl+ASY= github.com/go-fed/httpsig v0.1.0/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= github.com/go-fed/httpsig v0.1.1-0.20200204213531-0ef28562fabe h1:U71giCx5NjRn4Lb71UuprPHqhjxGv3Jqonb9fgcaJH8= github.com/go-fed/httpsig v0.1.1-0.20200204213531-0ef28562fabe/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= @@ -101,6 +106,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ= github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= @@ -156,8 +163,6 @@ github.com/writefreely/go-gopher v0.0.0-20220429181814-40127126f83b h1:h3NzB8OZ5 github.com/writefreely/go-gopher v0.0.0-20220429181814-40127126f83b/go.mod h1:T2UVVzt+R5KSSZe2xRSytnwc2M9AoDegi7foeIsik+M= github.com/writefreely/go-nodeinfo v1.2.0 h1:La+YbTCvmpTwFhBSlebWDDL81N88Qf/SCAvRLR7F8ss= github.com/writefreely/go-nodeinfo v1.2.0/go.mod h1:UTvE78KpcjYOlRHupZIiSEFcXHioTXuacCbHU+CAcPg= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/postrender.go b/postrender.go index 5be8d0c..57b8579 100644 --- a/postrender.go +++ b/postrender.go @@ -23,12 +23,17 @@ import ( "unicode" "unicode/utf8" + hashtag "github.com/abhinav/goldmark-hashtag" "github.com/microcosm-cc/bluemonday" stripmd "github.com/writeas/go-strip-markdown/v2" "github.com/writeas/impart" blackfriday "github.com/writeas/saturday" "github.com/writeas/web-core/log" "github.com/writeas/web-core/stringmanip" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/writefreely/writefreely/config" "github.com/writefreely/writefreely/parse" ) @@ -83,7 +88,7 @@ func (p *Post) formatContent(cfg *config.Config, c *Collection, isOwner bool, is p.handlePremiumContent(c, isOwner, isPostPage, cfg) p.Content = strings.Replace(p.Content, "<!--paid-->", "", 1) - p.HTMLTitle = template.HTML(applyBasicMarkdown([]byte(p.Title.String))) + p.HTMLTitle = template.HTML(applyBasicMarkdown([]byte(p.Title.String), cfg)) p.HTMLContent = template.HTML(applyMarkdown([]byte(p.Content), baseURL, cfg)) if exc := strings.Index(string(p.Content), ""); exc > -1 { p.HTMLExcerpt = template.HTML(applyMarkdown([]byte(p.Content[:exc]), baseURL, cfg)) @@ -120,7 +125,10 @@ func (p *PublicPost) augmentReadingDestination() { } func applyMarkdown(data []byte, baseURL string, cfg *config.Config) string { - return applyMarkdownSpecial(data, false, baseURL, cfg) + if cfg.App.MarkdownRenderer() == "goldmark" { + return applyCommonmarkSpecial(data, false, baseURL, cfg) + } + return applySaturdaySpecial(data, false, baseURL, cfg) } func disableYoutubeAutoplay(outHTML string) string { @@ -142,7 +150,74 @@ func disableYoutubeAutoplay(outHTML string) string { return outHTML } +type hashtagResolver struct { + Prefix string +} + +var _ hashtag.Resolver = hashtagResolver{} + +func (h hashtagResolver) ResolveHashtag(node *hashtag.Node) (destination []byte, err error) { + var buf bytes.Buffer + buf.WriteString(h.Prefix) + buf.Write(node.Tag) + return buf.Bytes(), nil +} + func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *config.Config) string { + if cfg.App.MarkdownRenderer() == "goldmark" { + return applyCommonmarkSpecial(data, skipNoFollow, baseURL, cfg) + } else { + return applySaturdaySpecial(data, skipNoFollow, baseURL, cfg) + } +} + +func applyCommonmarkSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *config.Config) string { + extensions := []goldmark.Extender{ + extension.GFM, + extension.DefinitionList, + extension.Typographer, + // but no footnotes, see https://github.com/writefreely/writefreely/issues/338 + } + if baseURL != "" { + tagPrefix := baseURL + "tag:" + if cfg.App.Chorus { + tagPrefix = "/read/t" + } + extensions = append(extensions, &hashtag.Extender{ + Resolver: hashtagResolver{Prefix: tagPrefix}, + }) + } + md := goldmark.New( + goldmark.WithExtensions( + &hashtag.Extender{ + Resolver: hashtagResolver{}, + }, + ), + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), + ), + ) + var buf bytes.Buffer + if err := md.Convert(data, &buf); err != nil { + log.Info("error rendering CommonMark: %v", err) + } + htm := buf.Bytes() + if baseURL != "" { + handlePrefix := cfg.App.Host + "/@/" + htm = []byte(mentionReg.ReplaceAll(htm, []byte("@$1$2"))) + } + // Strip out bad HTML + policy := getSanitizationPolicy() + policy.RequireNoFollowOnLinks(!skipNoFollow) + outHTML := string(policy.SanitizeBytes(htm)) + // Strip newlines on certain block elements that render with them + outHTML = blockReg.ReplaceAllString(outHTML, "<$1>") + outHTML = endBlockReg.ReplaceAllString(outHTML, "") + outHTML = disableYoutubeAutoplay(outHTML) + return outHTML +} + +func applySaturdaySpecial(data []byte, skipNoFollow bool, baseURL string, cfg *config.Config) string { mdExtensions := 0 | blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | @@ -181,7 +256,43 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c return outHTML } -func applyBasicMarkdown(data []byte) string { +func applyBasicMarkdown(data []byte, cfg *config.Config) string { + if cfg.App.MarkdownRenderer() == "goldmark" { + return applyBasicCommonmark(data) + } else { + return applyBasicSaturday(data) + } +} + +func applyBasicCommonmark(data []byte) string { + md := goldmark.New( + goldmark.WithExtensions( + extension.Strikethrough, + extension.Linkify, + extension.Typographer, + ), + ) + var inbuf bytes.Buffer + inbuf.WriteString("# ") + inbuf.Write(data) + var outbuf bytes.Buffer + if err := md.Convert(inbuf.Bytes(), &outbuf); err != nil { + log.Info("error rendering basic CommonMark: %v", err) + } + htm := outbuf.Bytes() + htm = bytes.TrimSpace(htm) + + htm = htm[len("

") : len(htm)-len("

")] + // Strip out bad HTML + policy := bluemonday.UGCPolicy() + policy.AllowAttrs("class", "id").Globally() + outHTML := string(policy.SanitizeBytes(htm)) + outHTML = markeddownReg.ReplaceAllString(outHTML, "$1") + outHTML = strings.TrimRightFunc(outHTML, unicode.IsSpace) + return outHTML +} + +func applyBasicSaturday(data []byte) string { if len(bytes.TrimSpace(data)) == 0 { return "" } From 223dfdea66b5e995756215647c5f736111647b1f Mon Sep 17 00:00:00 2001 From: mathew Date: Sat, 24 Dec 2022 10:04:48 -0600 Subject: [PATCH 2/4] CommonMark renderer options, unit test both renderers --- config/config.go | 39 +++++++++++++++++++++++++++++++++++++++ postrender.go | 35 ++++++++++++----------------------- postrender_test.go | 27 +++++++++++++++++++-------- 3 files changed, 70 insertions(+), 31 deletions(-) diff --git a/config/config.go b/config/config.go index 40738e1..4c6ff62 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,8 @@ import ( "github.com/go-ini/ini" "github.com/writeas/web-core/log" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" "golang.org/x/net/idna" ) @@ -171,6 +173,11 @@ type ( // Which Markdown renderer to use Renderer string `ini:"markdown_renderer"` + + // Options for the Goldmark renderer + RendererOptions string `ini:"markdown_options"` + // Conversion of options ready for the renderer + rendererExtensions []goldmark.Extender } // Config holds the complete configuration for running a writefreely instance @@ -255,6 +262,38 @@ func (ac AppCfg) MarkdownRenderer() string { return "saturday" } +func (ac AppCfg) RendererExtensions() []goldmark.Extender { + if ac.rendererExtensions != nil { + return ac.rendererExtensions + } + var extlist []goldmark.Extender + optlist := strings.FieldsFunc(ac.RendererOptions, func(r rune) bool { + return r == ' ' || r == '\t' || r == ',' + }) + for _, opt := range optlist { + switch opt { + case "table": + extlist = append(extlist, extension.Table) + case "strikethrough": + extlist = append(extlist, extension.Strikethrough) + case "linkify": + extlist = append(extlist, extension.Linkify) + case "tasklist": + extlist = append(extlist, extension.TaskList) + case "gfm": + extlist = append(extlist, extension.GFM) + case "definitionlist": + extlist = append(extlist, extension.DefinitionList) + case "typographer": + extlist = append(extlist, extension.Typographer) + case "cjk": + extlist = append(extlist, extension.CJK) + } + } + ac.rendererExtensions = extlist + return extlist +} + // Load reads the given configuration file, then parses and returns it as a Config. func Load(fname string) (*Config, error) { if fname == "" { diff --git a/postrender.go b/postrender.go index 57b8579..2d155a3 100644 --- a/postrender.go +++ b/postrender.go @@ -31,7 +31,6 @@ import ( "github.com/writeas/web-core/log" "github.com/writeas/web-core/stringmanip" "github.com/yuin/goldmark" - "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" "github.com/writefreely/writefreely/config" @@ -172,12 +171,7 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c } func applyCommonmarkSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *config.Config) string { - extensions := []goldmark.Extender{ - extension.GFM, - extension.DefinitionList, - extension.Typographer, - // but no footnotes, see https://github.com/writefreely/writefreely/issues/338 - } + extensions := cfg.App.RendererExtensions() if baseURL != "" { tagPrefix := baseURL + "tag:" if cfg.App.Chorus { @@ -188,11 +182,7 @@ func applyCommonmarkSpecial(data []byte, skipNoFollow bool, baseURL string, cfg }) } md := goldmark.New( - goldmark.WithExtensions( - &hashtag.Extender{ - Resolver: hashtagResolver{}, - }, - ), + goldmark.WithExtensions(extensions...), goldmark.WithParserOptions( parser.WithAutoHeadingID(), ), @@ -258,18 +248,16 @@ func applySaturdaySpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c func applyBasicMarkdown(data []byte, cfg *config.Config) string { if cfg.App.MarkdownRenderer() == "goldmark" { - return applyBasicCommonmark(data) + return applyBasicCommonmark(data, cfg) } else { return applyBasicSaturday(data) } } -func applyBasicCommonmark(data []byte) string { +func applyBasicCommonmark(data []byte, cfg *config.Config) string { md := goldmark.New( goldmark.WithExtensions( - extension.Strikethrough, - extension.Linkify, - extension.Typographer, + cfg.App.RendererExtensions()..., ), ) var inbuf bytes.Buffer @@ -395,12 +383,13 @@ func sanitizePost(content string) string { // choosing what to generate. In case a post has a title, this function will // fail, and logic should instead be implemented to skip this when there's no // title, like so: -// var desc string -// if title == "" { -// desc = postDescription(content, title, friendlyId) -// } else { -// desc = shortPostDescription(content) -// } +// +// var desc string +// if title == "" { +// desc = postDescription(content, title, friendlyId) +// } else { +// desc = shortPostDescription(content) +// } func postDescription(content, title, friendlyId string) string { maxLen := 140 diff --git a/postrender_test.go b/postrender_test.go index ec6bbdd..3ac2a77 100644 --- a/postrender_test.go +++ b/postrender_test.go @@ -10,7 +10,11 @@ package writefreely -import "testing" +import ( + "testing" + + "github.com/writefreely/writefreely/config" +) func TestApplyBasicMarkdown(t *testing.T) { tests := []struct { @@ -32,12 +36,19 @@ func TestApplyBasicMarkdown(t *testing.T) { {"date", "12. April", `12. April`}, {"table", "| Hi | There |", `| Hi | There |`}, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - res := applyBasicMarkdown([]byte(test.in)) - if res != test.result { - t.Errorf("%s: wanted %s, got %s", test.name, test.result, res) - } - }) + for _, renderer := range []string{"saturday", "goldmark"} { + cfg := &config.Config{ + App: config.AppCfg{ + Renderer: renderer, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res := applyBasicMarkdown([]byte(test.in), cfg) + if res != test.result { + t.Errorf("%s: wanted %s, got %s", test.name, test.result, res) + } + }) + } } } From e3bfdf948b2bfbd7f034c2c98f6d24aaf683fd6f Mon Sep 17 00:00:00 2001 From: mathew Date: Sat, 24 Dec 2022 10:05:19 -0600 Subject: [PATCH 3/4] Fix nil pointer exception bug --- templates.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates.go b/templates.go index ecd8750..44befb1 100644 --- a/templates.go +++ b/templates.go @@ -23,6 +23,7 @@ import ( "github.com/dustin/go-humanize" "github.com/writeas/web-core/l10n" "github.com/writeas/web-core/log" + "github.com/writefreely/writefreely/config" ) @@ -136,7 +137,7 @@ func InitTemplates(cfg *config.Config) error { log.Info("Loading pages...") // Initialize all static pages that use the base template filepath.Walk(filepath.Join(cfg.Server.PagesParentDir, pagesDir), func(path string, i os.FileInfo, err error) error { - if !i.IsDir() && !strings.HasPrefix(i.Name(), ".") { + if i != nil && !i.IsDir() && !strings.HasPrefix(i.Name(), ".") { key := i.Name() initPage(cfg.Server.PagesParentDir, path, key) } From 0e5927aae539e050888af8cdfdcddaf4334e5450 Mon Sep 17 00:00:00 2001 From: mathew Date: Sat, 24 Dec 2022 16:56:33 -0600 Subject: [PATCH 4/4] Allow GFM checkboxes in Goldmark mode --- postrender.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postrender.go b/postrender.go index 2d155a3..497f1a2 100644 --- a/postrender.go +++ b/postrender.go @@ -198,6 +198,9 @@ func applyCommonmarkSpecial(data []byte, skipNoFollow bool, baseURL string, cfg } // Strip out bad HTML policy := getSanitizationPolicy() + // Enable GFM checkboxes for CommonMark + // Technically we could skip this if the + policy.AllowAttrs("type", "disabled", "checked").OnElements("input") policy.RequireNoFollowOnLinks(!skipNoFollow) outHTML := string(policy.SanitizeBytes(htm)) // Strip newlines on certain block elements that render with them