Add data layer

This includes config changes, collections, posts, some post rendering
funcs, and actual database connection when the server starts up.

app.go

@@ -1,6 +1,7 @@
 package writefreely
 
 import (
+	"database/sql"
 	"flag"
 	"fmt"
 	_ "github.com/go-sql-driver/mysql"
@@ -21,6 +22,7 @@ const (
 
 type app struct {
 	router       *mux.Router
+	db           *datastore
 	cfg          *config.Config
 	keys         *keychain
 	sessionStore *sessions.CookieStore
@@ -62,11 +64,33 @@ func Serve() {
 	// Initialize modules
 	app.sessionStore = initSession(app)
 
+	// Check database configuration
+	if app.cfg.Database.User == "" || app.cfg.Database.Password == "" {
+		log.Error("Database user or password not set.")
+		os.Exit(1)
+	}
+	if app.cfg.Database.Host == "" {
+		app.cfg.Database.Host = "localhost"
+	}
+	if app.cfg.Database.Database == "" {
+		app.cfg.Database.Database = "writeas"
+	}
+
+	log.Info("Connecting to database...")
+	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=true", app.cfg.Database.User, app.cfg.Database.Password, app.cfg.Database.Host, app.cfg.Database.Port, app.cfg.Database.Database))
+	if err != nil {
+		log.Error("\n%s\n", err)
+		os.Exit(1)
+	}
+	app.db = &datastore{db}
+	defer shutdown(app)
+	app.db.SetMaxOpenConns(50)
+
 	r := mux.NewRouter()
 	handler := NewHandler(app.sessionStore)
 
 	// Handle app routes
-	initRoutes(handler, r, app.cfg)
+	initRoutes(handler, r, app.cfg, app.db)
 
 	// Handle static files
 	fs := http.FileServer(http.Dir(staticDir))
@@ -92,4 +116,6 @@ func Serve() {
 }
 
 func shutdown(app *app) {
+	log.Info("Closing database connection...")
+	app.db.Close()
 }

collections.go

@@ -0,0 +1,69 @@
+package writefreely
+
+import (
+	"database/sql"
+)
+
+type (
+	Collection struct {
+		ID              int64          `datastore:"id" json:"-"`
+		Alias           string         `datastore:"alias" schema:"alias" json:"alias"`
+		Title           string         `datastore:"title" schema:"title" json:"title"`
+		Description     string         `datastore:"description" schema:"description" json:"description"`
+		Direction       string         `schema:"dir" json:"dir,omitempty"`
+		Language        string         `schema:"lang" json:"lang,omitempty"`
+		StyleSheet      string         `datastore:"style_sheet" schema:"style_sheet" json:"style_sheet"`
+		Script          string         `datastore:"script" schema:"script" json:"script,omitempty"`
+		Public          bool           `datastore:"public" json:"public"`
+		Visibility      collVisibility `datastore:"private" json:"-"`
+		Format          string         `datastore:"format" json:"format,omitempty"`
+		Views           int64          `json:"views"`
+		OwnerID         int64          `datastore:"owner_id" json:"-"`
+		PublicOwner     bool           `datastore:"public_owner" json:"-"`
+		PreferSubdomain bool           `datastore:"prefer_subdomain" json:"-"`
+		Domain          string         `datastore:"domain" json:"domain,omitempty"`
+		IsDomainActive  bool           `datastore:"is_active" json:"-"`
+		IsSecure        bool           `datastore:"is_secure" json:"-"`
+		CustomHandle    string         `datastore:"handle" json:"-"`
+		Email           string         `json:"email,omitempty"`
+		URL             string         `json:"url,omitempty"`
+
+		app *app
+	}
+	CollectionObj struct {
+		Collection
+		TotalPosts int           `json:"total_posts"`
+		Owner      *User         `json:"owner,omitempty"`
+		Posts      *[]PublicPost `json:"posts,omitempty"`
+	}
+	SubmittedCollection struct {
+		// Data used for updating a given collection
+		ID      int64
+		OwnerID uint64
+
+		// Form helpers
+		PreferURL string `schema:"prefer_url" json:"prefer_url"`
+		Privacy   int    `schema:"privacy" json:"privacy"`
+		Pass      string `schema:"password" json:"password"`
+		Federate  bool   `schema:"federate" json:"federate"`
+		MathJax   bool   `schema:"mathjax" json:"mathjax"`
+		Handle    string `schema:"handle" json:"handle"`
+
+		// Actual collection values updated in the DB
+		Alias           *string         `schema:"alias" json:"alias"`
+		Title           *string         `schema:"title" json:"title"`
+		Description     *string         `schema:"description" json:"description"`
+		StyleSheet      *sql.NullString `schema:"style_sheet" json:"style_sheet"`
+		Script          *sql.NullString `schema:"script" json:"script"`
+		Visibility      *int            `schema:"visibility" json:"public"`
+		Format          *sql.NullString `schema:"format" json:"format"`
+		PreferSubdomain *bool           `schema:"prefer_subdomain" json:"prefer_subdomain"`
+		Domain          *sql.NullString `schema:"domain" json:"domain"`
+	}
+	CollectionFormat struct {
+		Format string
+	}
+)
+
+// collVisibility represents the visibility level for the collection.
+type collVisibility int

config/config.go

@@ -17,7 +17,8 @@ type (
 	DatabaseCfg struct {
 		Type     string `ini:"type"`
 		User     string `ini:"username"`
-		Pass string `ini:"password"`
+		Password string `ini:"password"`
+		Database string `ini:"database"`
 		Host     string `ini:"host"`
 		Port     int    `ini:"port"`
 	}

errors.go

@@ -7,5 +7,24 @@ import (
 
 // Commonly returned HTTP errors
 var (
-	ErrInternalCookieSession = impart.HTTPError{http.StatusInternalServerError, "Could not get cookie session."}
+	ErrBadAccessToken = impart.HTTPError{http.StatusUnauthorized, "Invalid access token."}
+	ErrNoAccessToken  = impart.HTTPError{http.StatusBadRequest, "Authorization token required."}
+
+	ErrForbiddenCollection  = impart.HTTPError{http.StatusForbidden, "You don't have permission to add to this collection."}
+	ErrUnauthorizedEditPost = impart.HTTPError{http.StatusUnauthorized, "Invalid editing credentials."}
+	ErrUnauthorizedGeneral  = impart.HTTPError{http.StatusUnauthorized, "You don't have permission to do that."}
+
+	ErrInternalGeneral = impart.HTTPError{http.StatusInternalServerError, "The humans messed something up. They've been notified."}
+
+	ErrCollectionPageNotFound = impart.HTTPError{http.StatusNotFound, "Collection page doesn't exist."}
+	ErrPostNotFound           = impart.HTTPError{Status: http.StatusNotFound, Message: "Post not found."}
+	ErrPostUnpublished        = impart.HTTPError{Status: http.StatusGone, Message: "Post unpublished by author."}
+	ErrPostFetchError         = impart.HTTPError{Status: http.StatusInternalServerError, Message: "We encountered an error getting the post. The humans have been alerted."}
+
+	ErrUserNotFound = impart.HTTPError{http.StatusNotFound, "User doesn't exist."}
+)
+
+// Post operation errors
+var (
+	ErrPostNoUpdatableVals = impart.HTTPError{http.StatusBadRequest, "Supply some properties to update."}
 )

postrender.go

@@ -0,0 +1,135 @@
+package writefreely
+
+import (
+	"bytes"
+	"github.com/microcosm-cc/bluemonday"
+	stripmd "github.com/writeas/go-strip-markdown"
+	"github.com/writeas/saturday"
+	"html"
+	"html/template"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+var (
+	blockReg        = regexp.MustCompile("<(ul|ol|blockquote)>\n")
+	endBlockReg     = regexp.MustCompile("</([a-z]+)>\n</(ul|ol|blockquote)>")
+	youtubeReg      = regexp.MustCompile("(https?://www.youtube.com/embed/[a-zA-Z0-9\\-_]+)(\\?[^\t\n\f\r \"']+)?")
+	titleElementReg = regexp.MustCompile("</?h[1-6]>")
+	hashtagReg      = regexp.MustCompile(`#([\p{L}\p{M}\d]+)`)
+	markeddownReg   = regexp.MustCompile("<p>(.+)</p>")
+)
+
+func (p *Post) formatContent(c *Collection, isOwner bool) {
+	baseURL := c.CanonicalURL()
+	if isOwner {
+		baseURL = "/" + c.Alias + "/"
+	}
+	newCon := hashtagReg.ReplaceAllFunc([]byte(p.Content), func(b []byte) []byte {
+		// Ensure we only replace "hashtags" that have already been extracted.
+		// `hashtagReg` catches everything, including any hash on the end of a
+		// URL, so we rely on p.Tags as the final word on whether or not to link
+		// a tag.
+		for _, t := range p.Tags {
+			if string(b) == "#"+t {
+				return bytes.Replace(b, []byte("#"+t), []byte("<a href=\""+baseURL+"tag:"+t+"\" class=\"hashtag\"><span>#</span><span class=\"p-category\">"+t+"</span></a>"), -1)
+			}
+		}
+		return b
+	})
+	p.HTMLTitle = template.HTML(applyBasicMarkdown([]byte(p.Title.String)))
+	p.HTMLContent = template.HTML(applyMarkdown([]byte(newCon)))
+	if exc := strings.Index(string(newCon), "<!--more-->"); exc > -1 {
+		p.HTMLExcerpt = template.HTML(applyMarkdown([]byte(newCon[:exc])))
+	}
+}
+
+func (p *PublicPost) formatContent(isOwner bool) {
+	p.Post.formatContent(&p.Collection.Collection, isOwner)
+}
+
+func applyMarkdown(data []byte) string {
+	return applyMarkdownSpecial(data, false)
+}
+
+func applyMarkdownSpecial(data []byte, skipNoFollow bool) string {
+	mdExtensions := 0 |
+		blackfriday.EXTENSION_TABLES |
+		blackfriday.EXTENSION_FENCED_CODE |
+		blackfriday.EXTENSION_AUTOLINK |
+		blackfriday.EXTENSION_STRIKETHROUGH |
+		blackfriday.EXTENSION_SPACE_HEADERS |
+		blackfriday.EXTENSION_AUTO_HEADER_IDS
+	htmlFlags := 0 |
+		blackfriday.HTML_USE_SMARTYPANTS |
+		blackfriday.HTML_SMARTYPANTS_DASHES
+
+	// Generate Markdown
+	md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions)
+	// Strip out bad HTML
+	policy := getSanitizationPolicy()
+	policy.RequireNoFollowOnLinks(!skipNoFollow)
+	outHTML := string(policy.SanitizeBytes(md))
+	// Strip newlines on certain block elements that render with them
+	outHTML = blockReg.ReplaceAllString(outHTML, "<$1>")
+	outHTML = endBlockReg.ReplaceAllString(outHTML, "</$1></$2>")
+	// Remove all query parameters on YouTube embed links
+	// TODO: make this more specific. Taking the nuclear approach here to strip ?autoplay=1
+	outHTML = youtubeReg.ReplaceAllString(outHTML, "$1")
+
+	return outHTML
+}
+
+func applyBasicMarkdown(data []byte) string {
+	mdExtensions := 0 |
+		blackfriday.EXTENSION_STRIKETHROUGH |
+		blackfriday.EXTENSION_SPACE_HEADERS |
+		blackfriday.EXTENSION_HEADER_IDS
+	htmlFlags := 0 |
+		blackfriday.HTML_SKIP_HTML |
+		blackfriday.HTML_USE_SMARTYPANTS |
+		blackfriday.HTML_SMARTYPANTS_DASHES
+
+	// Generate Markdown
+	md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions)
+	// Strip out bad HTML
+	policy := bluemonday.UGCPolicy()
+	policy.AllowAttrs("class", "id").Globally()
+	outHTML := string(policy.SanitizeBytes(md))
+	outHTML = markeddownReg.ReplaceAllString(outHTML, "$1")
+	outHTML = strings.TrimRightFunc(outHTML, unicode.IsSpace)
+
+	return outHTML
+}
+
+func postTitle(content, friendlyId string) string {
+	const maxTitleLen = 80
+
+	// Strip HTML tags with bluemonday's StrictPolicy, then unescape the HTML
+	// entities added in by sanitizing the content.
+	content = html.UnescapeString(bluemonday.StrictPolicy().Sanitize(content))
+
+	content = strings.TrimLeftFunc(stripmd.Strip(content), unicode.IsSpace)
+	eol := strings.IndexRune(content, '\n')
+	blankLine := strings.Index(content, "\n\n")
+	if blankLine != -1 && blankLine <= eol && blankLine <= assumedTitleLen {
+		return strings.TrimSpace(content[:blankLine])
+	} else if utf8.RuneCountInString(content) <= maxTitleLen {
+		return content
+	}
+	return friendlyId
+}
+
+func getSanitizationPolicy() *bluemonday.Policy {
+	policy := bluemonday.UGCPolicy()
+	policy.AllowAttrs("src", "style").OnElements("iframe", "video")
+	policy.AllowAttrs("frameborder", "width", "height").Matching(bluemonday.Integer).OnElements("iframe")
+	policy.AllowAttrs("allowfullscreen").OnElements("iframe")
+	policy.AllowAttrs("controls", "loop", "muted", "autoplay").OnElements("video")
+	policy.AllowAttrs("target").OnElements("a")
+	policy.AllowAttrs("style", "class", "id").Globally()
+	policy.AllowURLSchemes("http", "https", "mailto", "xmpp")
+	return policy
+}

posts.go

@@ -0,0 +1,178 @@
+package writefreely
+
+import (
+	"github.com/guregu/null"
+	"github.com/guregu/null/zero"
+	"github.com/kylemcc/twitter-text-go/extract"
+	"github.com/writeas/monday"
+	"github.com/writeas/slug"
+	"github.com/writeas/web-core/converter"
+	"github.com/writeas/web-core/parse"
+	"github.com/writeas/web-core/tags"
+	"html/template"
+	"regexp"
+	"time"
+)
+
+const (
+	// Post ID length bounds
+	minIDLen      = 10
+	maxIDLen      = 10
+	userPostIDLen = 10
+	postIDLen     = 10
+
+	postMetaDateFormat = "2006-01-02 15:04:05"
+)
+
+type (
+	AuthenticatedPost struct {
+		ID string `json:"id" schema:"id"`
+		*SubmittedPost
+	}
+
+	// SubmittedPost represents a post supplied by a client for publishing or
+	// updating. Since Title and Content can be updated to "", they are
+	// pointers that can be easily tested to detect changes.
+	SubmittedPost struct {
+		Slug     *string                  `json:"slug" schema:"slug"`
+		Title    *string                  `json:"title" schema:"title"`
+		Content  *string                  `json:"body" schema:"body"`
+		Font     string                   `json:"font" schema:"font"`
+		IsRTL    converter.NullJSONBool   `json:"rtl" schema:"rtl"`
+		Language converter.NullJSONString `json:"lang" schema:"lang"`
+		Created  *string                  `json:"created" schema:"created"`
+
+		// [{ "medium": "ev" }, { "twitter": "ilikebeans" }]
+		Crosspost []map[string]string `json:"crosspost" schema:"crosspost"`
+	}
+
+	// Post represents a post as found in the database.
+	Post struct {
+		ID             string        `db:"id" json:"id"`
+		Slug           null.String   `db:"slug" json:"slug,omitempty"`
+		Font           string        `db:"text_appearance" json:"appearance"`
+		Language       zero.String   `db:"language" json:"language"`
+		RTL            zero.Bool     `db:"rtl" json:"rtl"`
+		Privacy        int64         `db:"privacy" json:"-"`
+		OwnerID        null.Int      `db:"owner_id" json:"-"`
+		CollectionID   null.Int      `db:"collection_id" json:"-"`
+		PinnedPosition null.Int      `db:"pinned_position" json:"-"`
+		Created        time.Time     `db:"created" json:"created"`
+		Updated        time.Time     `db:"updated" json:"updated"`
+		ViewCount      int64         `db:"view_count" json:"-"`
+		EmbedViewCount int64         `db:"embed_view_count" json:"-"`
+		Title          zero.String   `db:"title" json:"title"`
+		HTMLTitle      template.HTML `db:"title" json:"-"`
+		Content        string        `db:"content" json:"body"`
+		HTMLContent    template.HTML `db:"content" json:"-"`
+		HTMLExcerpt    template.HTML `db:"content" json:"-"`
+		Tags           []string      `json:"tags"`
+		Images         []string      `json:"images,omitempty"`
+
+		OwnerName string `json:"owner,omitempty"`
+	}
+
+	// PublicPost holds properties for a publicly returned post, i.e. a post in
+	// a context where the viewer may not be the owner. As such, sensitive
+	// metadata for the post is hidden and properties supporting the display of
+	// the post are added.
+	PublicPost struct {
+		*Post
+		IsSubdomain bool           `json:"-"`
+		IsTopLevel  bool           `json:"-"`
+		Domain      string         `json:"-"`
+		DisplayDate string         `json:"-"`
+		Views       int64          `json:"views"`
+		Owner       *PublicUser    `json:"-"`
+		IsOwner     bool           `json:"-"`
+		Collection  *CollectionObj `json:"collection,omitempty"`
+	}
+
+	AnonymousAuthPost struct {
+		ID    string `json:"id"`
+		Token string `json:"token"`
+	}
+	ClaimPostRequest struct {
+		*AnonymousAuthPost
+		CollectionAlias  string `json:"collection"`
+		CreateCollection bool   `json:"create_collection"`
+
+		// Generated properties
+		Slug string `json:"-"`
+	}
+	ClaimPostResult struct {
+		ID           string      `json:"id,omitempty"`
+		Code         int         `json:"code,omitempty"`
+		ErrorMessage string      `json:"error_msg,omitempty"`
+		Post         *PublicPost `json:"post,omitempty"`
+	}
+)
+
+func (p *Post) processPost() PublicPost {
+	res := &PublicPost{Post: p, Views: 0}
+	res.Views = p.ViewCount
+	// TODO: move to own function
+	loc := monday.FuzzyLocale(p.Language.String)
+	res.DisplayDate = monday.Format(p.Created, monday.LongFormatsByLocale[loc], loc)
+
+	return *res
+}
+
+// TODO: merge this into getSlugFromPost or phase it out
+func getSlug(title, lang string) string {
+	return getSlugFromPost("", title, lang)
+}
+
+func getSlugFromPost(title, body, lang string) string {
+	if title == "" {
+		title = postTitle(body, body)
+	}
+	title = parse.PostLede(title, false)
+	// Truncate lede if needed
+	title, _ = parse.TruncToWord(title, 80)
+	if lang != "" && len(lang) == 2 {
+		return slug.MakeLang(title, lang)
+	}
+	return slug.Make(title)
+}
+
+// isFontValid returns whether or not the submitted post's appearance is valid.
+func (p *SubmittedPost) isFontValid() bool {
+	validFonts := map[string]bool{
+		"norm": true,
+		"sans": true,
+		"mono": true,
+		"wrap": true,
+		"code": true,
+	}
+
+	if _, valid := validFonts[p.Font]; valid {
+		return true
+	}
+	return false
+}
+
+func (p *Post) extractData() {
+	p.Tags = tags.Extract(p.Content)
+	p.extractImages()
+}
+
+var imageURLRegex = regexp.MustCompile(`(?i)^https?:\/\/[^ ]*\.(gif|png|jpg|jpeg)$`)
+
+func (p *Post) extractImages() {
+	matches := extract.ExtractUrls(p.Content)
+	urls := map[string]bool{}
+	for i := range matches {
+		u := matches[i].Text
+		if !imageURLRegex.MatchString(u) {
+			continue
+		}
+		urls[u] = true
+	}
+
+	resURLs := make([]string, 0)
+	for k := range urls {
+		resURLs = append(resURLs, k)
+	}
+	p.Images = resURLs
+}

routes.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-func initRoutes(handler *Handler, r *mux.Router, cfg *config.Config) {
+func initRoutes(handler *Handler, r *mux.Router, cfg *config.Config, db *datastore) {
 	isSingleUser := !cfg.App.MultiUser
 
 	// Write.as router