Add data layer
This includes config changes, collections, posts, some post rendering funcs, and actual database connection when the server starts up.
This commit is contained in:
Matt Baer
parent
f7430fb8bc
commit
0c1e1dd57e
28
app.go
28
app.go
|
|
@ -1,6 +1,7 @@
|
|||
package writefreely
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
|
@ -21,6 +22,7 @@ const (
|
|||
|
||||
type app struct {
|
||||
router *mux.Router
|
||||
db *datastore
|
||||
cfg *config.Config
|
||||
keys *keychain
|
||||
sessionStore *sessions.CookieStore
|
||||
|
|
@ -62,11 +64,33 @@ func Serve() {
|
|||
// Initialize modules
|
||||
app.sessionStore = initSession(app)
|
||||
|
||||
// Check database configuration
|
||||
if app.cfg.Database.User == "" || app.cfg.Database.Password == "" {
|
||||
log.Error("Database user or password not set.")
|
||||
os.Exit(1)
|
||||
}
|
||||
if app.cfg.Database.Host == "" {
|
||||
app.cfg.Database.Host = "localhost"
|
||||
}
|
||||
if app.cfg.Database.Database == "" {
|
||||
app.cfg.Database.Database = "writeas"
|
||||
}
|
||||
|
||||
log.Info("Connecting to database...")
|
||||
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=true", app.cfg.Database.User, app.cfg.Database.Password, app.cfg.Database.Host, app.cfg.Database.Port, app.cfg.Database.Database))
|
||||
if err != nil {
|
||||
log.Error("\n%s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
app.db = &datastore{db}
|
||||
defer shutdown(app)
|
||||
app.db.SetMaxOpenConns(50)
|
||||
|
||||
r := mux.NewRouter()
|
||||
handler := NewHandler(app.sessionStore)
|
||||
|
||||
// Handle app routes
|
||||
initRoutes(handler, r, app.cfg)
|
||||
initRoutes(handler, r, app.cfg, app.db)
|
||||
|
||||
// Handle static files
|
||||
fs := http.FileServer(http.Dir(staticDir))
|
||||
|
|
@ -92,4 +116,6 @@ func Serve() {
|
|||
}
|
||||
|
||||
func shutdown(app *app) {
|
||||
log.Info("Closing database connection...")
|
||||
app.db.Close()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
package writefreely
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
type (
|
||||
Collection struct {
|
||||
ID int64 `datastore:"id" json:"-"`
|
||||
Alias string `datastore:"alias" schema:"alias" json:"alias"`
|
||||
Title string `datastore:"title" schema:"title" json:"title"`
|
||||
Description string `datastore:"description" schema:"description" json:"description"`
|
||||
Direction string `schema:"dir" json:"dir,omitempty"`
|
||||
Language string `schema:"lang" json:"lang,omitempty"`
|
||||
StyleSheet string `datastore:"style_sheet" schema:"style_sheet" json:"style_sheet"`
|
||||
Script string `datastore:"script" schema:"script" json:"script,omitempty"`
|
||||
Public bool `datastore:"public" json:"public"`
|
||||
Visibility collVisibility `datastore:"private" json:"-"`
|
||||
Format string `datastore:"format" json:"format,omitempty"`
|
||||
Views int64 `json:"views"`
|
||||
OwnerID int64 `datastore:"owner_id" json:"-"`
|
||||
PublicOwner bool `datastore:"public_owner" json:"-"`
|
||||
PreferSubdomain bool `datastore:"prefer_subdomain" json:"-"`
|
||||
Domain string `datastore:"domain" json:"domain,omitempty"`
|
||||
IsDomainActive bool `datastore:"is_active" json:"-"`
|
||||
IsSecure bool `datastore:"is_secure" json:"-"`
|
||||
CustomHandle string `datastore:"handle" json:"-"`
|
||||
Email string `json:"email,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
|
||||
app *app
|
||||
}
|
||||
CollectionObj struct {
|
||||
Collection
|
||||
TotalPosts int `json:"total_posts"`
|
||||
Owner *User `json:"owner,omitempty"`
|
||||
Posts *[]PublicPost `json:"posts,omitempty"`
|
||||
}
|
||||
SubmittedCollection struct {
|
||||
// Data used for updating a given collection
|
||||
ID int64
|
||||
OwnerID uint64
|
||||
|
||||
// Form helpers
|
||||
PreferURL string `schema:"prefer_url" json:"prefer_url"`
|
||||
Privacy int `schema:"privacy" json:"privacy"`
|
||||
Pass string `schema:"password" json:"password"`
|
||||
Federate bool `schema:"federate" json:"federate"`
|
||||
MathJax bool `schema:"mathjax" json:"mathjax"`
|
||||
Handle string `schema:"handle" json:"handle"`
|
||||
|
||||
// Actual collection values updated in the DB
|
||||
Alias *string `schema:"alias" json:"alias"`
|
||||
Title *string `schema:"title" json:"title"`
|
||||
Description *string `schema:"description" json:"description"`
|
||||
StyleSheet *sql.NullString `schema:"style_sheet" json:"style_sheet"`
|
||||
Script *sql.NullString `schema:"script" json:"script"`
|
||||
Visibility *int `schema:"visibility" json:"public"`
|
||||
Format *sql.NullString `schema:"format" json:"format"`
|
||||
PreferSubdomain *bool `schema:"prefer_subdomain" json:"prefer_subdomain"`
|
||||
Domain *sql.NullString `schema:"domain" json:"domain"`
|
||||
}
|
||||
CollectionFormat struct {
|
||||
Format string
|
||||
}
|
||||
)
|
||||
|
||||
// collVisibility represents the visibility level for the collection.
|
||||
type collVisibility int
|
||||
|
|
@ -15,11 +15,12 @@ type (
|
|||
}
|
||||
|
||||
DatabaseCfg struct {
|
||||
Type string `ini:"type"`
|
||||
User string `ini:"username"`
|
||||
Pass string `ini:"password"`
|
||||
Host string `ini:"host"`
|
||||
Port int `ini:"port"`
|
||||
Type string `ini:"type"`
|
||||
User string `ini:"username"`
|
||||
Password string `ini:"password"`
|
||||
Database string `ini:"database"`
|
||||
Host string `ini:"host"`
|
||||
Port int `ini:"port"`
|
||||
}
|
||||
|
||||
AppCfg struct {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
21
errors.go
21
errors.go
|
|
@ -7,5 +7,24 @@ import (
|
|||
|
||||
// Commonly returned HTTP errors
|
||||
var (
|
||||
ErrInternalCookieSession = impart.HTTPError{http.StatusInternalServerError, "Could not get cookie session."}
|
||||
ErrBadAccessToken = impart.HTTPError{http.StatusUnauthorized, "Invalid access token."}
|
||||
ErrNoAccessToken = impart.HTTPError{http.StatusBadRequest, "Authorization token required."}
|
||||
|
||||
ErrForbiddenCollection = impart.HTTPError{http.StatusForbidden, "You don't have permission to add to this collection."}
|
||||
ErrUnauthorizedEditPost = impart.HTTPError{http.StatusUnauthorized, "Invalid editing credentials."}
|
||||
ErrUnauthorizedGeneral = impart.HTTPError{http.StatusUnauthorized, "You don't have permission to do that."}
|
||||
|
||||
ErrInternalGeneral = impart.HTTPError{http.StatusInternalServerError, "The humans messed something up. They've been notified."}
|
||||
|
||||
ErrCollectionPageNotFound = impart.HTTPError{http.StatusNotFound, "Collection page doesn't exist."}
|
||||
ErrPostNotFound = impart.HTTPError{Status: http.StatusNotFound, Message: "Post not found."}
|
||||
ErrPostUnpublished = impart.HTTPError{Status: http.StatusGone, Message: "Post unpublished by author."}
|
||||
ErrPostFetchError = impart.HTTPError{Status: http.StatusInternalServerError, Message: "We encountered an error getting the post. The humans have been alerted."}
|
||||
|
||||
ErrUserNotFound = impart.HTTPError{http.StatusNotFound, "User doesn't exist."}
|
||||
)
|
||||
|
||||
// Post operation errors
|
||||
var (
|
||||
ErrPostNoUpdatableVals = impart.HTTPError{http.StatusBadRequest, "Supply some properties to update."}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,135 @@
|
|||
package writefreely
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
stripmd "github.com/writeas/go-strip-markdown"
|
||||
"github.com/writeas/saturday"
|
||||
"html"
|
||||
"html/template"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
blockReg = regexp.MustCompile("<(ul|ol|blockquote)>\n")
|
||||
endBlockReg = regexp.MustCompile("</([a-z]+)>\n</(ul|ol|blockquote)>")
|
||||
youtubeReg = regexp.MustCompile("(https?://www.youtube.com/embed/[a-zA-Z0-9\\-_]+)(\\?[^\t\n\f\r \"']+)?")
|
||||
titleElementReg = regexp.MustCompile("</?h[1-6]>")
|
||||
hashtagReg = regexp.MustCompile(`#([\p{L}\p{M}\d]+)`)
|
||||
markeddownReg = regexp.MustCompile("<p>(.+)</p>")
|
||||
)
|
||||
|
||||
func (p *Post) formatContent(c *Collection, isOwner bool) {
|
||||
baseURL := c.CanonicalURL()
|
||||
if isOwner {
|
||||
baseURL = "/" + c.Alias + "/"
|
||||
}
|
||||
newCon := hashtagReg.ReplaceAllFunc([]byte(p.Content), func(b []byte) []byte {
|
||||
// Ensure we only replace "hashtags" that have already been extracted.
|
||||
// `hashtagReg` catches everything, including any hash on the end of a
|
||||
// URL, so we rely on p.Tags as the final word on whether or not to link
|
||||
// a tag.
|
||||
for _, t := range p.Tags {
|
||||
if string(b) == "#"+t {
|
||||
return bytes.Replace(b, []byte("#"+t), []byte("<a href=\""+baseURL+"tag:"+t+"\" class=\"hashtag\"><span>#</span><span class=\"p-category\">"+t+"</span></a>"), -1)
|
||||
}
|
||||
}
|
||||
return b
|
||||
})
|
||||
p.HTMLTitle = template.HTML(applyBasicMarkdown([]byte(p.Title.String)))
|
||||
p.HTMLContent = template.HTML(applyMarkdown([]byte(newCon)))
|
||||
if exc := strings.Index(string(newCon), "<!--more-->"); exc > -1 {
|
||||
p.HTMLExcerpt = template.HTML(applyMarkdown([]byte(newCon[:exc])))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PublicPost) formatContent(isOwner bool) {
|
||||
p.Post.formatContent(&p.Collection.Collection, isOwner)
|
||||
}
|
||||
|
||||
func applyMarkdown(data []byte) string {
|
||||
return applyMarkdownSpecial(data, false)
|
||||
}
|
||||
|
||||
func applyMarkdownSpecial(data []byte, skipNoFollow bool) string {
|
||||
mdExtensions := 0 |
|
||||
blackfriday.EXTENSION_TABLES |
|
||||
blackfriday.EXTENSION_FENCED_CODE |
|
||||
blackfriday.EXTENSION_AUTOLINK |
|
||||
blackfriday.EXTENSION_STRIKETHROUGH |
|
||||
blackfriday.EXTENSION_SPACE_HEADERS |
|
||||
blackfriday.EXTENSION_AUTO_HEADER_IDS
|
||||
htmlFlags := 0 |
|
||||
blackfriday.HTML_USE_SMARTYPANTS |
|
||||
blackfriday.HTML_SMARTYPANTS_DASHES
|
||||
|
||||
// Generate Markdown
|
||||
md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions)
|
||||
// Strip out bad HTML
|
||||
policy := getSanitizationPolicy()
|
||||
policy.RequireNoFollowOnLinks(!skipNoFollow)
|
||||
outHTML := string(policy.SanitizeBytes(md))
|
||||
// Strip newlines on certain block elements that render with them
|
||||
outHTML = blockReg.ReplaceAllString(outHTML, "<$1>")
|
||||
outHTML = endBlockReg.ReplaceAllString(outHTML, "</$1></$2>")
|
||||
// Remove all query parameters on YouTube embed links
|
||||
// TODO: make this more specific. Taking the nuclear approach here to strip ?autoplay=1
|
||||
outHTML = youtubeReg.ReplaceAllString(outHTML, "$1")
|
||||
|
||||
return outHTML
|
||||
}
|
||||
|
||||
func applyBasicMarkdown(data []byte) string {
|
||||
mdExtensions := 0 |
|
||||
blackfriday.EXTENSION_STRIKETHROUGH |
|
||||
blackfriday.EXTENSION_SPACE_HEADERS |
|
||||
blackfriday.EXTENSION_HEADER_IDS
|
||||
htmlFlags := 0 |
|
||||
blackfriday.HTML_SKIP_HTML |
|
||||
blackfriday.HTML_USE_SMARTYPANTS |
|
||||
blackfriday.HTML_SMARTYPANTS_DASHES
|
||||
|
||||
// Generate Markdown
|
||||
md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions)
|
||||
// Strip out bad HTML
|
||||
policy := bluemonday.UGCPolicy()
|
||||
policy.AllowAttrs("class", "id").Globally()
|
||||
outHTML := string(policy.SanitizeBytes(md))
|
||||
outHTML = markeddownReg.ReplaceAllString(outHTML, "$1")
|
||||
outHTML = strings.TrimRightFunc(outHTML, unicode.IsSpace)
|
||||
|
||||
return outHTML
|
||||
}
|
||||
|
||||
func postTitle(content, friendlyId string) string {
|
||||
const maxTitleLen = 80
|
||||
|
||||
// Strip HTML tags with bluemonday's StrictPolicy, then unescape the HTML
|
||||
// entities added in by sanitizing the content.
|
||||
content = html.UnescapeString(bluemonday.StrictPolicy().Sanitize(content))
|
||||
|
||||
content = strings.TrimLeftFunc(stripmd.Strip(content), unicode.IsSpace)
|
||||
eol := strings.IndexRune(content, '\n')
|
||||
blankLine := strings.Index(content, "\n\n")
|
||||
if blankLine != -1 && blankLine <= eol && blankLine <= assumedTitleLen {
|
||||
return strings.TrimSpace(content[:blankLine])
|
||||
} else if utf8.RuneCountInString(content) <= maxTitleLen {
|
||||
return content
|
||||
}
|
||||
return friendlyId
|
||||
}
|
||||
|
||||
func getSanitizationPolicy() *bluemonday.Policy {
|
||||
policy := bluemonday.UGCPolicy()
|
||||
policy.AllowAttrs("src", "style").OnElements("iframe", "video")
|
||||
policy.AllowAttrs("frameborder", "width", "height").Matching(bluemonday.Integer).OnElements("iframe")
|
||||
policy.AllowAttrs("allowfullscreen").OnElements("iframe")
|
||||
policy.AllowAttrs("controls", "loop", "muted", "autoplay").OnElements("video")
|
||||
policy.AllowAttrs("target").OnElements("a")
|
||||
policy.AllowAttrs("style", "class", "id").Globally()
|
||||
policy.AllowURLSchemes("http", "https", "mailto", "xmpp")
|
||||
return policy
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
package writefreely
|
||||
|
||||
import (
|
||||
"github.com/guregu/null"
|
||||
"github.com/guregu/null/zero"
|
||||
"github.com/kylemcc/twitter-text-go/extract"
|
||||
"github.com/writeas/monday"
|
||||
"github.com/writeas/slug"
|
||||
"github.com/writeas/web-core/converter"
|
||||
"github.com/writeas/web-core/parse"
|
||||
"github.com/writeas/web-core/tags"
|
||||
"html/template"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// Post ID length bounds
|
||||
minIDLen = 10
|
||||
maxIDLen = 10
|
||||
userPostIDLen = 10
|
||||
postIDLen = 10
|
||||
|
||||
postMetaDateFormat = "2006-01-02 15:04:05"
|
||||
)
|
||||
|
||||
type (
|
||||
AuthenticatedPost struct {
|
||||
ID string `json:"id" schema:"id"`
|
||||
*SubmittedPost
|
||||
}
|
||||
|
||||
// SubmittedPost represents a post supplied by a client for publishing or
|
||||
// updating. Since Title and Content can be updated to "", they are
|
||||
// pointers that can be easily tested to detect changes.
|
||||
SubmittedPost struct {
|
||||
Slug *string `json:"slug" schema:"slug"`
|
||||
Title *string `json:"title" schema:"title"`
|
||||
Content *string `json:"body" schema:"body"`
|
||||
Font string `json:"font" schema:"font"`
|
||||
IsRTL converter.NullJSONBool `json:"rtl" schema:"rtl"`
|
||||
Language converter.NullJSONString `json:"lang" schema:"lang"`
|
||||
Created *string `json:"created" schema:"created"`
|
||||
|
||||
// [{ "medium": "ev" }, { "twitter": "ilikebeans" }]
|
||||
Crosspost []map[string]string `json:"crosspost" schema:"crosspost"`
|
||||
}
|
||||
|
||||
// Post represents a post as found in the database.
|
||||
Post struct {
|
||||
ID string `db:"id" json:"id"`
|
||||
Slug null.String `db:"slug" json:"slug,omitempty"`
|
||||
Font string `db:"text_appearance" json:"appearance"`
|
||||
Language zero.String `db:"language" json:"language"`
|
||||
RTL zero.Bool `db:"rtl" json:"rtl"`
|
||||
Privacy int64 `db:"privacy" json:"-"`
|
||||
OwnerID null.Int `db:"owner_id" json:"-"`
|
||||
CollectionID null.Int `db:"collection_id" json:"-"`
|
||||
PinnedPosition null.Int `db:"pinned_position" json:"-"`
|
||||
Created time.Time `db:"created" json:"created"`
|
||||
Updated time.Time `db:"updated" json:"updated"`
|
||||
ViewCount int64 `db:"view_count" json:"-"`
|
||||
EmbedViewCount int64 `db:"embed_view_count" json:"-"`
|
||||
Title zero.String `db:"title" json:"title"`
|
||||
HTMLTitle template.HTML `db:"title" json:"-"`
|
||||
Content string `db:"content" json:"body"`
|
||||
HTMLContent template.HTML `db:"content" json:"-"`
|
||||
HTMLExcerpt template.HTML `db:"content" json:"-"`
|
||||
Tags []string `json:"tags"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
|
||||
OwnerName string `json:"owner,omitempty"`
|
||||
}
|
||||
|
||||
// PublicPost holds properties for a publicly returned post, i.e. a post in
|
||||
// a context where the viewer may not be the owner. As such, sensitive
|
||||
// metadata for the post is hidden and properties supporting the display of
|
||||
// the post are added.
|
||||
PublicPost struct {
|
||||
*Post
|
||||
IsSubdomain bool `json:"-"`
|
||||
IsTopLevel bool `json:"-"`
|
||||
Domain string `json:"-"`
|
||||
DisplayDate string `json:"-"`
|
||||
Views int64 `json:"views"`
|
||||
Owner *PublicUser `json:"-"`
|
||||
IsOwner bool `json:"-"`
|
||||
Collection *CollectionObj `json:"collection,omitempty"`
|
||||
}
|
||||
|
||||
AnonymousAuthPost struct {
|
||||
ID string `json:"id"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
ClaimPostRequest struct {
|
||||
*AnonymousAuthPost
|
||||
CollectionAlias string `json:"collection"`
|
||||
CreateCollection bool `json:"create_collection"`
|
||||
|
||||
// Generated properties
|
||||
Slug string `json:"-"`
|
||||
}
|
||||
ClaimPostResult struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Code int `json:"code,omitempty"`
|
||||
ErrorMessage string `json:"error_msg,omitempty"`
|
||||
Post *PublicPost `json:"post,omitempty"`
|
||||
}
|
||||
)
|
||||
|
||||
func (p *Post) processPost() PublicPost {
|
||||
res := &PublicPost{Post: p, Views: 0}
|
||||
res.Views = p.ViewCount
|
||||
// TODO: move to own function
|
||||
loc := monday.FuzzyLocale(p.Language.String)
|
||||
res.DisplayDate = monday.Format(p.Created, monday.LongFormatsByLocale[loc], loc)
|
||||
|
||||
return *res
|
||||
}
|
||||
|
||||
// TODO: merge this into getSlugFromPost or phase it out
|
||||
func getSlug(title, lang string) string {
|
||||
return getSlugFromPost("", title, lang)
|
||||
}
|
||||
|
||||
func getSlugFromPost(title, body, lang string) string {
|
||||
if title == "" {
|
||||
title = postTitle(body, body)
|
||||
}
|
||||
title = parse.PostLede(title, false)
|
||||
// Truncate lede if needed
|
||||
title, _ = parse.TruncToWord(title, 80)
|
||||
if lang != "" && len(lang) == 2 {
|
||||
return slug.MakeLang(title, lang)
|
||||
}
|
||||
return slug.Make(title)
|
||||
}
|
||||
|
||||
// isFontValid returns whether or not the submitted post's appearance is valid.
|
||||
func (p *SubmittedPost) isFontValid() bool {
|
||||
validFonts := map[string]bool{
|
||||
"norm": true,
|
||||
"sans": true,
|
||||
"mono": true,
|
||||
"wrap": true,
|
||||
"code": true,
|
||||
}
|
||||
|
||||
if _, valid := validFonts[p.Font]; valid {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Post) extractData() {
|
||||
p.Tags = tags.Extract(p.Content)
|
||||
p.extractImages()
|
||||
}
|
||||
|
||||
var imageURLRegex = regexp.MustCompile(`(?i)^https?:\/\/[^ ]*\.(gif|png|jpg|jpeg)$`)
|
||||
|
||||
func (p *Post) extractImages() {
|
||||
matches := extract.ExtractUrls(p.Content)
|
||||
urls := map[string]bool{}
|
||||
for i := range matches {
|
||||
u := matches[i].Text
|
||||
if !imageURLRegex.MatchString(u) {
|
||||
continue
|
||||
}
|
||||
urls[u] = true
|
||||
}
|
||||
|
||||
resURLs := make([]string, 0)
|
||||
for k := range urls {
|
||||
resURLs = append(resURLs, k)
|
||||
}
|
||||
p.Images = resURLs
|
||||
}
|
||||
Loading…
Reference in New Issue