add basic text file imports

this adds basic support for importing files as blog posts.

.txt and .md are supported at this time and the
collection is selectable, defaulting to draft.

if a collection is specified the post is federated.
This commit is contained in:
Rob Loranger 2019-08-16 14:27:24 -07:00
parent 8557119451
commit ee4fe2f4ad
No known key found for this signature in database
GPG Key ID: D6F1633A4F0903B8
7 changed files with 197 additions and 10 deletions

View File

@ -13,6 +13,13 @@ package writefreely
import (
"encoding/json"
"fmt"
"html/template"
"net/http"
"regexp"
"strings"
"sync"
"time"
"github.com/gorilla/mux"
"github.com/gorilla/sessions"
"github.com/guregu/null/zero"
@ -22,12 +29,6 @@ import (
"github.com/writeas/web-core/log"
"github.com/writeas/writefreely/author"
"github.com/writeas/writefreely/page"
"html/template"
"net/http"
"regexp"
"strings"
"sync"
"time"
)
type (

134
account_import.go Normal file
View File

@ -0,0 +1,134 @@
package writefreely
import (
"fmt"
"html/template"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"github.com/hashicorp/go-multierror"
"github.com/writeas/impart"
wfimport "github.com/writeas/import"
"github.com/writeas/web-core/log"
)
func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
// Fetch extra user data
p := NewUserPage(app, r, u, "Import", nil)
c, err := app.db.GetCollections(u)
if err != nil {
return impart.HTTPError{http.StatusInternalServerError, fmt.Sprintf("unable to fetch collections: %v", err)}
}
d := struct {
*UserPage
Collections *[]Collection
Flashes []template.HTML
}{
UserPage: p,
Collections: c,
Flashes: []template.HTML{},
}
flashes, _ := getSessionFlashes(app, w, r, nil)
for _, flash := range flashes {
d.Flashes = append(d.Flashes, template.HTML(flash))
}
showUserPage(w, "import", d)
return nil
}
func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
// limit 10MB per submission
r.ParseMultipartForm(10 << 20)
files := r.MultipartForm.File["files"]
var fileErrs []error
for _, formFile := range files {
// TODO: count uploaded files that succeed and report back with message
file, err := formFile.Open()
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename))
log.Error("import textfile: open from form: %v", err)
continue
}
defer file.Close()
tempFile, err := ioutil.TempFile("", "post-upload-*.txt")
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename))
log.Error("import textfile: create temp file: %v", err)
continue
}
defer tempFile.Close()
_, err = io.Copy(tempFile, file)
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename))
log.Error("import textfile: copy to temp: %v", err)
continue
}
info, err := tempFile.Stat()
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename))
log.Error("import textfile: stat temp file: %v", err)
continue
}
post, err := wfimport.FromFile(filepath.Join(os.TempDir(), info.Name()))
if err == wfimport.ErrEmptyFile {
// not a real error so don't log
_ = addSessionFlash(app, w, r, fmt.Sprintf("%s was empty, import skipped", formFile.Filename), nil)
continue
} else if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to read copy of %s", formFile.Filename))
log.Error("import textfile: file to post: %v", err)
continue
}
post.Collection = r.PostFormValue("collection")
coll, _ := app.db.GetCollection(post.Collection)
if coll == nil {
coll = &Collection{
ID: 0,
}
}
submittedPost := SubmittedPost{
Title: &post.Title,
Content: &post.Content,
Font: "norm",
}
if coll.ID != 0 && app.cfg.App.Federation {
token, err := app.db.GetAccessToken(u.ID)
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to authenticate uploading: %s", formFile.Filename))
log.Error("import textfile: get accesstoken: %+v", err)
continue
}
ownedPost, err := app.db.CreateOwnedPost(&submittedPost, token, coll.Alias, app.cfg.App.Host)
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to create owned post for %s", formFile.Filename))
log.Error("import textfile: create owned post: %v", err)
continue
}
go federatePost(app, ownedPost, coll.ID, false)
} else {
_, err = app.db.CreatePost(u.ID, coll.ID, &submittedPost)
if err != nil {
fileErrs = append(fileErrs, fmt.Errorf("failed to create post from %s", formFile.Filename))
log.Error("import textfile: create db post: %v", err)
continue
}
}
}
if len(fileErrs) != 0 {
_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil)
}
return impart.HTTPError{http.StatusFound, "/me/import"}
}

4
go.mod
View File

@ -28,6 +28,7 @@ require (
github.com/gorilla/schema v1.0.2
github.com/gorilla/sessions v1.1.3
github.com/guregu/null v3.4.0+incompatible
github.com/hashicorp/go-multierror v1.0.0
github.com/ikeikeikeike/go-sitemap-generator v1.0.1
github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2
github.com/imdario/mergo v0.3.7 // indirect
@ -56,6 +57,7 @@ require (
github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2
github.com/writeas/httpsig v1.0.0
github.com/writeas/impart v1.1.0
github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e
github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219
github.com/writeas/nerds v1.0.0
github.com/writeas/openssl-go v1.0.0 // indirect
@ -63,7 +65,7 @@ require (
github.com/writeas/slug v1.2.0
github.com/writeas/web-core v1.0.0
github.com/writefreely/go-nodeinfo v1.2.0
golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f // indirect
golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect
golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 // indirect
golang.org/x/sys v0.0.0-20190209173611-3b5209105503 // indirect

12
go.sum
View File

@ -1,3 +1,5 @@
code.as/core/socks v1.0.0 h1:SPQXNp4SbEwjOAP9VzUahLHak8SDqy5n+9cm9tpjZOs=
code.as/core/socks v1.0.0/go.mod h1:BAXBy5O9s2gmw6UxLqNJcVbWY7C/UPs+801CcSsfWOY=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f h1:m1tYqjD/N0vF/S8s/ZKz/eccUr8RAAcrOK2MhXeTegA=
@ -80,6 +82,10 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R
github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM=
github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/ikeikeikeike/go-sitemap-generator v1.0.1 h1:49Fn8gro/B12vCY8pf5/+/Jpr3kwB9TvP0MSymo69SY=
github.com/ikeikeikeike/go-sitemap-generator v1.0.1/go.mod h1:QI+zWsz6yQyxkG9LWNcnu0f7aiAE5tPdsZOsICgmd1c=
github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM=
@ -151,10 +157,16 @@ github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6Fk
github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE=
github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo=
github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2/go.mod h1:w2VxyRO/J5vfNjJHYVubsjUGHd3RLDoVciz0DE3ApOc=
github.com/writeas/go-writeas v1.1.0 h1:WHGm6wriBkxYAOGbvriXH8DlMUGOi6jhSZLUZKQ+4mQ=
github.com/writeas/go-writeas v1.1.0/go.mod h1:oh9U1rWaiE0p3kzdKwwvOpNXgp0P0IELI7OLOwV4fkA=
github.com/writeas/httpsig v1.0.0 h1:peIAoIA3DmlP8IG8tMNZqI4YD1uEnWBmkcC9OFPjt3A=
github.com/writeas/httpsig v1.0.0/go.mod h1:7ClMGSrSVXJbmiLa17bZ1LrG1oibGZmUMlh3402flPY=
github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE=
github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y=
github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvVuhO9UiQ9f+U1aR/x5B4MP7YQHaU=
github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs=
github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 h1:baEp0631C8sT2r/hqwypIw2snCFZa6h7U6TojoLHu/c=
github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219/go.mod h1:NyM35ayknT7lzO6O/1JpfgGyv+0W9Z9q7aE0J8bXxfQ=
github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo=

View File

@ -11,13 +11,14 @@
package writefreely
import (
"net/http"
"path/filepath"
"strings"
"github.com/gorilla/mux"
"github.com/writeas/go-webfinger"
"github.com/writeas/web-core/log"
"github.com/writefreely/go-nodeinfo"
"net/http"
"path/filepath"
"strings"
)
// InitStaticRoutes adds routes for serving static files.
@ -93,6 +94,8 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router {
me.HandleFunc("/posts/export.json", handler.Download(viewExportPosts, UserLevelUser)).Methods("GET")
me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET")
me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET")
me.HandleFunc("/import", handler.User(viewImport)).Methods("GET")
me.HandleFunc("/import", handler.User(handleImport)).Methods("POST")
me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET")
me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET")
me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET")

View File

@ -0,0 +1,33 @@
{{define "import"}}
{{template "header" .}}
<div class="snug content-container">
<h2 id="posts-header">Import</h2>
<p>Upload text or markdown files to import as posts.</p>
<div class="formContainer">
<form id="importPosts" class="import" enctype="multipart/form-data" action="/me/import" method="POST">
<label for="file" hidden>Browse files to upload</label>
<input class="fileInput" name="files" type="file" multiple accept=".txt,.md"/>
<br />
<label for="collection">Select a blog to import the posts under.</label>
<select name="collection">
{{range $i, $el := .Collections}}
<option value={{.Alias}}>
{{if .Title}}{{.Title}}{{else}}{{.Alias}}{{end}}
</option>
{{end}}
<option value="" selected>drafts</option>
</select>
<br />
<input type="submit" value="Import" />
</form>
</div>
{{if .Flashes}}
<ul class="errors">
{{range .Flashes}}<li class="urgent">{{.}}</li>{{end}}
</ul>
{{end}}
</div>
{{template "footer" .}}
{{end}}

View File

@ -27,6 +27,7 @@
{{if .IsAdmin}}<li><a href="/admin">Admin</a></li>{{end}}
<li><a href="/me/settings">Settings</a></li>
<li><a href="/me/export">Export</a></li>
<li><a href="/me/import">Import</a></li>
<li class="separator"><hr /></li>
<li><a href="/me/logout">Log out</a></li>
</ul></li>
@ -45,6 +46,7 @@
{{if .IsAdmin}}<li><a href="/admin">Admin dashboard</a></li>{{end}}
<li><a href="/me/settings">Account settings</a></li>
<li><a href="/me/export">Export</a></li>
<li><a href="/me/import">Import</a></li>
{{if .CanInvite}}<li><a href="/me/invites">Invite people</a></li>{{end}}
<li class="separator"><hr /></li>
<li><a href="/me/logout">Log out</a></li>