From ee4fe2f4adf635284080f91118d87675ad843b42 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Fri, 16 Aug 2019 14:27:24 -0700 Subject: [PATCH 01/20] add basic text file imports this adds basic support for importing files as blog posts. .txt and .md are supported at this time and the collection is selectable, defaulting to draft. if a collection is specified the post is federated. --- account.go | 13 +-- account_import.go | 134 +++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 12 +++ routes.go | 9 +- templates/user/import.tmpl | 33 +++++++ templates/user/include/header.tmpl | 2 + 7 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 account_import.go create mode 100644 templates/user/import.tmpl diff --git a/account.go b/account.go index 1cf259b..9db9b1f 100644 --- a/account.go +++ b/account.go @@ -13,6 +13,13 @@ package writefreely import ( "encoding/json" "fmt" + "html/template" + "net/http" + "regexp" + "strings" + "sync" + "time" + "github.com/gorilla/mux" "github.com/gorilla/sessions" "github.com/guregu/null/zero" @@ -22,12 +29,6 @@ import ( "github.com/writeas/web-core/log" "github.com/writeas/writefreely/author" "github.com/writeas/writefreely/page" - "html/template" - "net/http" - "regexp" - "strings" - "sync" - "time" ) type ( diff --git a/account_import.go b/account_import.go new file mode 100644 index 0000000..c8fe8a2 --- /dev/null +++ b/account_import.go @@ -0,0 +1,134 @@ +package writefreely + +import ( + "fmt" + "html/template" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/hashicorp/go-multierror" + "github.com/writeas/impart" + wfimport "github.com/writeas/import" + "github.com/writeas/web-core/log" +) + +func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { + // Fetch extra user data + p := NewUserPage(app, r, u, "Import", nil) + + c, err := app.db.GetCollections(u) + if err != nil { + return impart.HTTPError{http.StatusInternalServerError, fmt.Sprintf("unable to fetch collections: %v", err)} + } + + d := struct { + *UserPage + Collections *[]Collection + Flashes []template.HTML + }{ + UserPage: p, + Collections: c, + Flashes: []template.HTML{}, + } + + flashes, _ := getSessionFlashes(app, w, r, nil) + for _, flash := range flashes { + d.Flashes = append(d.Flashes, template.HTML(flash)) + } + + showUserPage(w, "import", d) + return nil +} + +func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { + // limit 10MB per submission + r.ParseMultipartForm(10 << 20) + files := r.MultipartForm.File["files"] + var fileErrs []error + for _, formFile := range files { + // TODO: count uploaded files that succeed and report back with message + file, err := formFile.Open() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) + log.Error("import textfile: open from form: %v", err) + continue + } + defer file.Close() + + tempFile, err := ioutil.TempFile("", "post-upload-*.txt") + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) + log.Error("import textfile: create temp file: %v", err) + continue + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, file) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) + log.Error("import textfile: copy to temp: %v", err) + continue + } + + info, err := tempFile.Stat() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) + log.Error("import textfile: stat temp file: %v", err) + continue + } + post, err := wfimport.FromFile(filepath.Join(os.TempDir(), info.Name())) + if err == wfimport.ErrEmptyFile { + // not a real error so don't log + _ = addSessionFlash(app, w, r, fmt.Sprintf("%s was empty, import skipped", formFile.Filename), nil) + continue + } else if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to read copy of %s", formFile.Filename)) + log.Error("import textfile: file to post: %v", err) + continue + } + + post.Collection = r.PostFormValue("collection") + coll, _ := app.db.GetCollection(post.Collection) + if coll == nil { + coll = &Collection{ + ID: 0, + } + } + + submittedPost := SubmittedPost{ + Title: &post.Title, + Content: &post.Content, + Font: "norm", + } + if coll.ID != 0 && app.cfg.App.Federation { + token, err := app.db.GetAccessToken(u.ID) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to authenticate uploading: %s", formFile.Filename)) + log.Error("import textfile: get accesstoken: %+v", err) + continue + } + ownedPost, err := app.db.CreateOwnedPost(&submittedPost, token, coll.Alias, app.cfg.App.Host) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create owned post for %s", formFile.Filename)) + log.Error("import textfile: create owned post: %v", err) + continue + } + go federatePost(app, ownedPost, coll.ID, false) + } else { + _, err = app.db.CreatePost(u.ID, coll.ID, &submittedPost) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create post from %s", formFile.Filename)) + log.Error("import textfile: create db post: %v", err) + continue + } + } + } + if len(fileErrs) != 0 { + _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) + } + + return impart.HTTPError{http.StatusFound, "/me/import"} +} diff --git a/go.mod b/go.mod index cc5fc57..8c62e9e 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/gorilla/schema v1.0.2 github.com/gorilla/sessions v1.1.3 github.com/guregu/null v3.4.0+incompatible + github.com/hashicorp/go-multierror v1.0.0 github.com/ikeikeikeike/go-sitemap-generator v1.0.1 github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 github.com/imdario/mergo v0.3.7 // indirect @@ -56,6 +57,7 @@ require ( github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 github.com/writeas/httpsig v1.0.0 github.com/writeas/impart v1.1.0 + github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 github.com/writeas/nerds v1.0.0 github.com/writeas/openssl-go v1.0.0 // indirect @@ -63,7 +65,7 @@ require ( github.com/writeas/slug v1.2.0 github.com/writeas/web-core v1.0.0 github.com/writefreely/go-nodeinfo v1.2.0 - golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f // indirect + golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 // indirect golang.org/x/sys v0.0.0-20190209173611-3b5209105503 // indirect diff --git a/go.sum b/go.sum index 8898bec..30e4a9d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +code.as/core/socks v1.0.0 h1:SPQXNp4SbEwjOAP9VzUahLHak8SDqy5n+9cm9tpjZOs= +code.as/core/socks v1.0.0/go.mod h1:BAXBy5O9s2gmw6UxLqNJcVbWY7C/UPs+801CcSsfWOY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f h1:m1tYqjD/N0vF/S8s/ZKz/eccUr8RAAcrOK2MhXeTegA= @@ -80,6 +82,10 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM= github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/ikeikeikeike/go-sitemap-generator v1.0.1 h1:49Fn8gro/B12vCY8pf5/+/Jpr3kwB9TvP0MSymo69SY= github.com/ikeikeikeike/go-sitemap-generator v1.0.1/go.mod h1:QI+zWsz6yQyxkG9LWNcnu0f7aiAE5tPdsZOsICgmd1c= github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM= @@ -151,10 +157,16 @@ github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6Fk github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2/go.mod h1:w2VxyRO/J5vfNjJHYVubsjUGHd3RLDoVciz0DE3ApOc= +github.com/writeas/go-writeas v1.1.0 h1:WHGm6wriBkxYAOGbvriXH8DlMUGOi6jhSZLUZKQ+4mQ= +github.com/writeas/go-writeas v1.1.0/go.mod h1:oh9U1rWaiE0p3kzdKwwvOpNXgp0P0IELI7OLOwV4fkA= github.com/writeas/httpsig v1.0.0 h1:peIAoIA3DmlP8IG8tMNZqI4YD1uEnWBmkcC9OFPjt3A= github.com/writeas/httpsig v1.0.0/go.mod h1:7ClMGSrSVXJbmiLa17bZ1LrG1oibGZmUMlh3402flPY= github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= +github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvVuhO9UiQ9f+U1aR/x5B4MP7YQHaU= +github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= +github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs= +github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 h1:baEp0631C8sT2r/hqwypIw2snCFZa6h7U6TojoLHu/c= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219/go.mod h1:NyM35ayknT7lzO6O/1JpfgGyv+0W9Z9q7aE0J8bXxfQ= github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo= diff --git a/routes.go b/routes.go index 724c532..022a7ea 100644 --- a/routes.go +++ b/routes.go @@ -11,13 +11,14 @@ package writefreely import ( + "net/http" + "path/filepath" + "strings" + "github.com/gorilla/mux" "github.com/writeas/go-webfinger" "github.com/writeas/web-core/log" "github.com/writefreely/go-nodeinfo" - "net/http" - "path/filepath" - "strings" ) // InitStaticRoutes adds routes for serving static files. @@ -93,6 +94,8 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { me.HandleFunc("/posts/export.json", handler.Download(viewExportPosts, UserLevelUser)).Methods("GET") me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET") me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET") + me.HandleFunc("/import", handler.User(viewImport)).Methods("GET") + me.HandleFunc("/import", handler.User(handleImport)).Methods("POST") me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET") me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET") me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET") diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl new file mode 100644 index 0000000..c258783 --- /dev/null +++ b/templates/user/import.tmpl @@ -0,0 +1,33 @@ +{{define "import"}} +{{template "header" .}} + +
+

Import

+

Upload text or markdown files to import as posts.

+
+
+ + +
+ + +
+ +
+
+ {{if .Flashes}} + + {{end}} +
+ +{{template "footer" .}} +{{end}} \ No newline at end of file diff --git a/templates/user/include/header.tmpl b/templates/user/include/header.tmpl index 312d0b8..e8fd908 100644 --- a/templates/user/include/header.tmpl +++ b/templates/user/include/header.tmpl @@ -27,6 +27,7 @@ {{if .IsAdmin}}
  • Admin
    • {{end}}
  • Settings
  • Export
    • +
  • Import
  • Log out
    • @@ -45,6 +46,7 @@ {{if .IsAdmin}}
  • Admin dashboard
    • {{end}}
  • Account settings
  • Export
    • +
  • Import
    • {{if .CanInvite}}
  • Invite people
    • {{end}}
  • Log out
    • From 0ca198c715a81161d79ba65c23046424a5fbe601 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Sat, 17 Aug 2019 16:18:40 -0700 Subject: [PATCH 02/20] include nice alert message on success different template action for partial or complete import success --- account_import.go | 25 +++++++++++++++++++++++-- templates/user/import.tmpl | 5 +++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/account_import.go b/account_import.go index c8fe8a2..3fd434e 100644 --- a/account_import.go +++ b/account_import.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "github.com/hashicorp/go-multierror" "github.com/writeas/impart" @@ -28,6 +29,8 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error *UserPage Collections *[]Collection Flashes []template.HTML + Message string + InfoMsg bool }{ UserPage: p, Collections: c, @@ -36,7 +39,14 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error flashes, _ := getSessionFlashes(app, w, r, nil) for _, flash := range flashes { - d.Flashes = append(d.Flashes, template.HTML(flash)) + if strings.HasPrefix(flash, "SUCCESS: ") { + d.Message = strings.TrimPrefix(flash, "SUCCESS: ") + } else if strings.HasPrefix(flash, "INFO: ") { + d.Message = strings.TrimPrefix(flash, "INFO: ") + d.InfoMsg = true + } else { + d.Flashes = append(d.Flashes, template.HTML(flash)) + } } showUserPage(w, "import", d) @@ -48,8 +58,9 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err r.ParseMultipartForm(10 << 20) files := r.MultipartForm.File["files"] var fileErrs []error + filesSubmitted := len(files) + var filesImported int for _, formFile := range files { - // TODO: count uploaded files that succeed and report back with message file, err := formFile.Open() if err != nil { fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) @@ -125,10 +136,20 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err continue } } + filesImported++ } if len(fileErrs) != 0 { _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) } + if filesImported == filesSubmitted { + verb := "posts" + if filesSubmitted == 1 { + verb = "post" + } + _ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil) + } else if filesImported > 0 { + _ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil) + } return impart.HTTPError{http.StatusFound, "/me/import"} } diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index c258783..3833095 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -2,6 +2,11 @@ {{template "header" .}}
    + {{if .Message}} +
    +

    {{.Message}}

    +
    + {{end}}

    Import

    Upload text or markdown files to import as posts.

    From 6c5d89ac86d3c51746a0501a2135e92e4ba16631 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Mon, 19 Aug 2019 09:05:52 -0700 Subject: [PATCH 03/20] move import post handler under /api handler for post request to import is now under /api/me/import form target updated also allow all plaintext files in form --- routes.go | 2 +- templates/user/import.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/routes.go b/routes.go index 022a7ea..7dcdc65 100644 --- a/routes.go +++ b/routes.go @@ -95,7 +95,6 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET") me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET") me.HandleFunc("/import", handler.User(viewImport)).Methods("GET") - me.HandleFunc("/import", handler.User(handleImport)).Methods("POST") me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET") me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET") me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET") @@ -108,6 +107,7 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { apiMe.HandleFunc("/password", handler.All(updatePassphrase)).Methods("POST") apiMe.HandleFunc("/self", handler.All(updateSettings)).Methods("POST") apiMe.HandleFunc("/invites", handler.User(handleCreateUserInvite)).Methods("POST") + apiMe.HandleFunc("/import", handler.User(handleImport)).Methods("POST") // Sign up validation write.HandleFunc("/api/alias", handler.All(handleUsernameCheck)).Methods("POST") diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 3833095..002cf14 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -10,9 +10,9 @@

    Import

    Upload text or markdown files to import as posts.

    -
    + - +
    +
    -
    - - -
    + + +
    - {{if .Flashes}} -
      - {{range .Flashes}}
    • {{.}}
      • {{end}} -
    - {{end}}
    {{template "footer" .}} diff --git a/templates/user/include/header.tmpl b/templates/user/include/header.tmpl index 3b57387..0704854 100644 --- a/templates/user/include/header.tmpl +++ b/templates/user/include/header.tmpl @@ -10,8 +10,8 @@
    • {{if .IsAdmin}}
  • Admin
    • {{end}}
  • Settings
    • +
  • Import posts
  • Export
    • -
  • Import
  • Log out
    • @@ -33,8 +33,8 @@
    • {{.Username}}
        {{if .IsAdmin}}
      • Admin dashboard
        • {{end}}
      • Account settings
        • +
      • Import posts
      • Export
        • -
      • Import
        • {{if .CanInvite}}
      • Invite people
        • {{end}}
      • Log out
        • diff --git a/templates/user/invite.tmpl b/templates/user/invite.tmpl index 1985bd5..4365e07 100644 --- a/templates/user/invite.tmpl +++ b/templates/user/invite.tmpl @@ -8,18 +8,7 @@ margin-left: 0.5em; margin-right: 0; } -label { - font-weight: bold; -} -select { - font-size: 1em; - width: 100%; - padding: 0.5rem; - display: block; - border-radius: 0.25rem; - margin: 0.5rem 0; -} -input, table.classy { +table.classy { width: 100%; } table.classy.export a { @@ -34,7 +23,7 @@ table td {

        Invite people

        Invite others to join {{.SiteName}} by generating and sharing invite links below.

        -
        +
        From 6860c0a3ff2352ee4c906c0be615f21495e5b304 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 9 Jan 2020 12:08:06 -0500 Subject: [PATCH 13/20] Fix collection logic on import - Only retrieve a collection from database if an alias is submitted - Only call GetCollection() once (previously, it was inside the loop) - Return error if user doesn't own the collection Ref T609 --- account_import.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/account_import.go b/account_import.go index 87882c6..6a8d807 100644 --- a/account_import.go +++ b/account_import.go @@ -56,6 +56,27 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { // limit 10MB per submission r.ParseMultipartForm(10 << 20) + + collAlias := r.PostFormValue("collection") + coll := &Collection{ + ID: 0, + } + var err error + if collAlias != "" { + coll, err = app.db.GetCollection(collAlias) + if err != nil { + log.Error("Unable to get collection for import: %s", err) + return err + } + // Only allow uploading to collection if current user is owner + if coll.OwnerID != u.ID { + err := ErrUnauthorizedGeneral + _ = addSessionFlash(app, w, r, err.Message, nil) + return err + } + coll.hostName = app.cfg.App.Host + } + files := r.MultipartForm.File["files"] var fileErrs []error filesSubmitted := len(files) @@ -105,14 +126,9 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err continue } - post.Collection = r.PostFormValue("collection") - coll, _ := app.db.GetCollection(post.Collection) - if coll == nil { - coll = &Collection{ - ID: 0, - } + if collAlias != "" { + post.Collection = collAlias } - coll.hostName = app.cfg.App.Host created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ Title: &post.Title, From 03eeca179e00d045bce79b0377b587068f5dd31a Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 9 Jan 2020 12:36:58 -0500 Subject: [PATCH 14/20] Fix potential resource leaks from defer calls in for loop This moves file operations inside the `for` loop into an anonymous func, so the `defer` calls don't wait until the end of the handler call to actually execute. Ref T609 --- account_import.go | 59 +++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/account_import.go b/account_import.go index 6a8d807..d0f0f8f 100644 --- a/account_import.go +++ b/account_import.go @@ -82,36 +82,45 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err filesSubmitted := len(files) var filesImported int for _, formFile := range files { - file, err := formFile.Open() - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) - log.Error("import textfile: open from form: %v", err) - continue - } - defer file.Close() + fname := "" + ok := func() bool { + file, err := formFile.Open() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) + log.Error("import textfile: open from form: %v", err) + return false + } + defer file.Close() - tempFile, err := ioutil.TempFile("", "post-upload-*.txt") - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) - log.Error("import textfile: create temp file: %v", err) - continue - } - defer tempFile.Close() + tempFile, err := ioutil.TempFile("", "post-upload-*.txt") + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) + log.Error("import textfile: create temp file: %v", err) + return false + } + defer tempFile.Close() - _, err = io.Copy(tempFile, file) - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) - log.Error("import textfile: copy to temp: %v", err) + _, err = io.Copy(tempFile, file) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) + log.Error("import textfile: copy to temp: %v", err) + return false + } + + info, err := tempFile.Stat() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) + log.Error("import textfile: stat temp file: %v", err) + return false + } + fname = info.Name() + return true + }() + if !ok { continue } - info, err := tempFile.Stat() - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) - log.Error("import textfile: stat temp file: %v", err) - continue - } - post, err := wfimport.FromFile(filepath.Join(os.TempDir(), info.Name())) + post, err := wfimport.FromFile(filepath.Join(os.TempDir(), fname)) if err == wfimport.ErrEmptyFile { // not a real error so don't log _ = addSessionFlash(app, w, r, fmt.Sprintf("%s was empty, import skipped", formFile.Filename), nil) From 18d3456a23be89229220e6dfc269ba64057ce137 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 9 Jan 2020 13:29:07 -0500 Subject: [PATCH 15/20] Tweak user-facing upload errors + internal logs Ref T609 --- account_import.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/account_import.go b/account_import.go index d0f0f8f..38cfcc3 100644 --- a/account_import.go +++ b/account_import.go @@ -86,31 +86,31 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err ok := func() bool { file, err := formFile.Open() if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) - log.Error("import textfile: open from form: %v", err) + fileErrs = append(fileErrs, fmt.Errorf("Unable to read file %s", formFile.Filename)) + log.Error("import file: open from form: %v", err) return false } defer file.Close() tempFile, err := ioutil.TempFile("", "post-upload-*.txt") if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) - log.Error("import textfile: create temp file: %v", err) + fileErrs = append(fileErrs, fmt.Errorf("Internal error for %s", formFile.Filename)) + log.Error("import file: create temp file %s: %v", formFile.Filename, err) return false } defer tempFile.Close() _, err = io.Copy(tempFile, file) if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) - log.Error("import textfile: copy to temp: %v", err) + fileErrs = append(fileErrs, fmt.Errorf("Internal error for %s", formFile.Filename)) + log.Error("import file: copy to temp location %s: %v", formFile.Filename, err) return false } info, err := tempFile.Stat() if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) - log.Error("import textfile: stat temp file: %v", err) + fileErrs = append(fileErrs, fmt.Errorf("Internal error for %s", formFile.Filename)) + log.Error("import file: stat temp file %s: %v", formFile.Filename, err) return false } fname = info.Name() From f5d21c8c1a2fa90696989b255fc54e9b7ebdffce Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 9 Jan 2020 13:29:30 -0500 Subject: [PATCH 16/20] Reorder federation check logic on upload Ref T609 --- account_import.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/account_import.go b/account_import.go index 38cfcc3..abd38f1 100644 --- a/account_import.go +++ b/account_import.go @@ -152,9 +152,8 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err continue } - // create public post - - if coll.ID != 0 && app.cfg.App.Federation { + // Federate post, if necessary + if app.cfg.App.Federation && coll.ID > 0 { go federatePost( app, &PublicPost{ From aae2f28bb673aa1ce7bda51b29e8174ae11bd53d Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Tue, 14 Jan 2020 08:59:30 -0800 Subject: [PATCH 17/20] pass original file modified date for imports --- account_import.go | 10 ++++++++++ templates/user/import.tmpl | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/account_import.go b/account_import.go index abd38f1..b34f3a7 100644 --- a/account_import.go +++ b/account_import.go @@ -1,6 +1,7 @@ package writefreely import ( + "encoding/json" "fmt" "html/template" "io" @@ -9,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/hashicorp/go-multierror" "github.com/writeas/impart" @@ -77,6 +79,12 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err coll.hostName = app.cfg.App.Host } + fileDates := make(map[string]int64) + err = json.Unmarshal([]byte(r.FormValue("fileDates")), &fileDates) + if err != nil { + log.Error("invalid form data for file dates: %v", err) + return impart.HTTPError{http.StatusBadRequest, "form data for file dates was invalid"} + } files := r.MultipartForm.File["files"] var fileErrs []error filesSubmitted := len(files) @@ -138,6 +146,8 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err if collAlias != "" { post.Collection = collAlias } + dateTime := time.Unix(fileDates[formFile.Filename], 0) + post.Created = &dateTime created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ Title: &post.Title, diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 2b6a4c3..7642023 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -29,8 +29,9 @@
        + +
        - {{template "footer" .}} {{end}} From 2b066997d131e298607bc6864bc17718871994c2 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 14 Jan 2020 12:21:11 -0500 Subject: [PATCH 18/20] Fix unix timestamp in file upload File API gives timestamp in milliseconds, not seconds, so this converts it correctly. Ref T609 --- account_import.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/account_import.go b/account_import.go index b34f3a7..a7b5f25 100644 --- a/account_import.go +++ b/account_import.go @@ -146,7 +146,8 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err if collAlias != "" { post.Collection = collAlias } - dateTime := time.Unix(fileDates[formFile.Filename], 0) + ts := fileDates[formFile.Filename] / 1000 // Get timestamp in seconds, not milliseconds + dateTime := time.Unix(ts, 0) post.Created = &dateTime created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ From 65e2e5126bb11ff32ba5d4cdf21c22e229a7a7ad Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 14 Jan 2020 12:24:57 -0500 Subject: [PATCH 19/20] Revert "Fix unix timestamp in file upload" This reverts commit 2b066997d131e298607bc6864bc17718871994c2. --- account_import.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/account_import.go b/account_import.go index a7b5f25..b34f3a7 100644 --- a/account_import.go +++ b/account_import.go @@ -146,8 +146,7 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err if collAlias != "" { post.Collection = collAlias } - ts := fileDates[formFile.Filename] / 1000 // Get timestamp in seconds, not milliseconds - dateTime := time.Unix(ts, 0) + dateTime := time.Unix(fileDates[formFile.Filename], 0) post.Created = &dateTime created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ From 3e97625cca86484a2836befe418cfc7af3fc70ab Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 14 Jan 2020 12:26:02 -0500 Subject: [PATCH 20/20] Fix Unix timestamps on client during import File API gives timestamp in milliseconds, not seconds, so this converts it on the client-side and sends it the correct time to the server. Ref T609 --- templates/user/import.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 7642023..3400e2f 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -48,7 +48,7 @@ const files = e.target.files; let dateMap = {}; for (let file of files) { - dateMap[file.name] = file.lastModified; + dateMap[file.name] = file.lastModified / 1000; } fileDates.value = JSON.stringify(dateMap); })