SuperSeriousBusiness
/
GoToSocial
mirror of https://github.com/superseriousbusiness/gotosocial
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

[feature] Implement `cache-control` and etags for static assets (#711) 

* start working on etag stuff

* add + use cache middleware

* generate etags on the fly

* remove unused field

* clean up filepath

* add license headers to cache files

* add attachgroup function to router interface

* move cache into web module

* rename a couple things

* remove attachStaticFS function from router

* rename + tidy up a few things

* mount assets filesystem

* create assetsFileInfoCache

* update comment

* simplify hash

* fix string fmt

* skip last mod chk, prefer strong etags w/long cache

* move base handler to its own file
this matches the modules in the api folder

* generate new etag if file was modified

* wrap strong etag in quotation marks as per spec

* clarify logic in avatar search

* make hashing a little niftier
This commit is contained in:
tobi 2022-07-18 12:55:06 +02:00 committed by GitHub
parent ab03318b7a
commit c3b6a5b0f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 466 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
internal/cache/account.go vendored
View File

@ -1,3 +1,21 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cache
import (

18
internal/cache/account_test.go vendored
View File

@ -1,3 +1,21 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cache_test
import (

18
internal/cache/status.go vendored
View File

@ -1,3 +1,21 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cache
import (

18
internal/cache/status_test.go vendored
View File

@ -1,3 +1,21 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cache_test
import (

7
internal/router/attach.go
View File

@ -39,3 +39,10 @@ func (r *router) AttachMiddleware(middleware gin.HandlerFunc) {
func (r *router) AttachNoRouteHandler(handler gin.HandlerFunc) {
r.engine.NoRoute(handler)
}
// AttachGroup attaches the given handlers into a group with the given relativePath as
// base path for that group. It then returns the *gin.RouterGroup so that the caller
// can add any extra middlewares etc specific to that group, as desired.
func (r *router) AttachGroup(relativePath string, handlers ...gin.HandlerFunc) *gin.RouterGroup {
return r.engine.Group(relativePath, handlers...)
}

11
internal/router/router.go
View File

@ -47,8 +47,10 @@ type Router interface {
AttachMiddleware(handler gin.HandlerFunc)
// Attach 404 NoRoute handler
AttachNoRouteHandler(handler gin.HandlerFunc)
// Add Gin StaticFS handler
AttachStaticFS(relativePath string, fs http.FileSystem)
// Attach a router group, and receive that group back.
// More middlewares and handlers can then be attached on
// the group by the caller.
AttachGroup(path string, handlers ...gin.HandlerFunc) *gin.RouterGroup
// Start the router
Start()
// Stop the router
@ -62,11 +64,6 @@ type router struct {
certManager *autocert.Manager
}
// Add Gin StaticFS handler
func (r *router) AttachStaticFS(relativePath string, fs http.FileSystem) {
r.engine.StaticFS(relativePath, fs)
}
// Start starts the router nicely. It will serve two handlers if letsencrypt is enabled, and only the web/API handler if letsencrypt is not enabled.
func (r *router) Start() {
// listen is the server start function, by

13
internal/web/fileserver.go → internal/web/assets.go
View File

@ -21,6 +21,8 @@ package web
import (
"net/http"
"strings"
"github.com/gin-gonic/gin"
)
type fileSystem struct {
@ -45,3 +47,14 @@ func (fs fileSystem) Open(path string) (http.File, error) {
return f, nil
}
func (m *Module) mountAssetsFilesystem(group *gin.RouterGroup) {
fs := fileSystem{http.Dir(m.webAssetsAbsFilePath)}
// use the cache middleware on all handlers in this group
group.Use(m.cacheControlMiddleware(fs))
// serve static file system in the root of this group,
// will end up being something like "/assets/"
group.StaticFS("/", fs)
}

138
internal/web/assetscache.go Normal file
View File

@ -0,0 +1,138 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package web
import (
// nolint:gosec
"crypto/sha1"
"encoding/hex"
"fmt"
"io"
"net/http"
"path"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
)
type eTagCacheEntry struct {
eTag string
fileLastModified time.Time
}
// generateEtag generates a strong (byte-for-byte) etag using
// the entirety of the provided reader.
func generateEtag(r io.Reader) (string, error) {
// nolint:gosec
hash := sha1.New()
if _, err := io.Copy(hash, r); err != nil {
return "", err
}
b := make([]byte, 0, sha1.Size)
b = hash.Sum(b)
return `"` + hex.EncodeToString(b) + `"`, nil
}
// getAssetFileInfo tries to fetch the ETag for the given filePath from the module's
// assetsETagCache. If it can't be found there, it uses the provided http.FileSystem
// to generate a new ETag to go in the cache, which it then returns.
func (m *Module) getAssetETag(filePath string, fs http.FileSystem) (string, error) {
file, err := fs.Open(filePath)
if err != nil {
return "", fmt.Errorf("error opening %s: %s", filePath, err)
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return "", fmt.Errorf("error statting %s: %s", filePath, err)
}
fileLastModified := fileInfo.ModTime()
if cachedETag, ok := m.assetsETagCache.Get(filePath); ok && !fileLastModified.After(cachedETag.fileLastModified) {
// only return our cached etag if the file wasn't
// modified since last time, otherwise generate a
// new one; eat fresh!
return cachedETag.eTag, nil
}
eTag, err := generateEtag(file)
if err != nil {
return "", fmt.Errorf("error generating etag: %s", err)
}
// put new entry in cache before we return
m.assetsETagCache.Set(filePath, eTagCacheEntry{
eTag: eTag,
fileLastModified: fileLastModified,
})
return eTag, nil
}
// cacheControlMiddleware implements Cache-Control header setting, and checks for
// files inside the given http.FileSystem.
//
// The middleware checks if the file has been modified using If-None-Match etag,
// if present. If the file hasn't been modified, the middleware returns 304.
//
// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match
// and: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
func (m *Module) cacheControlMiddleware(fs http.FileSystem) gin.HandlerFunc {
return func(c *gin.Context) {
// no-cache prevents clients using default caching or heuristic caching,
// and also ensures that clients will validate their cached version against
// the version stored on the server to keep up to date.
c.Header("Cache-Control", "no-cache")
ifNoneMatch := c.Request.Header.Get("If-None-Match")
// derive the path of the requested asset inside the provided filesystem
upath := c.Request.URL.Path
if !strings.HasPrefix(upath, "/") {
upath = "/" + upath
}
assetFilePath := strings.TrimPrefix(path.Clean(upath), assetsPath)
// either fetch etag from ttlcache or generate it
eTag, err := m.getAssetETag(assetFilePath, fs)
if err != nil {
logrus.Errorf("error getting ETag for %s: %s", assetFilePath, err)
return
}
// Regardless of what happens further down, set the etag header
// so that the client has the up-to-date version.
c.Header("Etag", eTag)
// If client already has latest version of the asset, 304 + bail.
if ifNoneMatch == eTag {
c.AbortWithStatus(http.StatusNotModified)
return
}
// else let the rest of the request be processed normally
}
}

157
internal/web/base.go
View File

@ -19,89 +19,14 @@
package web
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"path/filepath"
"strings"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
const (
confirmEmailPath = "/" + uris.ConfirmEmailPath
tokenParam = "token"
usernameKey = "username"
statusIDKey = "status"
profilePath = "/@:" + usernameKey
statusPath = profilePath + "/statuses/:" + statusIDKey
)
// Module implements the api.ClientModule interface for web pages.
type Module struct {
processor processing.Processor
assetsPath string
adminPath string
defaultAvatars []string
}
// New returns a new api.ClientModule for web pages.
func New(processor processing.Processor) (api.ClientModule, error) {
assetsBaseDir := config.GetWebAssetBaseDir()
if assetsBaseDir == "" {
return nil, fmt.Errorf("%s cannot be empty and must be a relative or absolute path", config.WebAssetBaseDirFlag())
}
assetsPath, err := filepath.Abs(assetsBaseDir)
if err != nil {
return nil, fmt.Errorf("error getting absolute path of %s: %s", assetsBaseDir, err)
}
defaultAvatarsPath := filepath.Join(assetsPath, "default_avatars")
defaultAvatarFiles, err := ioutil.ReadDir(defaultAvatarsPath)
if err != nil {
return nil, fmt.Errorf("error reading default avatars at %s: %s", defaultAvatarsPath, err)
}
defaultAvatars := []string{}
for _, f := range defaultAvatarFiles {
// ignore directories
if f.IsDir() {
continue
}
// ignore files bigger than 50kb
if f.Size() > 50000 {
continue
}
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(f.Name())), ".")
// take only files with simple extensions
switch extension {
case "svg", "jpeg", "jpg", "gif", "png":
defaultAvatarPath := fmt.Sprintf("/assets/default_avatars/%s", f.Name())
defaultAvatars = append(defaultAvatars, defaultAvatarPath)
default:
continue
}
}
return &Module{
processor: processor,
assetsPath: assetsPath,
adminPath: filepath.Join(assetsPath, "admin"),
defaultAvatars: defaultAvatars,
}, nil
}
func (m *Module) baseHandler(c *gin.Context) {
host := config.GetHost()
instance, err := m.processor.InstanceGet(c.Request.Context(), host)
@ -114,85 +39,3 @@ func (m *Module) baseHandler(c *gin.Context) {
"instance": instance,
})
}
// TODO: abstract the {admin, user}panel handlers in some way
func (m *Module) AdminPanelHandler(c *gin.Context) {
host := config.GetHost()
instance, err := m.processor.InstanceGet(c.Request.Context(), host)
if err != nil {
api.ErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGet)
return
}
c.HTML(http.StatusOK, "frontend.tmpl", gin.H{
"instance": instance,
"stylesheets": []string{
"/assets/Fork-Awesome/css/fork-awesome.min.css",
"/assets/dist/panels-admin-style.css",
},
"javascript": []string{
"/assets/dist/bundle.js",
"/assets/dist/admin-panel.js",
},
})
}
func (m *Module) UserPanelHandler(c *gin.Context) {
host := config.GetHost()
instance, err := m.processor.InstanceGet(c.Request.Context(), host)
if err != nil {
api.ErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGet)
return
}
c.HTML(http.StatusOK, "frontend.tmpl", gin.H{
"instance": instance,
"stylesheets": []string{
"/assets/Fork-Awesome/css/fork-awesome.min.css",
"/assets/dist/_colors.css",
"/assets/dist/base.css",
"/assets/dist/panels-user-style.css",
},
"javascript": []string{
"/assets/dist/bundle.js",
"/assets/dist/user-panel.js",
},
})
}
// Route satisfies the RESTAPIModule interface
func (m *Module) Route(s router.Router) error {
// serve static files from assets dir at /assets
s.AttachStaticFS("/assets", fileSystem{http.Dir(m.assetsPath)})
s.AttachHandler(http.MethodGet, "/admin", m.AdminPanelHandler)
// redirect /admin/ to /admin
s.AttachHandler(http.MethodGet, "/admin/", func(c *gin.Context) {
c.Redirect(http.StatusMovedPermanently, "/admin")
})
s.AttachHandler(http.MethodGet, "/user", m.UserPanelHandler)
// redirect /settings/ to /settings
s.AttachHandler(http.MethodGet, "/user/", func(c *gin.Context) {
c.Redirect(http.StatusMovedPermanently, "/user")
})
// serve front-page
s.AttachHandler(http.MethodGet, "/", m.baseHandler)
// serve profile pages at /@username
s.AttachHandler(http.MethodGet, profilePath, m.profileGETHandler)
// serve statuses
s.AttachHandler(http.MethodGet, statusPath, m.threadGETHandler)
// serve email confirmation page at /confirm_email?token=whatever
s.AttachHandler(http.MethodGet, confirmEmailPath, m.confirmEmailGETHandler)
// 404 handler
s.AttachNoRouteHandler(func(c *gin.Context) {
api.ErrorHandler(c, gtserror.NewErrorNotFound(errors.New(http.StatusText(http.StatusNotFound))), m.processor.InstanceGet)
})
return nil
}

73
internal/web/panels.go Normal file
View File

@ -0,0 +1,73 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package web
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
func (m *Module) UserPanelHandler(c *gin.Context) {
host := config.GetHost()
instance, err := m.processor.InstanceGet(c.Request.Context(), host)
if err != nil {
api.ErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGet)
return
}
c.HTML(http.StatusOK, "frontend.tmpl", gin.H{
"instance": instance,
"stylesheets": []string{
assetsPath + "/Fork-Awesome/css/fork-awesome.min.css",
assetsPath + "/dist/_colors.css",
assetsPath + "/dist/base.css",
assetsPath + "/dist/panels-user-style.css",
},
"javascript": []string{
assetsPath + "/dist/bundle.js",
assetsPath + "/dist/user-panel.js",
},
})
}
// TODO: abstract the {admin, user}panel handlers in some way
func (m *Module) AdminPanelHandler(c *gin.Context) {
host := config.GetHost()
instance, err := m.processor.InstanceGet(c.Request.Context(), host)
if err != nil {
api.ErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGet)
return
}
c.HTML(http.StatusOK, "frontend.tmpl", gin.H{
"instance": instance,
"stylesheets": []string{
assetsPath + "/Fork-Awesome/css/fork-awesome.min.css",
assetsPath + "/dist/panels-admin-style.css",
},
"javascript": []string{
assetsPath + "/dist/bundle.js",
assetsPath + "/dist/admin-panel.js",
},
})
}

159
internal/web/web.go Normal file
View File

@ -0,0 +1,159 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package web
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"path/filepath"
"strings"
"time"
"codeberg.org/gruf/go-cache/v2"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
const (
confirmEmailPath = "/" + uris.ConfirmEmailPath
profilePath = "/@:" + usernameKey
statusPath = profilePath + "/statuses/:" + statusIDKey
adminPanelPath = "/admin"
userPanelpath = "/user"
assetsPath = "/assets"
tokenParam = "token"
usernameKey = "username"
statusIDKey = "status"
)
// Module implements the api.ClientModule interface for web pages.
type Module struct {
processor processing.Processor
webAssetsAbsFilePath string
assetsETagCache cache.Cache[string, eTagCacheEntry]
defaultAvatars []string
}
// New returns a new api.ClientModule for web pages.
func New(processor processing.Processor) (api.ClientModule, error) {
webAssetsBaseDir := config.GetWebAssetBaseDir()
if webAssetsBaseDir == "" {
return nil, fmt.Errorf("%s cannot be empty and must be a relative or absolute path", config.WebAssetBaseDirFlag())
}
webAssetsAbsFilePath, err := filepath.Abs(webAssetsBaseDir)
if err != nil {
return nil, fmt.Errorf("error getting absolute path of %s: %s", webAssetsBaseDir, err)
}
defaultAvatarsAbsFilePath := filepath.Join(webAssetsAbsFilePath, "default_avatars")
defaultAvatarFiles, err := ioutil.ReadDir(defaultAvatarsAbsFilePath)
if err != nil {
return nil, fmt.Errorf("error reading default avatars at %s: %s", defaultAvatarsAbsFilePath, err)
}
defaultAvatars := []string{}
for _, f := range defaultAvatarFiles {
// ignore directories
if f.IsDir() {
continue
}
// ignore files bigger than 50kb
if f.Size() > 50000 {
continue
}
// get the name of the file, eg avatar.jpeg
fileName := f.Name()
// get just the .jpeg, for example, from avatar.jpeg
extensionWithDot := filepath.Ext(fileName)
// remove the leading . to just get, eg, jpeg
extension := strings.TrimPrefix(extensionWithDot, ".")
// take only files with simple extensions
// that we know will work OK as avatars
switch strings.ToLower(extension) {
case "svg", "jpeg", "jpg", "gif", "png":
avatar := fmt.Sprintf("%s/default_avatars/%s", assetsPath, f.Name())
defaultAvatars = append(defaultAvatars, avatar)
default:
continue
}
}
assetsETagCache := cache.New[string, eTagCacheEntry]()
assetsETagCache.SetTTL(time.Hour, false)
assetsETagCache.Start(time.Minute)
return &Module{
processor: processor,
webAssetsAbsFilePath: webAssetsAbsFilePath,
assetsETagCache: assetsETagCache,
defaultAvatars: defaultAvatars,
}, nil
}
// Route satisfies the RESTAPIModule interface
func (m *Module) Route(s router.Router) error {
// serve static files from assets dir at /assets
assetsGroup := s.AttachGroup(assetsPath)
m.mountAssetsFilesystem(assetsGroup)
s.AttachHandler(http.MethodGet, adminPanelPath, m.AdminPanelHandler)
// redirect /admin/ to /admin
s.AttachHandler(http.MethodGet, adminPanelPath+"/", func(c *gin.Context) {
c.Redirect(http.StatusMovedPermanently, adminPanelPath)
})
s.AttachHandler(http.MethodGet, userPanelpath, m.UserPanelHandler)
// redirect /settings/ to /settings
s.AttachHandler(http.MethodGet, userPanelpath+"/", func(c *gin.Context) {
c.Redirect(http.StatusMovedPermanently, userPanelpath)
})
// serve front-page
s.AttachHandler(http.MethodGet, "/", m.baseHandler)
// serve profile pages at /@username
s.AttachHandler(http.MethodGet, profilePath, m.profileGETHandler)
// serve statuses
s.AttachHandler(http.MethodGet, statusPath, m.threadGETHandler)
// serve email confirmation page at /confirm_email?token=whatever
s.AttachHandler(http.MethodGet, confirmEmailPath, m.confirmEmailGETHandler)
// 404 handler
s.AttachNoRouteHandler(func(c *gin.Context) {
api.ErrorHandler(c, gtserror.NewErrorNotFound(errors.New(http.StatusText(http.StatusNotFound))), m.processor.InstanceGet)
})
return nil
}