SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

[chore] The Big Middleware and API Refactor (tm) (#1250)

Browse Source 
* interim commit: start refactoring middlewares into package under router

* another interim commit, this is becoming a big job

* another fucking massive interim commit

* refactor bookmarks to new style

* ambassador, wiz zeze commits you are spoiling uz

* she compiles, we're getting there

* we're just normal men; we're just innocent men

* apiutil

* whoopsie

* i'm glad noone reads commit msgs haha :blob_sweat:

* use that weirdo go-bytesize library for maxMultipartMemory

* fix media module paths
This commit is contained in:
tobi
2023-01-02 13:10:50 +01:00
committed by GitHub
parent 560ff1209d
commit 941893a774
228 changed files with 3188 additions and 3047 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
internal/api/fileserver/fileserver.go Normal file
View File

@@ -0,0 +1,54 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fileserver
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/processing"
)
const (
// AccountIDKey is the url key for account id (an account ulid)
AccountIDKey = "account_id"
// MediaTypeKey is the url key for media type (usually something like attachment or header etc)
MediaTypeKey = "media_type"
// MediaSizeKey is the url key for the desired media size--original/small/static
MediaSizeKey = "media_size"
// FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg
FileNameKey = "file_name"
// FileServePath is the fileserve path minus the 'fileserver' prefix.
FileServePath = "/:" + AccountIDKey + "/:" + MediaTypeKey + "/:" + MediaSizeKey + "/:" + FileNameKey
)
type Module struct {
processor processing.Processor
}
func New(processor processing.Processor) *Module {
return &Module{
processor: processor,
}
}
func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) {
attachHandler(http.MethodGet, FileServePath, m.ServeFile)
attachHandler(http.MethodHead, FileServePath, m.ServeFile)
}

109
internal/api/fileserver/fileserver_test.go Normal file
View File

@@ -0,0 +1,109 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fileserver_test
import (
"context"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type FileserverTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
processor processing.Processor
mediaManager media.Manager
oauthServer oauth.Server
emailSender email.Sender
// standard suite models
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
testApplications map[string]*gtsmodel.Application
testUsers map[string]*gtsmodel.User
testAccounts map[string]*gtsmodel.Account
testAttachments map[string]*gtsmodel.MediaAttachment
// item being tested
fileServer *fileserver.Module
}
/*
TEST INFRASTRUCTURE
*/
func (suite *FileserverTestSuite) SetupSuite() {
testrig.InitTestConfig()
testrig.InitTestLog()
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewInMemoryStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker)
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.fileServer = fileserver.New(suite.processor)
}
func (suite *FileserverTestSuite) SetupTest() {
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
suite.testTokens = testrig.NewTestTokens()
suite.testClients = testrig.NewTestClients()
suite.testApplications = testrig.NewTestApplications()
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.testAttachments = testrig.NewTestAttachments()
}
func (suite *FileserverTestSuite) TearDownSuite() {
if err := suite.db.Stop(context.Background()); err != nil {
log.Panicf("error closing db connection: %s", err)
}
}
func (suite *FileserverTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}

130
internal/api/fileserver/servefile.go Normal file
View File

@@ -0,0 +1,130 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fileserver
import (
"bytes"
"fmt"
"io"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage.
//
// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found".
// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything.
func (m *Module) ServeFile(c *gin.Context) {
authed, err := oauth.Authed(c, false, false, false, false)
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
return
}
// We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows:
// "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]"
// "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension.
accountID := c.Param(AccountIDKey)
if accountID == "" {
err := fmt.Errorf("missing %s from request", AccountIDKey)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
return
}
mediaType := c.Param(MediaTypeKey)
if mediaType == "" {
err := fmt.Errorf("missing %s from request", MediaTypeKey)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
return
}
mediaSize := c.Param(MediaSizeKey)
if mediaSize == "" {
err := fmt.Errorf("missing %s from request", MediaSizeKey)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
return
}
fileName := c.Param(FileNameKey)
if fileName == "" {
err := fmt.Errorf("missing %s from request", FileNameKey)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
return
}
content, errWithCode := m.processor.FileGet(c.Request.Context(), authed, &apimodel.GetContentRequestForm{
AccountID: accountID,
MediaType: mediaType,
MediaSize: mediaSize,
FileName: fileName,
})
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGet)
return
}
defer func() {
// close content when we're done
if content.Content != nil {
if err := content.Content.Close(); err != nil {
log.Errorf("ServeFile: error closing readcloser: %s", err)
}
}
}()
if content.URL != nil {
c.Redirect(http.StatusFound, content.URL.String())
return
}
// TODO: if the requester only accepts text/html we should try to serve them *something*.
// This is mostly needed because when sharing a link to a gts-hosted file on something like mastodon, the masto servers will
// attempt to look up the content to provide a preview of the link, and they ask for text/html.
format, err := apiutil.NegotiateAccept(c, apiutil.MIME(content.ContentType))
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet)
return
}
// if this is a head request, just return info + throw the reader away
if c.Request.Method == http.MethodHead {
c.Header("Content-Type", format)
c.Header("Content-Length", strconv.FormatInt(content.ContentLength, 10))
c.Status(http.StatusOK)
return
}
// try to slurp the first few bytes to make sure we have something
b := bytes.NewBuffer(make([]byte, 0, 64))
if _, err := io.CopyN(b, content.Content, 64); err != nil {
err = fmt.Errorf("ServeFile: error reading from content: %w", err)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet)
return
}
// we're good, return the slurped bytes + the rest of the content
c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil)
}

272
internal/api/fileserver/servefile_test.go Normal file
View File

@@ -0,0 +1,272 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fileserver_test
import (
"context"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type ServeFileTestSuite struct {
FileserverTestSuite
}
// GetFile is just a convenience function to save repetition in this test suite.
// It takes the required params to serve a file, calls the handler, and returns
// the http status code, the response headers, and the parsed body bytes.
func (suite *ServeFileTestSuite) GetFile(
accountID string,
mediaType media.Type,
mediaSize media.Size,
filename string,
) (code int, headers http.Header, body []byte) {
recorder := httptest.NewRecorder()
ctx, _ := testrig.CreateGinTestContext(recorder, nil)
ctx.Request = httptest.NewRequest(http.MethodGet, "http://localhost:8080/whatever", nil)
ctx.Request.Header.Set("accept", "*/*")
ctx.AddParam(fileserver.AccountIDKey, accountID)
ctx.AddParam(fileserver.MediaTypeKey, string(mediaType))
ctx.AddParam(fileserver.MediaSizeKey, string(mediaSize))
ctx.AddParam(fileserver.FileNameKey, filename)
suite.fileServer.ServeFile(ctx)
code = recorder.Code
headers = recorder.Result().Header
var err error
body, err = ioutil.ReadAll(recorder.Body)
if err != nil {
suite.FailNow(err.Error())
}
return
}
// UncacheAttachment is a convenience function that uncaches the targetAttachment by
// removing its associated files from storage, and updating the database.
func (suite *ServeFileTestSuite) UncacheAttachment(targetAttachment *gtsmodel.MediaAttachment) {
ctx := context.Background()
cached := false
targetAttachment.Cached = &cached
if err := suite.db.UpdateByID(ctx, targetAttachment, targetAttachment.ID, "cached"); err != nil {
suite.FailNow(err.Error())
}
if err := suite.storage.Delete(ctx, targetAttachment.File.Path); err != nil {
suite.FailNow(err.Error())
}
if err := suite.storage.Delete(ctx, targetAttachment.Thumbnail.Path); err != nil {
suite.FailNow(err.Error())
}
}
func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"]
fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path)
if err != nil {
suite.FailNow(err.Error())
}
code, headers, body := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusOK, code)
suite.Equal("image/jpeg", headers.Get("content-type"))
suite.Equal(fileInStorage, body)
}
func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"]
fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path)
if err != nil {
suite.FailNow(err.Error())
}
code, headers, body := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusOK, code)
suite.Equal("image/jpeg", headers.Get("content-type"))
suite.Equal(fileInStorage, body)
}
func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path)
if err != nil {
suite.FailNow(err.Error())
}
code, headers, body := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusOK, code)
suite.Equal("image/jpeg", headers.Get("content-type"))
suite.Equal(fileInStorage, body)
}
func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path)
if err != nil {
suite.FailNow(err.Error())
}
code, headers, body := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusOK, code)
suite.Equal("image/jpeg", headers.Get("content-type"))
suite.Equal(fileInStorage, body)
}
func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path)
if err != nil {
suite.FailNow(err.Error())
}
// uncache the attachment so we'll have to refetch it from the 'remote' instance
suite.UncacheAttachment(targetAttachment)
code, headers, body := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusOK, code)
suite.Equal("image/jpeg", headers.Get("content-type"))
suite.Equal(fileInStorage, body)
}
func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path)
if err != nil {
suite.FailNow(err.Error())
}
// uncache the attachment so we'll have to refetch it from the 'remote' instance
suite.UncacheAttachment(targetAttachment)
code, headers, body := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusOK, code)
suite.Equal("image/jpeg", headers.Get("content-type"))
suite.Equal(fileInStorage, body)
}
func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
// uncache the attachment *and* set the remote URL to something that will return a 404
suite.UncacheAttachment(targetAttachment)
targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee"
if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil {
suite.FailNow(err.Error())
}
code, _, _ := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusNotFound, code)
}
func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() {
targetAttachment := &gtsmodel.MediaAttachment{}
*targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
// uncache the attachment *and* set the remote URL to something that will return a 404
suite.UncacheAttachment(targetAttachment)
targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee"
if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil {
suite.FailNow(err.Error())
}
code, _, _ := suite.GetFile(
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
)
suite.Equal(http.StatusNotFound, code)
}
// Callers trying to get some random-ass file that doesn't exist should just get a 404
func (suite *ServeFileTestSuite) TestServeFileNotFound() {
code, _, _ := suite.GetFile(
"01GMMY4G9B0QEG0PQK5Q5JGJWZ",
media.TypeAttachment,
media.SizeOriginal,
"01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg",
)
suite.Equal(http.StatusNotFound, code)
}
func TestServeFileTestSuite(t *testing.T) {
suite.Run(t, new(ServeFileTestSuite))
}