mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2025-06-05 21:59:39 +02:00
[feature] Clean up/uncache remote media (#407)
* Add whereNotEmptyAndNotNull * Add GetRemoteOlderThanDays * Add GetRemoteOlderThanDays * Add PruneRemote to Manager interface * Start implementing PruneRemote * add new attachment + status to tests * fix up and test GetRemoteOlderThan * fix bad import * PruneRemote: return number pruned * add Cached column to mediaattachment * update + test pruneRemote * update mediaTest * use Cached column * upstep bun to latest version * embed structs in mediaAttachment * migrate mediaAttachment to new format * don't default cached to true * select only remote media * update db dependencies * step bun back to last working version * update pruneRemote to use Cached field * fix storage path of test attachments * add recache logic to manager * fix trimmed aspect ratio * test prune and recache * return errwithcode * tidy up different paths for emoji vs attachment * fix incorrect thumbnail type being stored * expose TransportController to media processor * implement tee-ing recached content * add thoughts of dog to test fedi attachments * test get remote files * add comment on PruneRemote * add postData cleanup to recache * test thumbnail fetching * add incredible diagram * go mod tidy * buffer pipes for recache streaming * test for client stops reading after 1kb * add media-remote-cache-days to config * add cron package * wrap logrus so it's available to cron * start and stop cron jobs gracefully
This commit is contained in:
@@ -38,6 +38,6 @@ func (p *processor) MediaUpdate(ctx context.Context, authed *oauth.Auth, mediaAt
|
||||
return p.mediaProcessor.Update(ctx, authed.Account, mediaAttachmentID, form)
|
||||
}
|
||||
|
||||
func (p *processor) FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
|
||||
func (p *processor) FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
|
||||
return p.mediaProcessor.GetFile(ctx, authed.Account, form)
|
||||
}
|
||||
|
@@ -19,8 +19,11 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
|
||||
@@ -29,7 +32,7 @@ import (
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
)
|
||||
|
||||
func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
|
||||
func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
|
||||
// parse the form fields
|
||||
mediaSize, err := media.ParseMediaSize(form.MediaSize)
|
||||
if err != nil {
|
||||
@@ -46,74 +49,208 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("file name %s not parseable", form.FileName))
|
||||
}
|
||||
wantedMediaID := spl[0]
|
||||
expectedAccountID := form.AccountID
|
||||
|
||||
// get the account that owns the media and make sure it's not suspended
|
||||
acct, err := p.db.GetAccountByID(ctx, form.AccountID)
|
||||
acct, err := p.db.GetAccountByID(ctx, expectedAccountID)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", form.AccountID, err))
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", expectedAccountID, err))
|
||||
}
|
||||
if !acct.SuspendedAt.IsZero() {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", form.AccountID))
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", expectedAccountID))
|
||||
}
|
||||
|
||||
// make sure the requesting account and the media account don't block each other
|
||||
if account != nil {
|
||||
blocked, err := p.db.IsBlocked(ctx, account.ID, form.AccountID, true)
|
||||
blocked, err := p.db.IsBlocked(ctx, account.ID, expectedAccountID, true)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", form.AccountID, account.ID, err))
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", expectedAccountID, account.ID, err))
|
||||
}
|
||||
if blocked {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", form.AccountID, account.ID))
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", expectedAccountID, account.ID))
|
||||
}
|
||||
}
|
||||
|
||||
// the way we store emojis is a little different from the way we store other attachments,
|
||||
// so we need to take different steps depending on the media type being requested
|
||||
content := &apimodel.Content{}
|
||||
var storagePath string
|
||||
switch mediaType {
|
||||
case media.TypeEmoji:
|
||||
e := >smodel.Emoji{}
|
||||
if err := p.db.GetByID(ctx, wantedMediaID, e); err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedMediaID, err))
|
||||
}
|
||||
if e.Disabled {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedMediaID))
|
||||
}
|
||||
switch mediaSize {
|
||||
case media.SizeOriginal:
|
||||
content.ContentType = e.ImageContentType
|
||||
content.ContentLength = int64(e.ImageFileSize)
|
||||
storagePath = e.ImagePath
|
||||
case media.SizeStatic:
|
||||
content.ContentType = e.ImageStaticContentType
|
||||
content.ContentLength = int64(e.ImageStaticFileSize)
|
||||
storagePath = e.ImageStaticPath
|
||||
default:
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", mediaSize))
|
||||
}
|
||||
return p.getEmojiContent(ctx, wantedMediaID, mediaSize)
|
||||
case media.TypeAttachment, media.TypeHeader, media.TypeAvatar:
|
||||
a, err := p.db.GetAttachmentByID(ctx, wantedMediaID)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
|
||||
return p.getAttachmentContent(ctx, account, wantedMediaID, expectedAccountID, mediaSize)
|
||||
default:
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media type %s not recognized", mediaType))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, expectedAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) {
|
||||
attachmentContent := &apimodel.Content{}
|
||||
var storagePath string
|
||||
|
||||
// retrieve attachment from the database and do basic checks on it
|
||||
a, err := p.db.GetAttachmentByID(ctx, wantedMediaID)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
|
||||
}
|
||||
|
||||
if a.AccountID != expectedAccountID {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, expectedAccountID))
|
||||
}
|
||||
|
||||
// get file information from the attachment depending on the requested media size
|
||||
switch mediaSize {
|
||||
case media.SizeOriginal:
|
||||
attachmentContent.ContentType = a.File.ContentType
|
||||
attachmentContent.ContentLength = int64(a.File.FileSize)
|
||||
storagePath = a.File.Path
|
||||
case media.SizeSmall:
|
||||
attachmentContent.ContentType = a.Thumbnail.ContentType
|
||||
attachmentContent.ContentLength = int64(a.Thumbnail.FileSize)
|
||||
storagePath = a.Thumbnail.Path
|
||||
default:
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
|
||||
}
|
||||
|
||||
// if we have the media cached on our server already, we can now simply return it from storage
|
||||
if a.Cached {
|
||||
return p.streamFromStorage(storagePath, attachmentContent)
|
||||
}
|
||||
|
||||
// if we don't have it cached, then we can assume two things:
|
||||
// 1. this is remote media, since local media should never be uncached
|
||||
// 2. we need to fetch it again using a transport and the media manager
|
||||
remoteMediaIRI, err := url.Parse(a.RemoteURL)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err))
|
||||
}
|
||||
|
||||
// use an empty string as requestingUsername to use the instance account, unless the request for this
|
||||
// media has been http signed, then use the requesting account to make the request to remote server
|
||||
var requestingUsername string
|
||||
if requestingAccount != nil {
|
||||
requestingUsername = requestingAccount.Username
|
||||
}
|
||||
|
||||
var data media.DataFunc
|
||||
var postDataCallback media.PostDataCallbackFunc
|
||||
|
||||
if mediaSize == media.SizeSmall {
|
||||
// if it's the thumbnail that's requested then the user will have to wait a bit while we process the
|
||||
// large version and derive a thumbnail from it, so use the normal recaching procedure: fetch the media,
|
||||
// process it, then return the thumbnail data
|
||||
data = func(innerCtx context.Context) (io.Reader, int, error) {
|
||||
transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return transport.DereferenceMedia(innerCtx, remoteMediaIRI)
|
||||
}
|
||||
if a.AccountID != form.AccountID {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, form.AccountID))
|
||||
} else {
|
||||
// if it's the full-sized version being requested, we can cheat a bit by streaming data to the user as
|
||||
// it's retrieved from the remote server, using tee; this saves the user from having to wait while
|
||||
// we process the media on our side
|
||||
//
|
||||
// this looks a bit like this:
|
||||
//
|
||||
// http fetch buffered pipe
|
||||
// remote server ------------> data function ----------------> api caller
|
||||
// |
|
||||
// | tee
|
||||
// |
|
||||
// ▼
|
||||
// instance storage
|
||||
|
||||
// Buffer each end of the pipe, so that if the caller drops the connection during the flow, the tee
|
||||
// reader can continue without having to worry about tee-ing into a closed or blocked pipe.
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
bufferedWriter := bufio.NewWriterSize(pipeWriter, int(attachmentContent.ContentLength))
|
||||
bufferedReader := bufio.NewReaderSize(pipeReader, int(attachmentContent.ContentLength))
|
||||
|
||||
// the caller will read from the buffered reader, so it doesn't matter if they drop out without reading everything
|
||||
attachmentContent.Content = bufferedReader
|
||||
|
||||
data = func(innerCtx context.Context) (io.Reader, int, error) {
|
||||
transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
readCloser, fileSize, err := transport.DereferenceMedia(innerCtx, remoteMediaIRI)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
// everything read from the readCloser by the media manager will be written into the bufferedWriter
|
||||
teeReader := io.TeeReader(readCloser, bufferedWriter)
|
||||
return teeReader, fileSize, nil
|
||||
}
|
||||
switch mediaSize {
|
||||
case media.SizeOriginal:
|
||||
content.ContentType = a.File.ContentType
|
||||
content.ContentLength = int64(a.File.FileSize)
|
||||
storagePath = a.File.Path
|
||||
case media.SizeSmall:
|
||||
content.ContentType = a.Thumbnail.ContentType
|
||||
content.ContentLength = int64(a.Thumbnail.FileSize)
|
||||
storagePath = a.Thumbnail.Path
|
||||
default:
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
|
||||
|
||||
// close the pipewriter after data has been piped into it, so the reader on the other side doesn't block;
|
||||
// we don't need to close the reader here because that's the caller's responsibility
|
||||
postDataCallback = func(innerCtx context.Context) error {
|
||||
// flush the buffered writer into the buffer of the reader...
|
||||
if err := bufferedWriter.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// and close the underlying pipe writer
|
||||
if err := pipeWriter.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// put the media recached in the queue
|
||||
processingMedia, err := p.mediaManager.RecacheMedia(ctx, data, postDataCallback, wantedMediaID)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
|
||||
}
|
||||
|
||||
// if it's the thumbnail, stream the processed thumbnail from storage, after waiting for processing to finish
|
||||
if mediaSize == media.SizeSmall {
|
||||
// below function call blocks until all processing on the attachment has finished...
|
||||
if _, err := processingMedia.LoadAttachment(ctx); err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
|
||||
}
|
||||
// ... so now we can safely return it
|
||||
return p.streamFromStorage(storagePath, attachmentContent)
|
||||
}
|
||||
|
||||
return attachmentContent, nil
|
||||
}
|
||||
|
||||
func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) {
|
||||
emojiContent := &apimodel.Content{}
|
||||
var storagePath string
|
||||
|
||||
e := >smodel.Emoji{}
|
||||
if err := p.db.GetByID(ctx, wantedEmojiID, e); err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err))
|
||||
}
|
||||
|
||||
if e.Disabled {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedEmojiID))
|
||||
}
|
||||
|
||||
switch emojiSize {
|
||||
case media.SizeOriginal:
|
||||
emojiContent.ContentType = e.ImageContentType
|
||||
emojiContent.ContentLength = int64(e.ImageFileSize)
|
||||
storagePath = e.ImagePath
|
||||
case media.SizeStatic:
|
||||
emojiContent.ContentType = e.ImageStaticContentType
|
||||
emojiContent.ContentLength = int64(e.ImageStaticFileSize)
|
||||
storagePath = e.ImageStaticPath
|
||||
default:
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize))
|
||||
}
|
||||
|
||||
return p.streamFromStorage(storagePath, emojiContent)
|
||||
}
|
||||
|
||||
func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
|
||||
reader, err := p.storage.GetStream(storagePath)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))
|
||||
|
208
internal/processing/media/getfile_test.go
Normal file
208
internal/processing/media/getfile_test.go
Normal file
@@ -0,0 +1,208 @@
|
||||
/*
|
||||
GoToSocial
|
||||
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package media_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
)
|
||||
|
||||
type GetFileTestSuite struct {
|
||||
MediaStandardTestSuite
|
||||
}
|
||||
|
||||
func (suite *GetFileTestSuite) TestGetRemoteFileCached() {
|
||||
ctx := context.Background()
|
||||
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
fileName := path.Base(testAttachment.File.Path)
|
||||
requestingAccount := suite.testAccounts["local_account_1"]
|
||||
|
||||
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
|
||||
AccountID: testAttachment.AccountID,
|
||||
MediaType: string(media.TypeAttachment),
|
||||
MediaSize: string(media.SizeOriginal),
|
||||
FileName: fileName,
|
||||
})
|
||||
|
||||
suite.NoError(errWithCode)
|
||||
suite.NotNil(content)
|
||||
b, err := io.ReadAll(content.Content)
|
||||
suite.NoError(err)
|
||||
|
||||
if closer, ok := content.Content.(io.Closer); ok {
|
||||
suite.NoError(closer.Close())
|
||||
}
|
||||
|
||||
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, b)
|
||||
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].ContentType, content.ContentType)
|
||||
suite.EqualValues(len(suite.testRemoteAttachments[testAttachment.RemoteURL].Data), content.ContentLength)
|
||||
}
|
||||
|
||||
func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
|
||||
ctx := context.Background()
|
||||
|
||||
// uncache the file from local
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
testAttachment.Cached = false
|
||||
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
|
||||
suite.NoError(err)
|
||||
err = suite.storage.Delete(testAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
|
||||
suite.NoError(err)
|
||||
|
||||
// now fetch it
|
||||
fileName := path.Base(testAttachment.File.Path)
|
||||
requestingAccount := suite.testAccounts["local_account_1"]
|
||||
|
||||
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
|
||||
AccountID: testAttachment.AccountID,
|
||||
MediaType: string(media.TypeAttachment),
|
||||
MediaSize: string(media.SizeOriginal),
|
||||
FileName: fileName,
|
||||
})
|
||||
|
||||
suite.NoError(errWithCode)
|
||||
suite.NotNil(content)
|
||||
b, err := io.ReadAll(content.Content)
|
||||
suite.NoError(err)
|
||||
|
||||
if closer, ok := content.Content.(io.Closer); ok {
|
||||
suite.NoError(closer.Close())
|
||||
}
|
||||
|
||||
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, b)
|
||||
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].ContentType, content.ContentType)
|
||||
suite.EqualValues(len(suite.testRemoteAttachments[testAttachment.RemoteURL].Data), content.ContentLength)
|
||||
time.Sleep(2 * time.Second) // wait a few seconds for the media manager to finish doing stuff
|
||||
|
||||
// the attachment should be updated in the database
|
||||
dbAttachment, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
|
||||
suite.NoError(err)
|
||||
suite.True(dbAttachment.Cached)
|
||||
|
||||
// the file should be back in storage at the same path as before
|
||||
refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
|
||||
}
|
||||
|
||||
func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
|
||||
ctx := context.Background()
|
||||
|
||||
// uncache the file from local
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
testAttachment.Cached = false
|
||||
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
|
||||
suite.NoError(err)
|
||||
err = suite.storage.Delete(testAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
|
||||
suite.NoError(err)
|
||||
|
||||
// now fetch it
|
||||
fileName := path.Base(testAttachment.File.Path)
|
||||
requestingAccount := suite.testAccounts["local_account_1"]
|
||||
|
||||
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
|
||||
AccountID: testAttachment.AccountID,
|
||||
MediaType: string(media.TypeAttachment),
|
||||
MediaSize: string(media.SizeOriginal),
|
||||
FileName: fileName,
|
||||
})
|
||||
|
||||
suite.NoError(errWithCode)
|
||||
suite.NotNil(content)
|
||||
|
||||
// only read the first kilobyte and then stop
|
||||
b := make([]byte, 1024)
|
||||
_, err = content.Content.Read(b)
|
||||
suite.NoError(err)
|
||||
|
||||
// close the reader
|
||||
if closer, ok := content.Content.(io.Closer); ok {
|
||||
suite.NoError(closer.Close())
|
||||
}
|
||||
|
||||
time.Sleep(2 * time.Second) // wait a few seconds for the media manager to finish doing stuff
|
||||
|
||||
// the attachment should still be updated in the database even though the caller hung up
|
||||
dbAttachment, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
|
||||
suite.NoError(err)
|
||||
suite.True(dbAttachment.Cached)
|
||||
|
||||
// the file should be back in storage at the same path as before
|
||||
refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
|
||||
}
|
||||
|
||||
func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() {
|
||||
ctx := context.Background()
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
|
||||
// fetch the existing thumbnail bytes from storage first
|
||||
thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path)
|
||||
suite.NoError(err)
|
||||
|
||||
// uncache the file from local
|
||||
testAttachment.Cached = false
|
||||
err = suite.db.UpdateByPrimaryKey(ctx, testAttachment)
|
||||
suite.NoError(err)
|
||||
err = suite.storage.Delete(testAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
|
||||
suite.NoError(err)
|
||||
|
||||
// now fetch the thumbnail
|
||||
fileName := path.Base(testAttachment.File.Path)
|
||||
requestingAccount := suite.testAccounts["local_account_1"]
|
||||
|
||||
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
|
||||
AccountID: testAttachment.AccountID,
|
||||
MediaType: string(media.TypeAttachment),
|
||||
MediaSize: string(media.SizeSmall),
|
||||
FileName: fileName,
|
||||
})
|
||||
|
||||
suite.NoError(errWithCode)
|
||||
suite.NotNil(content)
|
||||
b, err := io.ReadAll(content.Content)
|
||||
suite.NoError(err)
|
||||
|
||||
if closer, ok := content.Content.(io.Closer); ok {
|
||||
suite.NoError(closer.Close())
|
||||
}
|
||||
|
||||
suite.Equal(thumbnailBytes, b)
|
||||
suite.Equal("image/jpeg", content.ContentType)
|
||||
suite.EqualValues(testAttachment.Thumbnail.FileSize, content.ContentLength)
|
||||
}
|
||||
|
||||
func TestGetFileTestSuite(t *testing.T) {
|
||||
suite.Run(t, &GetFileTestSuite{})
|
||||
}
|
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/transport"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
|
||||
)
|
||||
|
||||
@@ -36,24 +37,27 @@ type Processor interface {
|
||||
Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error)
|
||||
// Delete deletes the media attachment with the given ID, including all files pertaining to that attachment.
|
||||
Delete(ctx context.Context, mediaAttachmentID string) gtserror.WithCode
|
||||
GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
|
||||
// GetFile retrieves a file from storage and streams it back to the caller via an io.reader embedded in *apimodel.Content.
|
||||
GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode)
|
||||
GetMedia(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string) (*apimodel.Attachment, gtserror.WithCode)
|
||||
Update(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string, form *apimodel.AttachmentUpdateRequest) (*apimodel.Attachment, gtserror.WithCode)
|
||||
}
|
||||
|
||||
type processor struct {
|
||||
tc typeutils.TypeConverter
|
||||
mediaManager media.Manager
|
||||
storage *kv.KVStore
|
||||
db db.DB
|
||||
tc typeutils.TypeConverter
|
||||
mediaManager media.Manager
|
||||
transportController transport.Controller
|
||||
storage *kv.KVStore
|
||||
db db.DB
|
||||
}
|
||||
|
||||
// New returns a new media processor.
|
||||
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *kv.KVStore) Processor {
|
||||
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor {
|
||||
return &processor{
|
||||
tc: tc,
|
||||
mediaManager: mediaManager,
|
||||
storage: storage,
|
||||
db: db,
|
||||
tc: tc,
|
||||
mediaManager: mediaManager,
|
||||
transportController: transportController,
|
||||
storage: storage,
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
125
internal/processing/media/media_test.go
Normal file
125
internal/processing/media/media_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
GoToSocial
|
||||
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package media_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"codeberg.org/gruf/go-store/kv"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/transport"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
|
||||
"github.com/superseriousbusiness/gotosocial/testrig"
|
||||
)
|
||||
|
||||
type MediaStandardTestSuite struct {
|
||||
// standard suite interfaces
|
||||
suite.Suite
|
||||
db db.DB
|
||||
tc typeutils.TypeConverter
|
||||
storage *kv.KVStore
|
||||
mediaManager media.Manager
|
||||
transportController transport.Controller
|
||||
|
||||
// standard suite models
|
||||
testTokens map[string]*gtsmodel.Token
|
||||
testClients map[string]*gtsmodel.Client
|
||||
testApplications map[string]*gtsmodel.Application
|
||||
testUsers map[string]*gtsmodel.User
|
||||
testAccounts map[string]*gtsmodel.Account
|
||||
testAttachments map[string]*gtsmodel.MediaAttachment
|
||||
testStatuses map[string]*gtsmodel.Status
|
||||
testRemoteAttachments map[string]testrig.RemoteAttachmentFile
|
||||
|
||||
// module being tested
|
||||
mediaProcessor mediaprocessing.Processor
|
||||
}
|
||||
|
||||
func (suite *MediaStandardTestSuite) SetupSuite() {
|
||||
suite.testTokens = testrig.NewTestTokens()
|
||||
suite.testClients = testrig.NewTestClients()
|
||||
suite.testApplications = testrig.NewTestApplications()
|
||||
suite.testUsers = testrig.NewTestUsers()
|
||||
suite.testAccounts = testrig.NewTestAccounts()
|
||||
suite.testAttachments = testrig.NewTestAttachments()
|
||||
suite.testStatuses = testrig.NewTestStatuses()
|
||||
suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media")
|
||||
}
|
||||
|
||||
func (suite *MediaStandardTestSuite) SetupTest() {
|
||||
testrig.InitTestConfig()
|
||||
testrig.InitTestLog()
|
||||
|
||||
suite.db = testrig.NewTestDB()
|
||||
suite.tc = testrig.NewTestTypeConverter(suite.db)
|
||||
suite.storage = testrig.NewTestStorage()
|
||||
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
|
||||
suite.transportController = suite.mockTransportController()
|
||||
suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
|
||||
testrig.StandardDBSetup(suite.db, nil)
|
||||
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
|
||||
}
|
||||
|
||||
func (suite *MediaStandardTestSuite) TearDownTest() {
|
||||
testrig.StandardDBTeardown(suite.db)
|
||||
testrig.StandardStorageTeardown(suite.storage)
|
||||
}
|
||||
|
||||
func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller {
|
||||
do := func(req *http.Request) (*http.Response, error) {
|
||||
logrus.Debugf("received request for %s", req.URL)
|
||||
|
||||
responseBytes := []byte{}
|
||||
responseType := ""
|
||||
responseLength := 0
|
||||
|
||||
if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
|
||||
responseBytes = attachment.Data
|
||||
responseType = attachment.ContentType
|
||||
}
|
||||
|
||||
if len(responseBytes) != 0 {
|
||||
// we found something, so print what we're going to return
|
||||
logrus.Debugf("returning response %s", string(responseBytes))
|
||||
}
|
||||
responseLength = len(responseBytes)
|
||||
|
||||
reader := bytes.NewReader(responseBytes)
|
||||
readCloser := io.NopCloser(reader)
|
||||
response := &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: readCloser,
|
||||
ContentLength: int64(responseLength),
|
||||
Header: http.Header{
|
||||
"content-type": {responseType},
|
||||
},
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
mockClient := testrig.NewMockHTTPClient(do)
|
||||
return testrig.NewTestTransportController(mockClient, suite.db)
|
||||
}
|
@@ -115,7 +115,7 @@ type Processor interface {
|
||||
BlocksGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, limit int) (*apimodel.BlocksResponse, gtserror.WithCode)
|
||||
|
||||
// FileGet handles the fetching of a media attachment file via the fileserver.
|
||||
FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
|
||||
FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode)
|
||||
|
||||
// FollowRequestsGet handles the getting of the authed account's incoming follow requests
|
||||
FollowRequestsGet(ctx context.Context, auth *oauth.Auth) ([]apimodel.Account, gtserror.WithCode)
|
||||
@@ -270,7 +270,7 @@ func NewProcessor(
|
||||
streamingProcessor := streaming.New(db, oauthServer)
|
||||
accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator)
|
||||
adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
|
||||
mediaProcessor := mediaProcessor.New(db, tc, mediaManager, storage)
|
||||
mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
|
||||
userProcessor := user.New(db, emailSender)
|
||||
federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
|
||||
filter := visibility.NewFilter(db)
|
||||
|
@@ -52,7 +52,7 @@ func (suite *NotificationTestSuite) TestStreamNotification() {
|
||||
suite.NoError(err)
|
||||
|
||||
msg := <-openStream.Messages
|
||||
suite.Equal(`{"id":"01FH57SJCMDWQGEAJ0X08CE3WV","type":"follow","created_at":"2021-10-04T10:52:36+02:00","account":{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"bot":false,"created_at":"2021-09-26T12:52:36+02:00","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":0,"following_count":0,"statuses_count":0,"last_status_at":"","emojis":[],"fields":[]}}`, msg.Payload)
|
||||
suite.Equal(`{"id":"01FH57SJCMDWQGEAJ0X08CE3WV","type":"follow","created_at":"2021-10-04T10:52:36+02:00","account":{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"bot":false,"created_at":"2021-09-26T12:52:36+02:00","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":0,"following_count":0,"statuses_count":1,"last_status_at":"2021-09-20T10:40:37Z","emojis":[],"fields":[]}}`, msg.Payload)
|
||||
}
|
||||
|
||||
func TestNotificationTestSuite(t *testing.T) {
|
||||
|
Reference in New Issue
Block a user