diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go
index 4a8993598..709b3d481 100644
--- a/cmd/gotosocial/action/server/server.go
+++ b/cmd/gotosocial/action/server/server.go
@@ -239,7 +239,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
}
// perform initial media prune in case value of MediaRemoteCacheDays changed
- if err := processor.AdminMediaRemotePrune(ctx, viper.GetInt(config.Keys.MediaRemoteCacheDays)); err != nil {
+ if err := processor.AdminMediaPrune(ctx, viper.GetInt(config.Keys.MediaRemoteCacheDays)); err != nil {
return fmt.Errorf("error during initial media prune: %s", err)
}
diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml
index 7bfb96a35..f61bcbaea 100644
--- a/docs/api/swagger.yaml
+++ b/docs/api/swagger.yaml
@@ -2580,6 +2580,7 @@ paths:
- application/json
- application/xml
- application/x-www-form-urlencoded
+ description: Also cleans up unused headers + avatars from the media cache.
operationId: mediaCleanup
parameters:
- description: |-
diff --git a/internal/api/client/admin/mediacleanup.go b/internal/api/client/admin/mediacleanup.go
index 0a8852ff3..508840b23 100644
--- a/internal/api/client/admin/mediacleanup.go
+++ b/internal/api/client/admin/mediacleanup.go
@@ -33,6 +33,7 @@ import (
// MediaCleanupPOSTHandler swagger:operation POST /api/v1/admin/media_cleanup mediaCleanup
//
// Clean up remote media older than the specified number of days.
+// Also cleans up unused headers + avatars from the media cache.
//
// ---
// tags:
@@ -100,7 +101,7 @@ func (m *Module) MediaCleanupPOSTHandler(c *gin.Context) {
remoteCacheDays = 0
}
- if errWithCode := m.processor.AdminMediaRemotePrune(c.Request.Context(), remoteCacheDays); errWithCode != nil {
+ if errWithCode := m.processor.AdminMediaPrune(c.Request.Context(), remoteCacheDays); errWithCode != nil {
l.Debugf("error starting prune of remote media: %s", errWithCode.Error())
c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
return
diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go
index 4da80e757..fc3280ddf 100644
--- a/internal/db/bundb/media.go
+++ b/internal/db/bundb/media.go
@@ -72,3 +72,29 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l
}
return attachments, nil
}
+
+func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
+ attachments := []*gtsmodel.MediaAttachment{}
+
+ q := m.newMediaQ(&attachments).
+ WhereGroup(" AND ", func(innerQ *bun.SelectQuery) *bun.SelectQuery {
+ return innerQ.
+ WhereOr("media_attachment.avatar = true").
+ WhereOr("media_attachment.header = true")
+ }).
+ Order("media_attachment.id DESC")
+
+ if maxID != "" {
+ q = q.Where("media_attachment.id < ?", maxID)
+ }
+
+ if limit != 0 {
+ q = q.Limit(limit)
+ }
+
+ if err := q.Scan(ctx); err != nil {
+ return nil, m.conn.ProcessError(err)
+ }
+
+ return attachments, nil
+}
diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go
index 3138caf3b..f1809b3fb 100644
--- a/internal/db/bundb/media_test.go
+++ b/internal/db/bundb/media_test.go
@@ -43,6 +43,14 @@ func (suite *MediaTestSuite) TestGetOlder() {
suite.Len(attachments, 2)
}
+func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
+ ctx := context.Background()
+
+ attachments, err := suite.db.GetAvatarsAndHeaders(ctx, "", 20)
+ suite.NoError(err)
+ suite.Len(attachments, 2)
+}
+
func TestMediaTestSuite(t *testing.T) {
suite.Run(t, new(MediaTestSuite))
}
diff --git a/internal/db/media.go b/internal/db/media.go
index c734502a1..636fc61f2 100644
--- a/internal/db/media.go
+++ b/internal/db/media.go
@@ -35,4 +35,7 @@ type Media interface {
// The selected media attachments will be those with both a URL and a RemoteURL filled in.
// In other words, media attachments that originated remotely, and that we currently have cached locally.
GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
+ // GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers
+ // and avis may be in use or not; the caller should check this if it's important.
+ GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
}
diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go
index 20cc6d3bf..2cd287eea 100644
--- a/internal/gtsmodel/mediaattachment.go
+++ b/internal/gtsmodel/mediaattachment.go
@@ -34,7 +34,7 @@ type MediaAttachment struct {
Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video)
FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file
AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong
- Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID
+ Account *Account `validate:"-" bun:"rel:belongs-to,join:account_id=id"` // Account corresponding to accountID
Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders)
ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong
Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
diff --git a/internal/media/manager.go b/internal/media/manager.go
index 5b4a01021..60290e4ff 100644
--- a/internal/media/manager.go
+++ b/internal/media/manager.go
@@ -32,6 +32,9 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
)
+// selectPruneLimit is the amount of media entries to select at a time from the db when pruning
+const selectPruneLimit = 20
+
// Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs.
type Manager interface {
// ProcessMedia begins the process of decoding and storing the given data as an attachment.
@@ -66,10 +69,19 @@ type Manager interface {
ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error)
// RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote.
RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error)
- // PruneRemote prunes all remote media cached on this instance that's older than the given amount of days.
+
+ // PruneAllRemote prunes all remote media attachments cached on this instance which are older than the given amount of days.
// 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size),
// and setting 'cached' to false on the associated attachment.
- PruneRemote(ctx context.Context, olderThanDays int) (int, error)
+ //
+ // The returned int is the amount of media that was pruned by this function.
+ PruneAllRemote(ctx context.Context, olderThanDays int) (int, error)
+ // PruneAllMeta prunes unused meta media -- currently, this means unused avatars + headers, but can also be extended
+ // to include things like attachments that were uploaded on this server but left unused, etc.
+ //
+ // The returned int is the amount of media that was pruned by this function.
+ PruneAllMeta(ctx context.Context) (int, error)
+
// Stop stops the underlying worker pool of the manager. It should be called
// when closing GoToSocial in order to cleanly finish any in-progress jobs.
// It will block until workers are finished processing.
@@ -128,53 +140,8 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
return nil, err
}
- // start remote cache cleanup cronjob if configured
- cacheCleanupDays := viper.GetInt(config.Keys.MediaRemoteCacheDays)
- if cacheCleanupDays != 0 {
- // we need a way of cancelling running jobs if the media manager is told to stop
- pruneCtx, pruneCancel := context.WithCancel(context.Background())
-
- // create a new cron instance and add a function to it
- c := cron.New(cron.WithLogger(&logrusWrapper{}))
-
- pruneFunc := func() {
- begin := time.Now()
- pruned, err := m.PruneRemote(pruneCtx, cacheCleanupDays)
- if err != nil {
- logrus.Errorf("media manager: error pruning remote cache: %s", err)
- return
- }
- logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin))
- }
-
- // run every night
- entryID, err := c.AddFunc("@midnight", pruneFunc)
- if err != nil {
- pruneCancel()
- return nil, fmt.Errorf("error starting media manager remote cache cleanup job: %s", err)
- }
-
- // since we're running a cron job, we should define how the manager should stop them
- m.stopCronJobs = func() error {
- // try to stop any jobs gracefully by waiting til they're finished
- cronCtx := c.Stop()
-
- select {
- case <-cronCtx.Done():
- logrus.Infof("media manager: cron finished jobs and stopped gracefully")
- case <-time.After(1 * time.Minute):
- logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close")
- break
- }
-
- // whether the job is finished neatly or we had to wait a minute, cancel the context on the prune job
- pruneCancel()
- return nil
- }
-
- // now start all the cron stuff we've lined up
- c.Start()
- logrus.Infof("media manager: next scheduled remote cache cleanup is %q", c.Entry(entryID).Next)
+ if err := scheduleCleanupJobs(m); err != nil {
+ return nil, err
}
return m, nil
@@ -213,9 +180,7 @@ func (m *manager) Stop() error {
emojiErr := m.emojiWorker.Stop()
var cronErr error
-
if m.stopCronJobs != nil {
- // only set if cache prune age > 0
cronErr = m.stopCronJobs()
}
@@ -224,5 +189,60 @@ func (m *manager) Stop() error {
} else if emojiErr != nil {
return emojiErr
}
+
return cronErr
}
+
+func scheduleCleanupJobs(m *manager) error {
+ // create a new cron instance for scheduling cleanup jobs
+ c := cron.New(cron.WithLogger(&logrusWrapper{}))
+ pruneCtx, pruneCancel := context.WithCancel(context.Background())
+
+ if _, err := c.AddFunc("@midnight", func() {
+ begin := time.Now()
+ pruned, err := m.PruneAllMeta(pruneCtx)
+ if err != nil {
+ logrus.Errorf("media manager: error pruning meta: %s", err)
+ return
+ }
+ logrus.Infof("media manager: pruned %d meta entries in %s", pruned, time.Since(begin))
+ }); err != nil {
+ pruneCancel()
+ return fmt.Errorf("error starting media manager meta cleanup job: %s", err)
+ }
+
+ // start remote cache cleanup cronjob if configured
+ if mediaRemoteCacheDays := viper.GetInt(config.Keys.MediaRemoteCacheDays); mediaRemoteCacheDays > 0 {
+ if _, err := c.AddFunc("@midnight", func() {
+ begin := time.Now()
+ pruned, err := m.PruneAllRemote(pruneCtx, mediaRemoteCacheDays)
+ if err != nil {
+ logrus.Errorf("media manager: error pruning remote cache: %s", err)
+ return
+ }
+ logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin))
+ }); err != nil {
+ pruneCancel()
+ return fmt.Errorf("error starting media manager remote cache cleanup job: %s", err)
+ }
+ }
+
+ // try to stop any jobs gracefully by waiting til they're finished
+ m.stopCronJobs = func() error {
+ cronCtx := c.Stop()
+
+ select {
+ case <-cronCtx.Done():
+ logrus.Infof("media manager: cron finished jobs and stopped gracefully")
+ case <-time.After(1 * time.Minute):
+ logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close jobs")
+ break
+ }
+
+ pruneCancel()
+ return nil
+ }
+
+ c.Start()
+ return nil
+}
diff --git a/internal/media/media_test.go b/internal/media/media_test.go
index ee0fd8eea..1b5011801 100644
--- a/internal/media/media_test.go
+++ b/internal/media/media_test.go
@@ -34,6 +34,7 @@ type MediaStandardTestSuite struct {
storage *kv.KVStore
manager media.Manager
testAttachments map[string]*gtsmodel.MediaAttachment
+ testAccounts map[string]*gtsmodel.Account
}
func (suite *MediaStandardTestSuite) SetupSuite() {
@@ -48,6 +49,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
testrig.StandardDBSetup(suite.db, nil)
suite.testAttachments = testrig.NewTestAttachments()
+ suite.testAccounts = testrig.NewTestAccounts()
suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage)
}
diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go
new file mode 100644
index 000000000..aa838d2a4
--- /dev/null
+++ b/internal/media/prunemeta.go
@@ -0,0 +1,87 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+ "context"
+
+ "codeberg.org/gruf/go-store/storage"
+ "github.com/sirupsen/logrus"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (m *manager) PruneAllMeta(ctx context.Context) (int, error) {
+ var totalPruned int
+ var maxID string
+ var attachments []*gtsmodel.MediaAttachment
+ var err error
+
+ // select 20 attachments at a time and prune them
+ for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) {
+ // use the id of the last attachment in the slice as the next 'maxID' value
+ l := len(attachments)
+ logrus.Tracef("PruneAllMeta: got %d attachments with maxID < %s", l, maxID)
+ maxID = attachments[l-1].ID
+
+ // prune each attachment that meets one of the following criteria:
+ // - has no owning account in the database
+ // - is a header but isn't the owning account's current header
+ // - is an avatar but isn't the owning account's current avatar
+ for _, attachment := range attachments {
+ if attachment.Account == nil ||
+ (attachment.Header && attachment.ID != attachment.Account.HeaderMediaAttachmentID) ||
+ (attachment.Avatar && attachment.ID != attachment.Account.AvatarMediaAttachmentID) {
+ if err := m.pruneOneAvatarOrHeader(ctx, attachment); err != nil {
+ return totalPruned, err
+ }
+ totalPruned++
+ }
+ }
+ }
+
+ // make sure we don't have a real error when we leave the loop
+ if err != nil && err != db.ErrNoEntries {
+ return totalPruned, err
+ }
+
+ logrus.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned)
+ return totalPruned, nil
+}
+
+func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
+ if attachment.File.Path != "" {
+ // delete the full size attachment from storage
+ logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path)
+ if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ return err
+ }
+ }
+
+ if attachment.Thumbnail.Path != "" {
+ // delete the thumbnail from storage
+ logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path)
+ if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ return err
+ }
+ }
+
+ // delete the attachment entry completely
+ return m.db.DeleteByID(ctx, attachment.ID, >smodel.MediaAttachment{})
+}
diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go
new file mode 100644
index 000000000..1358208a8
--- /dev/null
+++ b/internal/media/prunemeta_test.go
@@ -0,0 +1,131 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+ "context"
+ "testing"
+
+ "codeberg.org/gruf/go-store/storage"
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+type PruneMetaTestSuite struct {
+ MediaStandardTestSuite
+}
+
+func (suite *PruneMetaTestSuite) TestPruneMeta() {
+ ctx := context.Background()
+
+ // start by clearing zork's avatar + header
+ zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
+ zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
+ zork := suite.testAccounts["local_account_1"]
+ zork.AvatarMediaAttachmentID = ""
+ zork.HeaderMediaAttachmentID = ""
+ if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil {
+ panic(err)
+ }
+
+ totalPruned, err := suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(2, totalPruned)
+
+ // media should no longer be stored
+ _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+
+ // attachments should no longer be in the db
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+}
+
+func (suite *PruneMetaTestSuite) TestPruneMetaTwice() {
+ ctx := context.Background()
+
+ // start by clearing zork's avatar + header
+ zork := suite.testAccounts["local_account_1"]
+ zork.AvatarMediaAttachmentID = ""
+ zork.HeaderMediaAttachmentID = ""
+ if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil {
+ panic(err)
+ }
+
+ totalPruned, err := suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(2, totalPruned)
+
+ // final prune should prune nothing, since the first prune already happened
+ totalPruned, err = suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(0, totalPruned)
+}
+func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() {
+ ctx := context.Background()
+
+ // start by clearing zork's avatar + header
+ zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
+ zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
+ zork := suite.testAccounts["local_account_1"]
+ zork.AvatarMediaAttachmentID = ""
+ zork.HeaderMediaAttachmentID = ""
+ if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil {
+ panic(err)
+ }
+
+ // set zork's unused header as belonging to turtle
+ turtle := suite.testAccounts["local_account_1"]
+ zorkOldHeader.AccountID = turtle.ID
+ if err := suite.db.UpdateByPrimaryKey(ctx, zorkOldHeader); err != nil {
+ panic(err)
+ }
+
+ totalPruned, err := suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(2, totalPruned)
+
+ // media should no longer be stored
+ _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+
+ // attachments should no longer be in the db
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+}
+
+func TestPruneMetaTestSuite(t *testing.T) {
+ suite.Run(t, &PruneMetaTestSuite{})
+}
diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go
index 372f7bbb9..f7b77d32e 100644
--- a/internal/media/pruneremote.go
+++ b/internal/media/pruneremote.go
@@ -29,10 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
-// amount of media attachments to select at a time from the db when pruning
-const selectPruneLimit = 20
-
-func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, error) {
+func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) {
var totalPruned int
// convert days into a duration string
@@ -40,23 +37,23 @@ func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, erro
// parse the duration string into a duration
olderThanHours, err := time.ParseDuration(olderThanHoursString)
if err != nil {
- return totalPruned, fmt.Errorf("PruneRemote: %d", err)
+ return totalPruned, fmt.Errorf("PruneAllRemote: %d", err)
}
// 'subtract' that from the time now to give our threshold
olderThan := time.Now().Add(-olderThanHours)
- logrus.Infof("PruneRemote: pruning media older than %s", olderThan)
+ logrus.Infof("PruneAllRemote: pruning media older than %s", olderThan)
// select 20 attachments at a time and prune them
for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
// use the age of the oldest attachment (the last one in the slice) as the next 'older than' value
l := len(attachments)
- logrus.Tracef("PruneRemote: got %d attachments older than %s", l, olderThan)
+ logrus.Tracef("PruneAllRemote: got %d attachments older than %s", l, olderThan)
olderThan = attachments[l-1].CreatedAt
// prune each attachment
for _, attachment := range attachments {
- if err := m.PruneOne(ctx, attachment); err != nil {
+ if err := m.pruneOneRemote(ctx, attachment); err != nil {
return totalPruned, err
}
totalPruned++
@@ -68,14 +65,14 @@ func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, erro
return totalPruned, err
}
- logrus.Infof("PruneRemote: finished pruning remote media: pruned %d entries", totalPruned)
+ logrus.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned)
return totalPruned, nil
}
-func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
+func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
if attachment.File.Path != "" {
// delete the full size attachment from storage
- logrus.Tracef("PruneOne: deleting %s", attachment.File.Path)
+ logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
@@ -84,7 +81,7 @@ func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttach
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
- logrus.Tracef("PruneOne: deleting %s", attachment.Thumbnail.Path)
+ logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go
index c9d040a6f..31c5128ff 100644
--- a/internal/media/pruneremote_test.go
+++ b/internal/media/pruneremote_test.go
@@ -37,7 +37,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
suite.True(testAttachment.Cached)
- totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
+ totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
@@ -49,12 +49,12 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
}
func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
- totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
+ totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
// final prune should prune nothing, since the first prune already happened
- totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1)
+ totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(0, totalPrunedAgain)
}
@@ -63,7 +63,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
ctx := context.Background()
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
- totalPruned, err := suite.manager.PruneRemote(ctx, 1)
+ totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
@@ -116,7 +116,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
suite.NoError(err)
// Now attempt to prune remote for item with db entry no file
- totalPruned, err := suite.manager.PruneRemote(ctx, 1)
+ totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
}
diff --git a/internal/processing/admin.go b/internal/processing/admin.go
index 10f3ff8ba..cbbea05b1 100644
--- a/internal/processing/admin.go
+++ b/internal/processing/admin.go
@@ -54,6 +54,6 @@ func (p *processor) AdminDomainBlockDelete(ctx context.Context, authed *oauth.Au
return p.adminProcessor.DomainBlockDelete(ctx, authed.Account, id)
}
-func (p *processor) AdminMediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode {
- return p.adminProcessor.MediaRemotePrune(ctx, mediaRemoteCacheDays)
+func (p *processor) AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode {
+ return p.adminProcessor.MediaPrune(ctx, mediaRemoteCacheDays)
}
diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go
index 6779f59b7..c528f0fb8 100644
--- a/internal/processing/admin/admin.go
+++ b/internal/processing/admin/admin.go
@@ -41,7 +41,7 @@ type Processor interface {
DomainBlockDelete(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.DomainBlock, gtserror.WithCode)
AccountAction(ctx context.Context, account *gtsmodel.Account, form *apimodel.AdminAccountActionRequest) gtserror.WithCode
EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode)
- MediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode
+ MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode
}
type processor struct {
diff --git a/internal/processing/admin/mediaremoteprune.go b/internal/processing/admin/mediaprune.go
similarity index 60%
rename from internal/processing/admin/mediaremoteprune.go
rename to internal/processing/admin/mediaprune.go
index e4a50cab8..0e6abe028 100644
--- a/internal/processing/admin/mediaremoteprune.go
+++ b/internal/processing/admin/mediaprune.go
@@ -26,18 +26,27 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
-func (p *processor) MediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode {
+func (p *processor) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode {
if mediaRemoteCacheDays < 0 {
- err := fmt.Errorf("invalid value for mediaRemoteCacheDays prune: value was %d, cannot be less than 0", mediaRemoteCacheDays)
+ err := fmt.Errorf("MediaPrune: invalid value for mediaRemoteCacheDays prune: value was %d, cannot be less than 0", mediaRemoteCacheDays)
return gtserror.NewErrorBadRequest(err, err.Error())
}
go func() {
- pruned, err := p.mediaManager.PruneRemote(ctx, mediaRemoteCacheDays)
+ pruned, err := p.mediaManager.PruneAllRemote(ctx, mediaRemoteCacheDays)
if err != nil {
- logrus.Errorf("MediaRemotePrune: error pruning: %s", err)
+ logrus.Errorf("MediaPrune: error pruning remote cache: %s", err)
} else {
- logrus.Infof("MediaRemotePrune: pruned %d entries", pruned)
+ logrus.Infof("MediaPrune: pruned %d remote cache entries", pruned)
+ }
+ }()
+
+ go func() {
+ pruned, err := p.mediaManager.PruneAllMeta(ctx)
+ if err != nil {
+ logrus.Errorf("MediaPrune: error pruning meta: %s", err)
+ } else {
+ logrus.Infof("MediaPrune: pruned %d meta entries", pruned)
}
}()
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
index d30f2f37e..f34cc568f 100644
--- a/internal/processing/processor.go
+++ b/internal/processing/processor.go
@@ -114,7 +114,7 @@ type Processor interface {
// AdminDomainBlockDelete deletes one domain block, specified by ID, returning the deleted domain block.
AdminDomainBlockDelete(ctx context.Context, authed *oauth.Auth, id string) (*apimodel.DomainBlock, gtserror.WithCode)
// AdminMediaRemotePrune triggers a prune of remote media according to the given number of mediaRemoteCacheDays
- AdminMediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode
+ AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode
// AppCreate processes the creation of a new API application
AppCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.ApplicationCreateRequest) (*apimodel.Application, error)