From a8e6bdfa33f3232ebc8f241b9c90e4da9191a627 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Fri, 3 Mar 2023 23:02:23 +0000 Subject: [PATCH] [performance] cache media attachments (#1525) * replace concurrency worker pools with base models in State.Workers, update code and tests accordingly * add media attachment caching, slightly tweak default cache config * further tweak default cache config values * replace other media attachment db calls to go through cache * update envparsing test * fix delete media attachment sql * fix media sql query * invalidate cached media entries during status create / update * fix envparsing test * fix typo in panic log message... * add 'updated_at' column during UpdateAttachment * remove unused func --------- Signed-off-by: kim --- example/config.yaml | 26 ++++---- internal/cache/gts.go | 24 ++++++++ internal/config/config.go | 4 ++ internal/config/defaults.go | 26 ++++---- internal/config/helpers.gen.go | 75 +++++++++++++++++++++++ internal/db/bundb/media.go | 86 +++++++++++++++++++-------- internal/db/bundb/status.go | 22 ++++++- internal/db/media.go | 11 +++- internal/media/processingmedia.go | 8 +-- internal/media/prune.go | 4 +- internal/processing/instance.go | 2 +- internal/processing/media/delete.go | 2 +- internal/processing/media/unattach.go | 2 +- test/envparsing.sh | 2 +- testrig/config.go | 2 +- 15 files changed, 235 insertions(+), 61 deletions(-) diff --git a/example/config.yaml b/example/config.yaml index 2239e0ba6..74c47dd8b 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -239,13 +239,13 @@ cache: # ttl = cached object lifetime # sweep-freq = frequency to look for stale cache objects - account-max-size: 100 + account-max-size: 500 account-ttl: "5m" - account-sweep-freq: "10s" + account-sweep-freq: "30s" block-max-size: 100 block-ttl: "5m" - block-sweep-freq: "10s" + block-sweep-freq: "30s" domain-block-max-size: 1000 domain-block-ttl: "24h" @@ -253,35 +253,39 @@ cache: emoji-max-size: 500 emoji-ttl: "5m" - emoji-sweep-freq: "10s" + emoji-sweep-freq: "30s" emoji-category-max-size: 100 emoji-category-ttl: "5m" - emoji-category-sweep-freq: "10s" + emoji-category-sweep-freq: "30s" + + media-max-size: 500 + media-ttl: "5m" + media-sweep-freq: "30s" mention-max-size: 500 mention-ttl: "5m" - mention-sweep-freq: "10s" + mention-sweep-freq: "30s" notification-max-size: 500 notification-ttl: "5m" - notification-sweep-freq: "10s" + notification-sweep-freq: "30s" report-max-size: 100 report-ttl: "5m" - report-sweep-freq: "10s" + report-sweep-freq: "30s" status-max-size: 500 status-ttl: "5m" - status-sweep-freq: "10s" + status-sweep-freq: "30s" tombstone-max-size: 100 tombstone-ttl: "5m" - tombstone-sweep-freq: "10s" + tombstone-sweep-freq: "30s" user-max-size: 100 user-ttl: "5m" - user-sweep-freq: "10s" + user-sweep-freq: "30s" ###################### ##### WEB CONFIG ##### diff --git a/internal/cache/gts.go b/internal/cache/gts.go index f3f7a33ef..253dc47b2 100644 --- a/internal/cache/gts.go +++ b/internal/cache/gts.go @@ -54,6 +54,9 @@ type GTSCaches interface { // Mention provides access to the gtsmodel Mention database cache. Mention() *result.Cache[*gtsmodel.Mention] + // Media provides access to the gtsmodel Media database cache. + Media() *result.Cache[*gtsmodel.MediaAttachment] + // Notification provides access to the gtsmodel Notification database cache. Notification() *result.Cache[*gtsmodel.Notification] @@ -81,6 +84,7 @@ type gtsCaches struct { domainBlock *domain.BlockCache emoji *result.Cache[*gtsmodel.Emoji] emojiCategory *result.Cache[*gtsmodel.EmojiCategory] + media *result.Cache[*gtsmodel.MediaAttachment] mention *result.Cache[*gtsmodel.Mention] notification *result.Cache[*gtsmodel.Notification] report *result.Cache[*gtsmodel.Report] @@ -95,6 +99,7 @@ func (c *gtsCaches) Init() { c.initDomainBlock() c.initEmoji() c.initEmojiCategory() + c.initMedia() c.initMention() c.initNotification() c.initReport() @@ -119,6 +124,9 @@ func (c *gtsCaches) Start() { tryUntil("starting gtsmodel.EmojiCategory cache", 5, func() bool { return c.emojiCategory.Start(config.GetCacheGTSEmojiCategorySweepFreq()) }) + tryUntil("starting gtsmodel.MediaAttachment cache", 5, func() bool { + return c.media.Start(config.GetCacheGTSMediaSweepFreq()) + }) tryUntil("starting gtsmodel.Mention cache", 5, func() bool { return c.mention.Start(config.GetCacheGTSMentionSweepFreq()) }) @@ -145,6 +153,7 @@ func (c *gtsCaches) Stop() { tryUntil("stopping gtsmodel.DomainBlock cache", 5, c.domainBlock.Stop) tryUntil("stopping gtsmodel.Emoji cache", 5, c.emoji.Stop) tryUntil("stopping gtsmodel.EmojiCategory cache", 5, c.emojiCategory.Stop) + tryUntil("stopping gtsmodel.MediaAttachment cache", 5, c.media.Stop) tryUntil("stopping gtsmodel.Mention cache", 5, c.mention.Stop) tryUntil("stopping gtsmodel.Notification cache", 5, c.notification.Stop) tryUntil("stopping gtsmodel.Report cache", 5, c.report.Stop) @@ -173,6 +182,10 @@ func (c *gtsCaches) EmojiCategory() *result.Cache[*gtsmodel.EmojiCategory] { return c.emojiCategory } +func (c *gtsCaches) Media() *result.Cache[*gtsmodel.MediaAttachment] { + return c.media +} + func (c *gtsCaches) Mention() *result.Cache[*gtsmodel.Mention] { return c.mention } @@ -258,6 +271,17 @@ func (c *gtsCaches) initEmojiCategory() { c.emojiCategory.SetTTL(config.GetCacheGTSEmojiCategoryTTL(), true) } +func (c *gtsCaches) initMedia() { + c.media = result.New([]result.Lookup{ + {Name: "ID"}, + }, func(m1 *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment { + m2 := new(gtsmodel.MediaAttachment) + *m2 = *m1 + return m2 + }, config.GetCacheGTSMediaMaxSize()) + c.media.SetTTL(config.GetCacheGTSMediaTTL(), true) +} + func (c *gtsCaches) initMention() { c.mention = result.New([]result.Lookup{ {Name: "ID"}, diff --git a/internal/config/config.go b/internal/config/config.go index 5673b76dd..fdfda8583 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -177,6 +177,10 @@ type GTSCacheConfiguration struct { EmojiCategoryTTL time.Duration `name:"emoji-category-ttl"` EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"` + MediaMaxSize int `name:"media-max-size"` + MediaTTL time.Duration `name:"media-ttl"` + MediaSweepFreq time.Duration `name:"media-sweep-freq"` + MentionMaxSize int `name:"mention-max-size"` MentionTTL time.Duration `name:"mention-ttl"` MentionSweepFreq time.Duration `name:"mention-sweep-freq"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 528419e97..e9dd2b743 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -116,13 +116,13 @@ var Defaults = Configuration{ Cache: CacheConfiguration{ GTS: GTSCacheConfiguration{ - AccountMaxSize: 100, + AccountMaxSize: 500, AccountTTL: time.Minute * 5, - AccountSweepFreq: time.Second * 10, + AccountSweepFreq: time.Second * 30, BlockMaxSize: 100, BlockTTL: time.Minute * 5, - BlockSweepFreq: time.Second * 10, + BlockSweepFreq: time.Second * 30, DomainBlockMaxSize: 1000, DomainBlockTTL: time.Hour * 24, @@ -130,35 +130,39 @@ var Defaults = Configuration{ EmojiMaxSize: 500, EmojiTTL: time.Minute * 5, - EmojiSweepFreq: time.Second * 10, + EmojiSweepFreq: time.Second * 30, EmojiCategoryMaxSize: 100, EmojiCategoryTTL: time.Minute * 5, - EmojiCategorySweepFreq: time.Second * 10, + EmojiCategorySweepFreq: time.Second * 30, + + MediaMaxSize: 500, + MediaTTL: time.Minute * 5, + MediaSweepFreq: time.Second * 30, MentionMaxSize: 500, MentionTTL: time.Minute * 5, - MentionSweepFreq: time.Second * 10, + MentionSweepFreq: time.Second * 30, NotificationMaxSize: 500, NotificationTTL: time.Minute * 5, - NotificationSweepFreq: time.Second * 10, + NotificationSweepFreq: time.Second * 30, ReportMaxSize: 100, ReportTTL: time.Minute * 5, - ReportSweepFreq: time.Second * 10, + ReportSweepFreq: time.Second * 30, StatusMaxSize: 500, StatusTTL: time.Minute * 5, - StatusSweepFreq: time.Second * 10, + StatusSweepFreq: time.Second * 30, TombstoneMaxSize: 100, TombstoneTTL: time.Minute * 5, - TombstoneSweepFreq: time.Second * 10, + TombstoneSweepFreq: time.Second * 30, UserMaxSize: 100, UserTTL: time.Minute * 5, - UserSweepFreq: time.Second * 10, + UserSweepFreq: time.Second * 30, }, }, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 41c56a571..5ea7b61b6 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -2426,6 +2426,81 @@ func GetCacheGTSEmojiCategorySweepFreq() time.Duration { // SetCacheGTSEmojiCategorySweepFreq safely sets the value for global configuration 'Cache.GTS.EmojiCategorySweepFreq' field func SetCacheGTSEmojiCategorySweepFreq(v time.Duration) { global.SetCacheGTSEmojiCategorySweepFreq(v) } +// GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field +func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) { + st.mutex.Lock() + v = st.config.Cache.GTS.MediaMaxSize + st.mutex.Unlock() + return +} + +// SetCacheGTSMediaMaxSize safely sets the Configuration value for state's 'Cache.GTS.MediaMaxSize' field +func (st *ConfigState) SetCacheGTSMediaMaxSize(v int) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.GTS.MediaMaxSize = v + st.reloadToViper() +} + +// CacheGTSMediaMaxSizeFlag returns the flag name for the 'Cache.GTS.MediaMaxSize' field +func CacheGTSMediaMaxSizeFlag() string { return "cache-gts-media-max-size" } + +// GetCacheGTSMediaMaxSize safely fetches the value for global configuration 'Cache.GTS.MediaMaxSize' field +func GetCacheGTSMediaMaxSize() int { return global.GetCacheGTSMediaMaxSize() } + +// SetCacheGTSMediaMaxSize safely sets the value for global configuration 'Cache.GTS.MediaMaxSize' field +func SetCacheGTSMediaMaxSize(v int) { global.SetCacheGTSMediaMaxSize(v) } + +// GetCacheGTSMediaTTL safely fetches the Configuration value for state's 'Cache.GTS.MediaTTL' field +func (st *ConfigState) GetCacheGTSMediaTTL() (v time.Duration) { + st.mutex.Lock() + v = st.config.Cache.GTS.MediaTTL + st.mutex.Unlock() + return +} + +// SetCacheGTSMediaTTL safely sets the Configuration value for state's 'Cache.GTS.MediaTTL' field +func (st *ConfigState) SetCacheGTSMediaTTL(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.GTS.MediaTTL = v + st.reloadToViper() +} + +// CacheGTSMediaTTLFlag returns the flag name for the 'Cache.GTS.MediaTTL' field +func CacheGTSMediaTTLFlag() string { return "cache-gts-media-ttl" } + +// GetCacheGTSMediaTTL safely fetches the value for global configuration 'Cache.GTS.MediaTTL' field +func GetCacheGTSMediaTTL() time.Duration { return global.GetCacheGTSMediaTTL() } + +// SetCacheGTSMediaTTL safely sets the value for global configuration 'Cache.GTS.MediaTTL' field +func SetCacheGTSMediaTTL(v time.Duration) { global.SetCacheGTSMediaTTL(v) } + +// GetCacheGTSMediaSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field +func (st *ConfigState) GetCacheGTSMediaSweepFreq() (v time.Duration) { + st.mutex.Lock() + v = st.config.Cache.GTS.MediaSweepFreq + st.mutex.Unlock() + return +} + +// SetCacheGTSMediaSweepFreq safely sets the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field +func (st *ConfigState) SetCacheGTSMediaSweepFreq(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.GTS.MediaSweepFreq = v + st.reloadToViper() +} + +// CacheGTSMediaSweepFreqFlag returns the flag name for the 'Cache.GTS.MediaSweepFreq' field +func CacheGTSMediaSweepFreqFlag() string { return "cache-gts-media-sweep-freq" } + +// GetCacheGTSMediaSweepFreq safely fetches the value for global configuration 'Cache.GTS.MediaSweepFreq' field +func GetCacheGTSMediaSweepFreq() time.Duration { return global.GetCacheGTSMediaSweepFreq() } + +// SetCacheGTSMediaSweepFreq safely sets the value for global configuration 'Cache.GTS.MediaSweepFreq' field +func SetCacheGTSMediaSweepFreq(v time.Duration) { global.SetCacheGTSMediaSweepFreq(v) } + // GetCacheGTSMentionMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MentionMaxSize' field func (st *ConfigState) GetCacheGTSMentionMaxSize() (v int) { st.mutex.Lock() diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index 6975df85f..8f6985bad 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -34,39 +34,69 @@ type mediaDB struct { state *state.State } -func (m *mediaDB) newMediaQ(i *gtsmodel.MediaAttachment) *bun.SelectQuery { - return m.conn. - NewSelect(). - Model(i) -} - func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, db.Error) { return m.getAttachment( ctx, "ID", func(attachment *gtsmodel.MediaAttachment) error { - return m.newMediaQ(attachment).Where("? = ?", bun.Ident("media_attachment.id"), id).Scan(ctx) + return m.conn.NewSelect(). + Model(attachment). + Where("? = ?", bun.Ident("media_attachment.id"), id). + Scan(ctx) }, id, ) } -func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) { - attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids)) +func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) { + return m.state.Caches.GTS.Media().Load(lookup, func() (*gtsmodel.MediaAttachment, error) { + var attachment gtsmodel.MediaAttachment - for _, id := range ids { - // Attempt fetch from DB - attachment, err := m.GetAttachmentByID(ctx, id) - if err != nil { - log.Errorf(ctx, "error getting attachment %q: %v", id, err) - continue + // Not cached! Perform database query + if err := dbQuery(&attachment); err != nil { + return nil, m.conn.ProcessError(err) } - // Append attachment - attachments = append(attachments, attachment) + return &attachment, nil + }, keyParts...) +} + +func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error { + return m.state.Caches.GTS.Media().Store(media, func() error { + _, err := m.conn.NewInsert().Model(media).Exec(ctx) + return m.conn.ProcessError(err) + }) +} + +func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error { + media.UpdatedAt = time.Now() + if len(columns) > 0 { + // If we're updating by column, ensure "updated_at" is included. + columns = append(columns, "updated_at") } - return attachments, nil + return m.state.Caches.GTS.Media().Store(media, func() error { + _, err := m.conn.NewUpdate(). + Model(media). + Where("? = ?", bun.Ident("media_attachment.id"), media.ID). + Column(columns...). + Exec(ctx) + return m.conn.ProcessError(err) + }) +} + +func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error { + // Attempt to delete from database. + if _, err := m.conn.NewDelete(). + TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")). + Where("? = ?", bun.Ident("media_attachment.id"), id). + Exec(ctx); err != nil { + return m.conn.ProcessError(err) + } + + // Invalidate this media item from the cache. + m.state.Caches.GTS.Media().Invalidate("ID", id) + return nil } func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) { @@ -183,14 +213,20 @@ func (m *mediaDB) CountLocalUnattachedOlderThan(ctx context.Context, olderThan t return count, nil } -func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) { - // Fetch attachment from database - // todo: cache this lookup - attachment := new(gtsmodel.MediaAttachment) +func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) { + attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids)) - if err := dbQuery(attachment); err != nil { - return nil, m.conn.ProcessError(err) + for _, id := range ids { + // Attempt fetch from DB + attachment, err := m.GetAttachmentByID(ctx, id) + if err != nil { + log.Errorf(ctx, "error getting attachment %q: %v", id, err) + continue + } + + // Append attachment + attachments = append(attachments, attachment) } - return attachment, nil + return attachments, nil } diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 72e44068d..3bbf67657 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -188,7 +188,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g } func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Error { - return s.state.Caches.GTS.Status().Store(status, func() error { + err := s.state.Caches.GTS.Status().Store(status, func() error { // It is safe to run this database transaction within cache.Store // as the cache does not attempt a mutex lock until AFTER hook. // @@ -248,6 +248,17 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er return err }) }) + if err != nil { + // already processed + return err + } + + for _, id := range status.AttachmentIDs { + // Clear updated media attachment IDs from cache + s.state.Caches.GTS.Media().Invalidate("ID", id) + } + + return nil } func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) db.Error { @@ -317,11 +328,18 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co Exec(ctx) return err }); err != nil { + // already processed return err } - // Drop any old value from cache by this ID + for _, id := range status.AttachmentIDs { + // Clear updated media attachment IDs from cache + s.state.Caches.GTS.Media().Invalidate("ID", id) + } + + // Drop any old status value from cache by this ID s.state.Caches.GTS.Status().Invalidate("ID", status.ID) + return nil } diff --git a/internal/db/media.go b/internal/db/media.go index 3756bd58f..131d0ab08 100644 --- a/internal/db/media.go +++ b/internal/db/media.go @@ -27,9 +27,18 @@ import ( // Media contains functions related to creating/getting/removing media attachments. type Media interface { - // GetAttachmentByID gets a single attachment by its ID + // GetAttachmentByID gets a single attachment by its ID. GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error) + // PutAttachment inserts the given attachment into the database. + PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error + + // UpdateAttachment will update the given attachment in the database. + UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error + + // DeleteAttachment deletes the attachment with given ID from the database. + DeleteAttachment(ctx context.Context, id string) error + // GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given // olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words). // diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index c00c203a6..a315fa387 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -123,13 +123,13 @@ func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment, } if p.recache { - // Existing attachment we're recaching, so only need to update. - err = p.mgr.state.DB.UpdateByID(ctx, p.media, p.media.ID) + // Existing attachment we're recaching, so only update. + err = p.mgr.state.DB.UpdateAttachment(ctx, p.media) return err } - // New attachment, first time caching. - err = p.mgr.state.DB.Put(ctx, p.media) + // First time caching this attachment, insert it. + err = p.mgr.state.DB.PutAttachment(ctx, p.media) return err }) diff --git a/internal/media/prune.go b/internal/media/prune.go index 9e73fd2a3..aeddcef4c 100644 --- a/internal/media/prune.go +++ b/internal/media/prune.go @@ -320,7 +320,7 @@ func (m *manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.Med } // Delete attachment completely. - return m.state.DB.DeleteByID(ctx, attachment.ID, attachment) + return m.state.DB.DeleteAttachment(ctx, attachment.ID) } func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { @@ -332,7 +332,7 @@ func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.Me attachment.UpdatedAt = time.Now() cached := false attachment.Cached = &cached - return m.state.DB.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") + return m.state.DB.UpdateAttachment(ctx, attachment, "updated_at", "cached") } func (m *manager) removeFiles(ctx context.Context, keys ...string) (int, error) { diff --git a/internal/processing/instance.go b/internal/processing/instance.go index 3ca807af3..944b62a91 100644 --- a/internal/processing/instance.go +++ b/internal/processing/instance.go @@ -233,7 +233,7 @@ func (p *Processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe } else if form.AvatarDescription != nil && ia.AvatarMediaAttachment != nil { // process just the description for the existing avatar ia.AvatarMediaAttachment.Description = *form.AvatarDescription - if err := p.state.DB.UpdateByID(ctx, ia.AvatarMediaAttachment, ia.AvatarMediaAttachmentID, "description"); err != nil { + if err := p.state.DB.UpdateAttachment(ctx, ia.AvatarMediaAttachment, "description"); err != nil { return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error updating instance avatar description: %s", err)) } } diff --git a/internal/processing/media/delete.go b/internal/processing/media/delete.go index 02bd6cd0d..c5a398094 100644 --- a/internal/processing/media/delete.go +++ b/internal/processing/media/delete.go @@ -40,7 +40,7 @@ func (p *Processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr } // delete the attachment - if err := p.state.DB.DeleteByID(ctx, mediaAttachmentID, attachment); err != nil && !errors.Is(err, db.ErrNoEntries) { + if err := p.state.DB.DeleteAttachment(ctx, mediaAttachmentID); err != nil && !errors.Is(err, db.ErrNoEntries) { errs = append(errs, fmt.Sprintf("remove attachment: %s", err)) } diff --git a/internal/processing/media/unattach.go b/internal/processing/media/unattach.go index 7c6f7dbac..8db602239 100644 --- a/internal/processing/media/unattach.go +++ b/internal/processing/media/unattach.go @@ -49,7 +49,7 @@ func (p *Processor) Unattach(ctx context.Context, account *gtsmodel.Account, med attachment.UpdatedAt = time.Now() attachment.StatusID = "" - if err := p.state.DB.UpdateByID(ctx, attachment, attachment.ID, updatingColumns...); err != nil { + if err := p.state.DB.UpdateAttachment(ctx, attachment, updatingColumns...); err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("db error updating attachment: %s", err)) } diff --git a/test/envparsing.sh b/test/envparsing.sh index 603dcadca..053ac4644 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":true,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-admin-groups":["steamy"],"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","request-id-header":"X-Trace-Id","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":30000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":30000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":30000000000,"emoji-ttl":300000000000,"media-max-size":500,"media-sweep-freq":30000000000,"media-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":30000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":30000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":30000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":30000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":30000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":30000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":true,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-admin-groups":["steamy"],"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","request-id-header":"X-Trace-Id","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic diff --git a/testrig/config.go b/testrig/config.go index 140d73e6a..d470c8703 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -34,7 +34,7 @@ func InitTestConfig() { } var testDefaults = config.Configuration{ - LogLevel: "trace", + LogLevel: "info", LogDbQueries: true, ApplicationName: "gotosocial", LandingPageUser: "",