mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2025-06-05 21:59:39 +02:00
[performance] overhaul struct (+ result) caching library for simplicity, performance and multiple-result lookups (#2535)
* rewrite cache library as codeberg.org/gruf/go-structr, implement in gotosocial
* use actual go-structr release version (not just commit hash)
* revert go toolchain changes (damn you go for auto changing this)
* fix go mod woes
* ensure %w is used in calls to errs.Appendf()
* fix error checking
* fix possible panic
* remove unnecessary start/stop functions, move to main Cache{} struct, add note regarding which caches require start/stop
* fix copy-paste artifact... 😇
* fix all comment copy-paste artifacts
* remove dropID() function, now we can just use slices.DeleteFunc()
* use util.Deduplicate() instead of collate(), move collate to util
* move orderByIDs() to util package and "generify"
* add a util.DeleteIf() function, use this to delete entries on failed population
* use slices.DeleteFunc() instead of util.DeleteIf() (i had the logic mixed up in my head somehow lol)
* add note about how collate differs from deduplicate
This commit is contained in:
kim
GitHub
278
internal/cache/cache.go
vendored
278
internal/cache/cache.go
vendored
@@ -18,8 +18,9 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/cache/headerfilter"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
)
|
||||
|
||||
@@ -49,198 +50,59 @@ type Caches struct {
|
||||
func (c *Caches) Init() {
|
||||
log.Infof(nil, "init: %p", c)
|
||||
|
||||
c.GTS.Init()
|
||||
c.Visibility.Init()
|
||||
|
||||
// Setup cache invalidate hooks.
|
||||
// !! READ THE METHOD COMMENT
|
||||
c.setuphooks()
|
||||
c.initAccount()
|
||||
c.initAccountNote()
|
||||
c.initApplication()
|
||||
c.initBlock()
|
||||
c.initBlockIDs()
|
||||
c.initBoostOfIDs()
|
||||
c.initDomainAllow()
|
||||
c.initDomainBlock()
|
||||
c.initEmoji()
|
||||
c.initEmojiCategory()
|
||||
c.initFollow()
|
||||
c.initFollowIDs()
|
||||
c.initFollowRequest()
|
||||
c.initFollowRequestIDs()
|
||||
c.initInReplyToIDs()
|
||||
c.initInstance()
|
||||
c.initList()
|
||||
c.initListEntry()
|
||||
c.initMarker()
|
||||
c.initMedia()
|
||||
c.initMention()
|
||||
c.initNotification()
|
||||
c.initPoll()
|
||||
c.initPollVote()
|
||||
c.initPollVoteIDs()
|
||||
c.initReport()
|
||||
c.initStatus()
|
||||
c.initStatusFave()
|
||||
c.initTag()
|
||||
c.initThreadMute()
|
||||
c.initStatusFaveIDs()
|
||||
c.initTombstone()
|
||||
c.initUser()
|
||||
c.initWebfinger()
|
||||
c.initVisibility()
|
||||
}
|
||||
|
||||
// Start will start both the GTS and AP cache collections.
|
||||
// Start will start any caches that require a background
|
||||
// routine, which usually means any kind of TTL caches.
|
||||
func (c *Caches) Start() {
|
||||
log.Infof(nil, "start: %p", c)
|
||||
|
||||
c.GTS.Start()
|
||||
c.Visibility.Start()
|
||||
tryUntil("starting *gtsmodel.Webfinger cache", 5, func() bool {
|
||||
return c.GTS.Webfinger.Start(5 * time.Minute)
|
||||
})
|
||||
}
|
||||
|
||||
// Stop will stop both the GTS and AP cache collections.
|
||||
// Stop will stop any caches that require a background
|
||||
// routine, which usually means any kind of TTL caches.
|
||||
func (c *Caches) Stop() {
|
||||
log.Infof(nil, "stop: %p", c)
|
||||
|
||||
c.GTS.Stop()
|
||||
c.Visibility.Stop()
|
||||
}
|
||||
|
||||
// setuphooks sets necessary cache invalidation hooks between caches,
|
||||
// as an invalidation indicates a database INSERT / UPDATE / DELETE.
|
||||
// NOTE THEY ARE ONLY CALLED WHEN THE ITEM IS IN THE CACHE, SO FOR
|
||||
// HOOKS TO BE CALLED ON DELETE YOU MUST FIRST POPULATE IT IN THE CACHE.
|
||||
func (c *Caches) setuphooks() {
|
||||
c.GTS.Account().SetInvalidateCallback(func(account *gtsmodel.Account) {
|
||||
// Invalidate account ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", account.ID)
|
||||
c.Visibility.Invalidate("RequesterID", account.ID)
|
||||
|
||||
// Invalidate this account's
|
||||
// following / follower lists.
|
||||
// (see FollowIDs() comment for details).
|
||||
c.GTS.FollowIDs().InvalidateAll(
|
||||
">"+account.ID,
|
||||
"l>"+account.ID,
|
||||
"<"+account.ID,
|
||||
"l<"+account.ID,
|
||||
)
|
||||
|
||||
// Invalidate this account's
|
||||
// follow requesting / request lists.
|
||||
// (see FollowRequestIDs() comment for details).
|
||||
c.GTS.FollowRequestIDs().InvalidateAll(
|
||||
">"+account.ID,
|
||||
"<"+account.ID,
|
||||
)
|
||||
|
||||
// Invalidate this account's block lists.
|
||||
c.GTS.BlockIDs().Invalidate(account.ID)
|
||||
})
|
||||
|
||||
c.GTS.Block().SetInvalidateCallback(func(block *gtsmodel.Block) {
|
||||
// Invalidate block origin account ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", block.AccountID)
|
||||
c.Visibility.Invalidate("RequesterID", block.AccountID)
|
||||
|
||||
// Invalidate block target account ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", block.TargetAccountID)
|
||||
c.Visibility.Invalidate("RequesterID", block.TargetAccountID)
|
||||
|
||||
// Invalidate source account's block lists.
|
||||
c.GTS.BlockIDs().Invalidate(block.AccountID)
|
||||
})
|
||||
|
||||
c.GTS.EmojiCategory().SetInvalidateCallback(func(category *gtsmodel.EmojiCategory) {
|
||||
// Invalidate any emoji in this category.
|
||||
c.GTS.Emoji().Invalidate("CategoryID", category.ID)
|
||||
})
|
||||
|
||||
c.GTS.Follow().SetInvalidateCallback(func(follow *gtsmodel.Follow) {
|
||||
// Invalidate follow request with this same ID.
|
||||
c.GTS.FollowRequest().Invalidate("ID", follow.ID)
|
||||
|
||||
// Invalidate any related list entries.
|
||||
c.GTS.ListEntry().Invalidate("FollowID", follow.ID)
|
||||
|
||||
// Invalidate follow origin account ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", follow.AccountID)
|
||||
c.Visibility.Invalidate("RequesterID", follow.AccountID)
|
||||
|
||||
// Invalidate follow target account ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", follow.TargetAccountID)
|
||||
c.Visibility.Invalidate("RequesterID", follow.TargetAccountID)
|
||||
|
||||
// Invalidate source account's following
|
||||
// lists, and destination's follwer lists.
|
||||
// (see FollowIDs() comment for details).
|
||||
c.GTS.FollowIDs().InvalidateAll(
|
||||
">"+follow.AccountID,
|
||||
"l>"+follow.AccountID,
|
||||
"<"+follow.AccountID,
|
||||
"l<"+follow.AccountID,
|
||||
"<"+follow.TargetAccountID,
|
||||
"l<"+follow.TargetAccountID,
|
||||
">"+follow.TargetAccountID,
|
||||
"l>"+follow.TargetAccountID,
|
||||
)
|
||||
})
|
||||
|
||||
c.GTS.FollowRequest().SetInvalidateCallback(func(followReq *gtsmodel.FollowRequest) {
|
||||
// Invalidate follow with this same ID.
|
||||
c.GTS.Follow().Invalidate("ID", followReq.ID)
|
||||
|
||||
// Invalidate source account's followreq
|
||||
// lists, and destinations follow req lists.
|
||||
// (see FollowRequestIDs() comment for details).
|
||||
c.GTS.FollowRequestIDs().InvalidateAll(
|
||||
">"+followReq.AccountID,
|
||||
"<"+followReq.AccountID,
|
||||
">"+followReq.TargetAccountID,
|
||||
"<"+followReq.TargetAccountID,
|
||||
)
|
||||
})
|
||||
|
||||
c.GTS.List().SetInvalidateCallback(func(list *gtsmodel.List) {
|
||||
// Invalidate all cached entries of this list.
|
||||
c.GTS.ListEntry().Invalidate("ListID", list.ID)
|
||||
})
|
||||
|
||||
c.GTS.Media().SetInvalidateCallback(func(media *gtsmodel.MediaAttachment) {
|
||||
if *media.Avatar || *media.Header {
|
||||
// Invalidate cache of attaching account.
|
||||
c.GTS.Account().Invalidate("ID", media.AccountID)
|
||||
}
|
||||
|
||||
if media.StatusID != "" {
|
||||
// Invalidate cache of attaching status.
|
||||
c.GTS.Status().Invalidate("ID", media.StatusID)
|
||||
}
|
||||
})
|
||||
|
||||
c.GTS.Poll().SetInvalidateCallback(func(poll *gtsmodel.Poll) {
|
||||
// Invalidate all cached votes of this poll.
|
||||
c.GTS.PollVote().Invalidate("PollID", poll.ID)
|
||||
|
||||
// Invalidate cache of poll vote IDs.
|
||||
c.GTS.PollVoteIDs().Invalidate(poll.ID)
|
||||
})
|
||||
|
||||
c.GTS.PollVote().SetInvalidateCallback(func(vote *gtsmodel.PollVote) {
|
||||
// Invalidate cached poll (contains no. votes).
|
||||
c.GTS.Poll().Invalidate("ID", vote.PollID)
|
||||
|
||||
// Invalidate cache of poll vote IDs.
|
||||
c.GTS.PollVoteIDs().Invalidate(vote.PollID)
|
||||
})
|
||||
|
||||
c.GTS.Status().SetInvalidateCallback(func(status *gtsmodel.Status) {
|
||||
// Invalidate status ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", status.ID)
|
||||
|
||||
for _, id := range status.AttachmentIDs {
|
||||
// Invalidate each media by the IDs we're aware of.
|
||||
// This must be done as the status table is aware of
|
||||
// the media IDs in use before the media table is
|
||||
// aware of the status ID they are linked to.
|
||||
//
|
||||
// c.GTS.Media().Invalidate("StatusID") will not work.
|
||||
c.GTS.Media().Invalidate("ID", id)
|
||||
}
|
||||
|
||||
if status.BoostOfID != "" {
|
||||
// Invalidate boost ID list of the original status.
|
||||
c.GTS.BoostOfIDs().Invalidate(status.BoostOfID)
|
||||
}
|
||||
|
||||
if status.InReplyToID != "" {
|
||||
// Invalidate in reply to ID list of original status.
|
||||
c.GTS.InReplyToIDs().Invalidate(status.InReplyToID)
|
||||
}
|
||||
|
||||
if status.PollID != "" {
|
||||
// Invalidate cache of attached poll ID.
|
||||
c.GTS.Poll().Invalidate("ID", status.PollID)
|
||||
}
|
||||
})
|
||||
|
||||
c.GTS.StatusFave().SetInvalidateCallback(func(fave *gtsmodel.StatusFave) {
|
||||
// Invalidate status fave ID list for this status.
|
||||
c.GTS.StatusFaveIDs().Invalidate(fave.StatusID)
|
||||
})
|
||||
|
||||
c.GTS.User().SetInvalidateCallback(func(user *gtsmodel.User) {
|
||||
// Invalidate local account ID cached visibility.
|
||||
c.Visibility.Invalidate("ItemID", user.AccountID)
|
||||
c.Visibility.Invalidate("RequesterID", user.AccountID)
|
||||
})
|
||||
tryUntil("stopping *gtsmodel.Webfinger cache", 5, c.GTS.Webfinger.Stop)
|
||||
}
|
||||
|
||||
// Sweep will sweep all the available caches to ensure none
|
||||
@@ -250,30 +112,30 @@ func (c *Caches) setuphooks() {
|
||||
// require an eviction on every single write, which adds
|
||||
// significant overhead to all cache writes.
|
||||
func (c *Caches) Sweep(threshold float64) {
|
||||
c.GTS.Account().Trim(threshold)
|
||||
c.GTS.AccountNote().Trim(threshold)
|
||||
c.GTS.Block().Trim(threshold)
|
||||
c.GTS.BlockIDs().Trim(threshold)
|
||||
c.GTS.Emoji().Trim(threshold)
|
||||
c.GTS.EmojiCategory().Trim(threshold)
|
||||
c.GTS.Follow().Trim(threshold)
|
||||
c.GTS.FollowIDs().Trim(threshold)
|
||||
c.GTS.FollowRequest().Trim(threshold)
|
||||
c.GTS.FollowRequestIDs().Trim(threshold)
|
||||
c.GTS.Instance().Trim(threshold)
|
||||
c.GTS.List().Trim(threshold)
|
||||
c.GTS.ListEntry().Trim(threshold)
|
||||
c.GTS.Marker().Trim(threshold)
|
||||
c.GTS.Media().Trim(threshold)
|
||||
c.GTS.Mention().Trim(threshold)
|
||||
c.GTS.Notification().Trim(threshold)
|
||||
c.GTS.Poll().Trim(threshold)
|
||||
c.GTS.Report().Trim(threshold)
|
||||
c.GTS.Status().Trim(threshold)
|
||||
c.GTS.StatusFave().Trim(threshold)
|
||||
c.GTS.Tag().Trim(threshold)
|
||||
c.GTS.ThreadMute().Trim(threshold)
|
||||
c.GTS.Tombstone().Trim(threshold)
|
||||
c.GTS.User().Trim(threshold)
|
||||
c.GTS.Account.Trim(threshold)
|
||||
c.GTS.AccountNote.Trim(threshold)
|
||||
c.GTS.Block.Trim(threshold)
|
||||
c.GTS.BlockIDs.Trim(threshold)
|
||||
c.GTS.Emoji.Trim(threshold)
|
||||
c.GTS.EmojiCategory.Trim(threshold)
|
||||
c.GTS.Follow.Trim(threshold)
|
||||
c.GTS.FollowIDs.Trim(threshold)
|
||||
c.GTS.FollowRequest.Trim(threshold)
|
||||
c.GTS.FollowRequestIDs.Trim(threshold)
|
||||
c.GTS.Instance.Trim(threshold)
|
||||
c.GTS.List.Trim(threshold)
|
||||
c.GTS.ListEntry.Trim(threshold)
|
||||
c.GTS.Marker.Trim(threshold)
|
||||
c.GTS.Media.Trim(threshold)
|
||||
c.GTS.Mention.Trim(threshold)
|
||||
c.GTS.Notification.Trim(threshold)
|
||||
c.GTS.Poll.Trim(threshold)
|
||||
c.GTS.Report.Trim(threshold)
|
||||
c.GTS.Status.Trim(threshold)
|
||||
c.GTS.StatusFave.Trim(threshold)
|
||||
c.GTS.Tag.Trim(threshold)
|
||||
c.GTS.ThreadMute.Trim(threshold)
|
||||
c.GTS.Tombstone.Trim(threshold)
|
||||
c.GTS.User.Trim(threshold)
|
||||
c.Visibility.Trim(threshold)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user