SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

[performance] cache more database calls, reduce required database calls overall (#3290)

Browse Source 
* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name
This commit is contained in:
kim
2024-09-16 16:46:09 +00:00
committed by GitHub
parent 0567b319c6
commit 84279f6a6a
68 changed files with 1683 additions and 2135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
internal/cache/db.go vendored
View File

@@ -29,9 +29,6 @@ type DBCaches struct {
// Account provides access to the gtsmodel Account database cache.
Account StructCache[*gtsmodel.Account]
// AccountIDsFollowingTag caches account IDs following a given tag ID.
AccountIDsFollowingTag SliceCache[string]
// AccountNote provides access to the gtsmodel Note database cache.
AccountNote StructCache[*gtsmodel.AccountNote]
@@ -88,10 +85,23 @@ type DBCaches struct {
// FollowIDs provides access to the follower / following IDs database cache.
// THIS CACHE IS KEYED AS THE FOLLOWING {prefix}{accountID} WHERE PREFIX IS:
// - '>' for following IDs
// - 'l>' for local following IDs
// - '<' for follower IDs
// - 'l<' for local follower IDs
//
// - '>{$accountID}' for following IDs
// e.g. FollowIDs.Load(">" + account.ID, func() {})
// which will load a slice of follows IDs FROM account.
//
// - 'l>{$accountID}' for local following IDs
// e.g. FollowIDs.Load("l>" + account.ID, func() {})
// which will load a slice of LOCAL follows IDs FROM account.
//
// - '<{$accountID}' for follower IDs
// e.g. FollowIDs.Load("<" + account.ID, func() {})
// which will load a slice of follows IDs TARGETTING account.
//
// - 'l<{$accountID}' for local follower IDs
// e.g. FollowIDs.Load("l<" + account.ID, func() {})
// which will load a slice of LOCAL follows IDs TARGETTING account.
//
FollowIDs SliceCache[string]
// FollowRequest provides access to the gtsmodel FollowRequest database cache.
@@ -99,10 +109,30 @@ type DBCaches struct {
// FollowRequestIDs provides access to the follow requester / requesting IDs database
// cache. THIS CACHE IS KEYED AS THE FOLLOWING {prefix}{accountID} WHERE PREFIX IS:
// - '>' for following IDs
// - '<' for follower IDs
//
// - '>{$accountID}' for follow request IDs
// e.g. FollowRequestIDs.Load(">" + account.ID, func() {})
// which will load a slice of follow request IDs TARGETTING account.
//
// - '<{$accountID}' for follow request IDs
// e.g. FollowRequestIDs.Load("<" + account.ID, func() {})
// which will load a slice of follow request IDs FROM account.
//
FollowRequestIDs SliceCache[string]
// FollowingTagIDs provides access to account IDs following / tag IDs followed by
// account db cache. THIS CACHE IS KEYED AS THE FOLLOWING {prefix}{id} WHERE:
//
// - '>{$accountID}' for tag IDs followed by account
// e.g. FollowingTagIDs.Load(">" + account.ID, func() {})
// which will load a slice of tag IDs followed by account.
//
// - '<{$tagIDs}' for account IDs following tag
// e.g. FollowingTagIDs.Load("<" + tag.ID, func() {})
// which will load a slice of account IDs following tag.
//
FollowingTagIDs SliceCache[string]
// Instance provides access to the gtsmodel Instance database cache.
Instance StructCache[*gtsmodel.Instance]
@@ -115,8 +145,31 @@ type DBCaches struct {
// List provides access to the gtsmodel List database cache.
List StructCache[*gtsmodel.List]
// ListEntry provides access to the gtsmodel ListEntry database cache.
ListEntry StructCache[*gtsmodel.ListEntry]
// ListIDs provides access to the list IDs owned by account / list IDs follow
// contained in db cache. THIS CACHE IS KEYED AS FOLLOWING {prefix}{id} WHERE:
//
// - 'a{$accountID}' for list IDs owned by account
// e.g. ListIDs.Load("a" + account.ID, func() {})
// which will load a slice of list IDs owned by account.
//
// - 'f{$followID}' for list IDs follow contained in
// e.g. ListIDs.Load("f" + follow.ID, func() {})
// which will load a slice of list IDs containing follow.
//
ListIDs SliceCache[string]
// ListedIDs provides access to the account IDs in list / follow IDs in
// list db cache. THIS CACHE IS KEYED AS FOLLOWING {prefix}{id} WHERE:
//
// - 'a{listID}' for account IDs in list ID
// e.g. ListedIDs.Load("a" + list.ID, func() {})
// which will load a slice of account IDs in list.
//
// - 'f{listID}' for follow IDs in list ID
// e.g. ListedIDs.Load("f" + list.ID, func() {})
// which will load a slice of follow IDs in list.
//
ListedIDs SliceCache[string]
// Marker provides access to the gtsmodel Marker database cache.
Marker StructCache[*gtsmodel.Marker]
@@ -151,10 +204,10 @@ type DBCaches struct {
// Status provides access to the gtsmodel Status database cache.
Status StructCache[*gtsmodel.Status]
// StatusBookmark ...
// StatusBookmark provides access to the gtsmodel StatusBookmark database cache.
StatusBookmark StructCache[*gtsmodel.StatusBookmark]
// StatusBookmarkIDs ...
// StatusBookmarkIDs provides access to the status bookmark IDs list database cache.
StatusBookmarkIDs SliceCache[string]
// StatusFave provides access to the gtsmodel StatusFave database cache.
@@ -166,9 +219,6 @@ type DBCaches struct {
// Tag provides access to the gtsmodel Tag database cache.
Tag StructCache[*gtsmodel.Tag]
// TagIDsFollowedByAccount caches tag IDs followed by a given account ID.
TagIDsFollowedByAccount SliceCache[string]
// ThreadMute provides access to the gtsmodel ThreadMute database cache.
ThreadMute StructCache[*gtsmodel.ThreadMute]
@@ -243,17 +293,6 @@ func (c *Caches) initAccount() {
})
}
func (c *Caches) initAccountIDsFollowingTag() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheAccountIDsFollowingTagMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
c.DB.AccountIDsFollowingTag.Init(0, cap)
}
func (c *Caches) initAccountNote() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
@@ -761,6 +800,17 @@ func (c *Caches) initFollowRequestIDs() {
c.DB.FollowRequestIDs.Init(0, cap)
}
func (c *Caches) initFollowingTagIDs() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheFollowingTagIDsMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
c.DB.FollowingTagIDs.Init(0, cap)
}
func (c *Caches) initInReplyToIDs() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
@@ -860,7 +910,6 @@ func (c *Caches) initList() {
// will be populated separately.
// See internal/db/bundb/list.go.
l2.Account = nil
l2.ListEntries = nil
return l2
}
@@ -876,37 +925,26 @@ func (c *Caches) initList() {
})
}
func (c *Caches) initListEntry() {
func (c *Caches) initListIDs() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
sizeofListEntry(), // model in-mem size.
config.GetCacheListEntryMemRatio(),
cap := calculateSliceCacheMax(
config.GetCacheListIDsMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
copyF := func(l1 *gtsmodel.ListEntry) *gtsmodel.ListEntry {
l2 := new(gtsmodel.ListEntry)
*l2 = *l1
c.DB.ListIDs.Init(0, cap)
}
// Don't include ptr fields that
// will be populated separately.
// See internal/db/bundb/list.go.
l2.Follow = nil
func (c *Caches) initListedIDs() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheListedIDsMemRatio(),
)
return l2
}
log.Infof(nil, "cache size = %d", cap)
c.DB.ListEntry.Init(structr.CacheConfig[*gtsmodel.ListEntry]{
Indices: []structr.IndexConfig{
{Fields: "ID"},
{Fields: "ListID", Multiple: true},
{Fields: "FollowID", Multiple: true},
},
MaxSize: cap,
IgnoreErr: ignoreErrors,
Copy: copyF,
})
c.DB.ListedIDs.Init(0, cap)
}
func (c *Caches) initMarker() {
@@ -1368,17 +1406,6 @@ func (c *Caches) initTag() {
})
}
func (c *Caches) initTagIDsFollowedByAccount() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheTagIDsFollowedByAccountMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
c.DB.TagIDsFollowedByAccount.Init(0, cap)
}
func (c *Caches) initThreadMute() {
cap := calculateResultCacheMax(
sizeofThreadMute(), // model in-mem size.