[feature/performance] Store account stats in separate table (#2831)

* [feature/performance] Store account stats in separate table, get stats from remote

* test account stats

* add some missing increment / decrement calls

* change stats function signatures

* rejig logging a bit

* use lock when updating stats

internal/cache/cache.go

@@ -52,9 +52,9 @@ func (c *Caches) Init() {
 	log.Infof(nil, "init: %p", c)
 
 	c.initAccount()
-	c.initAccountCounts()
 	c.initAccountNote()
 	c.initAccountSettings()
+	c.initAccountStats()
 	c.initApplication()
 	c.initBlock()
 	c.initBlockIDs()
@@ -124,6 +124,7 @@ func (c *Caches) Sweep(threshold float64) {
 	c.GTS.Account.Trim(threshold)
 	c.GTS.AccountNote.Trim(threshold)
 	c.GTS.AccountSettings.Trim(threshold)
+	c.GTS.AccountStats.Trim(threshold)
 	c.GTS.Block.Trim(threshold)
 	c.GTS.BlockIDs.Trim(threshold)
 	c.GTS.Emoji.Trim(threshold)

internal/cache/db.go

@@ -20,7 +20,6 @@ package cache
 import (
 	"time"
 
-	"codeberg.org/gruf/go-cache/v3/simple"
 	"codeberg.org/gruf/go-cache/v3/ttl"
 	"codeberg.org/gruf/go-structr"
 	"github.com/superseriousbusiness/gotosocial/internal/cache/domain"
@@ -36,16 +35,12 @@ type GTSCaches struct {
 	// AccountNote provides access to the gtsmodel Note database cache.
 	AccountNote StructCache[*gtsmodel.AccountNote]
 
-	// TEMPORARY CACHE TO ALLEVIATE SLOW COUNT QUERIES,
-	// (in time will be removed when these IDs are cached).
-	AccountCounts *simple.Cache[string, struct {
-		Statuses int
-		Pinned   int
-	}]
-
 	// AccountSettings provides access to the gtsmodel AccountSettings database cache.
 	AccountSettings StructCache[*gtsmodel.AccountSettings]
 
+	// AccountStats provides access to the gtsmodel AccountStats database cache.
+	AccountStats StructCache[*gtsmodel.AccountStats]
+
 	// Application provides access to the gtsmodel Application database cache.
 	Application StructCache[*gtsmodel.Application]
 
@@ -200,6 +195,7 @@ func (c *Caches) initAccount() {
 		a2.AlsoKnownAs = nil
 		a2.Move = nil
 		a2.Settings = nil
+		a2.Stats = nil
 
 		return a2
 	}
@@ -223,22 +219,6 @@ func (c *Caches) initAccount() {
 	})
 }
 
-func (c *Caches) initAccountCounts() {
-	// Simply use size of accounts cache,
-	// as this cache will be very small.
-	cap := c.GTS.Account.Cap()
-	if cap == 0 {
-		panic("must be initialized before accounts")
-	}
-
-	log.Infof(nil, "cache size = %d", cap)
-
-	c.GTS.AccountCounts = simple.New[string, struct {
-		Statuses int
-		Pinned   int
-	}](0, cap)
-}
-
 func (c *Caches) initAccountNote() {
 	// Calculate maximum cache size.
 	cap := calculateResultCacheMax(
@@ -295,6 +275,29 @@ func (c *Caches) initAccountSettings() {
 	})
 }
 
+func (c *Caches) initAccountStats() {
+	// Calculate maximum cache size.
+	cap := calculateResultCacheMax(
+		sizeofAccountStats(), // model in-mem size.
+		config.GetCacheAccountStatsMemRatio(),
+	)
+
+	log.Infof(nil, "cache size = %d", cap)
+
+	c.GTS.AccountStats.Init(structr.CacheConfig[*gtsmodel.AccountStats]{
+		Indices: []structr.IndexConfig{
+			{Fields: "AccountID"},
+		},
+		MaxSize:   cap,
+		IgnoreErr: ignoreErrors,
+		Copy: func(s1 *gtsmodel.AccountStats) *gtsmodel.AccountStats {
+			s2 := new(gtsmodel.AccountStats)
+			*s2 = *s1
+			return s2
+		},
+	})
+}
+
 func (c *Caches) initApplication() {
 	// Calculate maximum cache size.
 	cap := calculateResultCacheMax(

internal/cache/invalidate.go

@@ -27,8 +27,8 @@ import (
 // HOOKS TO BE CALLED ON DELETE YOU MUST FIRST POPULATE IT IN THE CACHE.
 
 func (c *Caches) OnInvalidateAccount(account *gtsmodel.Account) {
-	// Invalidate status counts for this account.
-	c.GTS.AccountCounts.Invalidate(account.ID)
+	// Invalidate stats for this account.
+	c.GTS.AccountStats.Invalidate("AccountID", account.ID)
 
 	// Invalidate account ID cached visibility.
 	c.Visibility.Invalidate("ItemID", account.ID)
@@ -168,8 +168,8 @@ func (c *Caches) OnInvalidatePollVote(vote *gtsmodel.PollVote) {
 }
 
 func (c *Caches) OnInvalidateStatus(status *gtsmodel.Status) {
-	// Invalidate status counts for this account.
-	c.GTS.AccountCounts.Invalidate(status.AccountID)
+	// Invalidate stats for this account.
+	c.GTS.AccountStats.Invalidate("AccountID", status.AccountID)
 
 	// Invalidate status ID cached visibility.
 	c.Visibility.Invalidate("ItemID", status.ID)

internal/cache/size.go

@@ -264,6 +264,17 @@ func sizeofAccountSettings() uintptr {
 	}))
 }
 
+func sizeofAccountStats() uintptr {
+	return uintptr(size.Of(&gtsmodel.AccountStats{
+		AccountID:           exampleID,
+		FollowersCount:      util.Ptr(100),
+		FollowingCount:      util.Ptr(100),
+		StatusesCount:       util.Ptr(100),
+		StatusesPinnedCount: util.Ptr(100),
+		LastStatusAt:        exampleTime,
+	}))
+}
+
 func sizeofApplication() uintptr {
 	return uintptr(size.Of(&gtsmodel.Application{
 		ID:           exampleID,