mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2025-06-05 21:59:39 +02:00
[feature] Prune timelines once per hour to plug memory leak (#1117)
* export highest/lowest ULIDs as proper const * add stop + start to timeline manager, other small fixes * unexport unused interface funcs + tidy up * add LastGot func * add timeline Prune function * test prune * update lastGot
This commit is contained in:
tobi
GitHub
@@ -26,154 +26,12 @@ import (
|
||||
|
||||
"codeberg.org/gruf/go-kv"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/id"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
)
|
||||
|
||||
func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID string, sinceID string, minID string) error {
|
||||
l := log.WithFields(kv.Fields{
|
||||
|
||||
{"amount", amount},
|
||||
{"maxID", maxID},
|
||||
{"sinceID", sinceID},
|
||||
{"minID", minID},
|
||||
}...)
|
||||
|
||||
var err error
|
||||
|
||||
// maxID is defined but sinceID isn't so take from behind
|
||||
if maxID != "" && sinceID == "" {
|
||||
l.Debug("preparing behind maxID")
|
||||
err = t.PrepareBehind(ctx, maxID, amount)
|
||||
}
|
||||
|
||||
// maxID isn't defined, but sinceID || minID are, so take x before
|
||||
if maxID == "" && sinceID != "" {
|
||||
l.Debug("preparing before sinceID")
|
||||
err = t.PrepareBefore(ctx, sinceID, false, amount)
|
||||
}
|
||||
if maxID == "" && minID != "" {
|
||||
l.Debug("preparing before minID")
|
||||
err = t.PrepareBefore(ctx, minID, false, amount)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *timeline) PrepareBehind(ctx context.Context, itemID string, amount int) error {
|
||||
// lazily initialize prepared items if it hasn't been done already
|
||||
if t.preparedItems.data == nil {
|
||||
t.preparedItems.data = &list.List{}
|
||||
t.preparedItems.data.Init()
|
||||
}
|
||||
|
||||
if err := t.IndexBehind(ctx, itemID, amount); err != nil {
|
||||
return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", itemID, err)
|
||||
}
|
||||
|
||||
// if the itemindex is nil, nothing has been indexed yet so there's nothing to prepare
|
||||
if t.itemIndex.data == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var prepared int
|
||||
var preparing bool
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
prepareloop:
|
||||
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
|
||||
entry, ok := e.Value.(*itemIndexEntry)
|
||||
if !ok {
|
||||
return errors.New("PrepareBehind: could not parse e as itemIndexEntry")
|
||||
}
|
||||
|
||||
if !preparing {
|
||||
// we haven't hit the position we need to prepare from yet
|
||||
if entry.itemID == itemID {
|
||||
preparing = true
|
||||
}
|
||||
}
|
||||
|
||||
if preparing {
|
||||
if err := t.prepare(ctx, entry.itemID); err != nil {
|
||||
// there's been an error
|
||||
if err != db.ErrNoEntries {
|
||||
// it's a real error
|
||||
return fmt.Errorf("PrepareBehind: error preparing item with id %s: %s", entry.itemID, err)
|
||||
}
|
||||
// the status just doesn't exist (anymore) so continue to the next one
|
||||
continue
|
||||
}
|
||||
if prepared == amount {
|
||||
// we're done
|
||||
break prepareloop
|
||||
}
|
||||
prepared++
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *timeline) PrepareBefore(ctx context.Context, statusID string, include bool, amount int) error {
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
|
||||
// lazily initialize prepared posts if it hasn't been done already
|
||||
if t.preparedItems.data == nil {
|
||||
t.preparedItems.data = &list.List{}
|
||||
t.preparedItems.data.Init()
|
||||
}
|
||||
|
||||
// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
|
||||
if t.itemIndex.data == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var prepared int
|
||||
var preparing bool
|
||||
prepareloop:
|
||||
for e := t.itemIndex.data.Back(); e != nil; e = e.Prev() {
|
||||
entry, ok := e.Value.(*itemIndexEntry)
|
||||
if !ok {
|
||||
return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
|
||||
}
|
||||
|
||||
if !preparing {
|
||||
// we haven't hit the position we need to prepare from yet
|
||||
if entry.itemID == statusID {
|
||||
preparing = true
|
||||
if !include {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if preparing {
|
||||
if err := t.prepare(ctx, entry.itemID); err != nil {
|
||||
// there's been an error
|
||||
if err != db.ErrNoEntries {
|
||||
// it's a real error
|
||||
return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.itemID, err)
|
||||
}
|
||||
// the status just doesn't exist (anymore) so continue to the next one
|
||||
continue
|
||||
}
|
||||
if prepared == amount {
|
||||
// we're done
|
||||
break prepareloop
|
||||
}
|
||||
prepared++
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
|
||||
l := log.WithFields(kv.Fields{
|
||||
|
||||
{"amount", amount},
|
||||
}...)
|
||||
l := log.WithFields(kv.Fields{{"amount", amount}}...)
|
||||
|
||||
// lazily initialize prepared posts if it hasn't been done already
|
||||
if t.preparedItems.data == nil {
|
||||
@@ -182,10 +40,10 @@ func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
|
||||
}
|
||||
|
||||
// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
|
||||
if t.itemIndex.data == nil {
|
||||
if t.indexedItems.data == nil {
|
||||
l.Debug("postindex.data was nil, indexing behind highest possible ID")
|
||||
if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", amount); err != nil {
|
||||
return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err)
|
||||
if err := t.indexBehind(ctx, id.Highest, amount); err != nil {
|
||||
return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", id.Highest, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,12 +52,12 @@ func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
|
||||
defer t.Unlock()
|
||||
var prepared int
|
||||
prepareloop:
|
||||
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
|
||||
for e := t.indexedItems.data.Front(); e != nil; e = e.Next() {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
entry, ok := e.Value.(*itemIndexEntry)
|
||||
entry, ok := e.Value.(*indexedItemsEntry)
|
||||
if !ok {
|
||||
return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
|
||||
}
|
||||
@@ -226,6 +84,142 @@ prepareloop:
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID string, sinceID string, minID string) error {
|
||||
l := log.WithFields(kv.Fields{
|
||||
{"amount", amount},
|
||||
{"maxID", maxID},
|
||||
{"sinceID", sinceID},
|
||||
{"minID", minID},
|
||||
}...)
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case maxID != "" && sinceID == "":
|
||||
l.Debug("preparing behind maxID")
|
||||
err = t.prepareBehind(ctx, maxID, amount)
|
||||
case maxID == "" && sinceID != "":
|
||||
l.Debug("preparing before sinceID")
|
||||
err = t.prepareBefore(ctx, sinceID, false, amount)
|
||||
case maxID == "" && minID != "":
|
||||
l.Debug("preparing before minID")
|
||||
err = t.prepareBefore(ctx, minID, false, amount)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// prepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
|
||||
// If include is true, then the given item ID will also be prepared, otherwise only entries behind it will be prepared.
|
||||
func (t *timeline) prepareBehind(ctx context.Context, itemID string, amount int) error {
|
||||
// lazily initialize prepared items if it hasn't been done already
|
||||
if t.preparedItems.data == nil {
|
||||
t.preparedItems.data = &list.List{}
|
||||
t.preparedItems.data.Init()
|
||||
}
|
||||
|
||||
if err := t.indexBehind(ctx, itemID, amount); err != nil {
|
||||
return fmt.Errorf("prepareBehind: error indexing behind id %s: %s", itemID, err)
|
||||
}
|
||||
|
||||
// if the itemindex is nil, nothing has been indexed yet so there's nothing to prepare
|
||||
if t.indexedItems.data == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var prepared int
|
||||
var preparing bool
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
prepareloop:
|
||||
for e := t.indexedItems.data.Front(); e != nil; e = e.Next() {
|
||||
entry, ok := e.Value.(*indexedItemsEntry)
|
||||
if !ok {
|
||||
return errors.New("prepareBehind: could not parse e as itemIndexEntry")
|
||||
}
|
||||
|
||||
if !preparing {
|
||||
// we haven't hit the position we need to prepare from yet
|
||||
if entry.itemID == itemID {
|
||||
preparing = true
|
||||
}
|
||||
}
|
||||
|
||||
if preparing {
|
||||
if err := t.prepare(ctx, entry.itemID); err != nil {
|
||||
// there's been an error
|
||||
if err != db.ErrNoEntries {
|
||||
// it's a real error
|
||||
return fmt.Errorf("prepareBehind: error preparing item with id %s: %s", entry.itemID, err)
|
||||
}
|
||||
// the status just doesn't exist (anymore) so continue to the next one
|
||||
continue
|
||||
}
|
||||
if prepared == amount {
|
||||
// we're done
|
||||
break prepareloop
|
||||
}
|
||||
prepared++
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *timeline) prepareBefore(ctx context.Context, statusID string, include bool, amount int) error {
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
|
||||
// lazily initialize prepared posts if it hasn't been done already
|
||||
if t.preparedItems.data == nil {
|
||||
t.preparedItems.data = &list.List{}
|
||||
t.preparedItems.data.Init()
|
||||
}
|
||||
|
||||
// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
|
||||
if t.indexedItems.data == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var prepared int
|
||||
var preparing bool
|
||||
prepareloop:
|
||||
for e := t.indexedItems.data.Back(); e != nil; e = e.Prev() {
|
||||
entry, ok := e.Value.(*indexedItemsEntry)
|
||||
if !ok {
|
||||
return errors.New("prepareBefore: could not parse e as a postIndexEntry")
|
||||
}
|
||||
|
||||
if !preparing {
|
||||
// we haven't hit the position we need to prepare from yet
|
||||
if entry.itemID == statusID {
|
||||
preparing = true
|
||||
if !include {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if preparing {
|
||||
if err := t.prepare(ctx, entry.itemID); err != nil {
|
||||
// there's been an error
|
||||
if err != db.ErrNoEntries {
|
||||
// it's a real error
|
||||
return fmt.Errorf("prepareBefore: error preparing status with id %s: %s", entry.itemID, err)
|
||||
}
|
||||
// the status just doesn't exist (anymore) so continue to the next one
|
||||
continue
|
||||
}
|
||||
if prepared == amount {
|
||||
// we're done
|
||||
break prepareloop
|
||||
}
|
||||
prepared++
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *timeline) prepare(ctx context.Context, itemID string) error {
|
||||
// trigger the caller-provided prepare function
|
||||
prepared, err := t.prepareFunction(ctx, t.accountID, itemID)
|
||||
@@ -245,7 +239,9 @@ func (t *timeline) prepare(ctx context.Context, itemID string) error {
|
||||
return t.preparedItems.insertPrepared(ctx, preparedItemsEntry)
|
||||
}
|
||||
|
||||
func (t *timeline) OldestPreparedItemID(ctx context.Context) (string, error) {
|
||||
// oldestPreparedItemID returns the id of the rearmost (ie., the oldest) prepared item, or an error if something goes wrong.
|
||||
// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
|
||||
func (t *timeline) oldestPreparedItemID(ctx context.Context) (string, error) {
|
||||
var id string
|
||||
if t.preparedItems == nil || t.preparedItems.data == nil {
|
||||
// return an empty string if prepared items hasn't been initialized yet
|
||||
@@ -260,7 +256,7 @@ func (t *timeline) OldestPreparedItemID(ctx context.Context) (string, error) {
|
||||
|
||||
entry, ok := e.Value.(*preparedItemsEntry)
|
||||
if !ok {
|
||||
return id, errors.New("OldestPreparedItemID: could not parse e as a preparedItemsEntry")
|
||||
return id, errors.New("oldestPreparedItemID: could not parse e as a preparedItemsEntry")
|
||||
}
|
||||
|
||||
return entry.itemID, nil
|
||||
|
||||
Reference in New Issue
Block a user