SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

[feature] Prune timelines once per hour to plug memory leak (#1117)

Browse Source 
* export highest/lowest ULIDs as proper const

* add stop + start to timeline manager, other small fixes

* unexport unused interface funcs + tidy up

* add LastGot func

* add timeline Prune function

* test prune

* update lastGot
This commit is contained in:
tobi
2022-11-22 19:38:10 +01:00
committed by GitHub
parent 90bbcf1bcf
commit 50dc179d33
16 changed files with 594 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
internal/timeline/get.go
View File

@@ -23,6 +23,7 @@ import (
"context"
"errors"
"fmt"
"time"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -30,16 +31,27 @@ import (
const retries = 5
func (t *timeline) LastGot() time.Time {
t.Lock()
defer t.Unlock()
return t.lastGot
}
func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error) {
l := log.WithFields(kv.Fields{
{"accountID", t.accountID},
{"amount", amount},
{"maxID", maxID},
{"sinceID", sinceID},
{"minID", minID},
}...)
l.Debug("entering get")
l.Debug("entering get and updating t.lastGot")
// regardless of what happens below, update the
// last time Get was called for this timeline
t.Lock()
t.lastGot = time.Now()
t.Unlock()
var items []Preparable
var err error
@@ -47,7 +59,7 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
// no params are defined to just fetch from the top
// this is equivalent to a user asking for the top x items from their timeline
if maxID == "" && sinceID == "" && minID == "" {
items, err = t.GetXFromTop(ctx, amount)
items, err = t.getXFromTop(ctx, amount)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
@@ -67,7 +79,7 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
// this is equivalent to a user asking for the next x items from their timeline, starting from maxID
if maxID != "" && sinceID == "" {
attempts := 0
items, err = t.GetXBehindID(ctx, amount, maxID, &attempts)
items, err = t.getXBehindID(ctx, amount, maxID, &attempts)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
@@ -86,25 +98,26 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
// maxID is defined and sinceID || minID are as well, so take a slice between them
// this is equivalent to a user asking for items older than x but newer than y
if maxID != "" && sinceID != "" {
items, err = t.GetXBetweenID(ctx, amount, maxID, minID)
items, err = t.getXBetweenID(ctx, amount, maxID, minID)
}
if maxID != "" && minID != "" {
items, err = t.GetXBetweenID(ctx, amount, maxID, minID)
items, err = t.getXBetweenID(ctx, amount, maxID, minID)
}
// maxID isn't defined, but sinceID || minID are, so take x before
// this is equivalent to a user asking for items newer than x (eg., refreshing the top of their timeline)
if maxID == "" && sinceID != "" {
items, err = t.GetXBeforeID(ctx, amount, sinceID, true)
items, err = t.getXBeforeID(ctx, amount, sinceID, true)
}
if maxID == "" && minID != "" {
items, err = t.GetXBeforeID(ctx, amount, minID, true)
items, err = t.getXBeforeID(ctx, amount, minID, true)
}
return items, err
}
func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]Preparable, error) {
// getXFromTop returns x amount of items from the top of the timeline, from newest to oldest.
func (t *timeline) getXFromTop(ctx context.Context, amount int) ([]Preparable, error) {
// make a slice of preparedItems with the length we need to return
preparedItems := make([]Preparable, 0, amount)
@@ -124,7 +137,7 @@ func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]Preparable, e
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXFromTop: could not parse e as a preparedItemsEntry")
return nil, errors.New("getXFromTop: could not parse e as a preparedItemsEntry")
}
preparedItems = append(preparedItems, entry.prepared)
served++
@@ -136,9 +149,12 @@ func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]Preparable, e
return preparedItems, nil
}
func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]Preparable, error) {
// getXBehindID returns x amount of items from the given id onwards, from newest to oldest.
// This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?max_id=WHATEVER
func (t *timeline) getXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]Preparable, error) {
l := log.WithFields(kv.Fields{
{"amount", amount},
{"behindID", behindID},
{"attempts", attempts},
@@ -164,7 +180,7 @@ findMarkLoop:
position++
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBehindID: could not parse e as a preparedPostsEntry")
}
if entry.itemID <= behindID {
@@ -177,10 +193,10 @@ findMarkLoop:
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
// this can happen when a user asks for really old items
if behindIDMark == nil {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
if err := t.prepareBehind(ctx, behindID, amount); err != nil {
return nil, fmt.Errorf("getXBehindID: error preparing behind and including ID %s", behindID)
}
oldestID, err := t.OldestPreparedItemID(ctx)
oldestID, err := t.oldestPreparedItemID(ctx)
if err != nil {
return nil, err
}
@@ -196,13 +212,13 @@ findMarkLoop:
l.Tracef("exceeded retries looking for behindID %s", behindID)
return items, nil
}
l.Trace("trying GetXBehindID again")
return t.GetXBehindID(ctx, amount, behindID, attempts)
l.Trace("trying getXBehindID again")
return t.getXBehindID(ctx, amount, behindID, attempts)
}
// make sure we have enough items prepared behind it to return what we're being asked for
if t.preparedItems.data.Len() < amount+position {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
if err := t.prepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
}
@@ -213,7 +229,7 @@ serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBehindID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
@@ -227,7 +243,11 @@ serveloop:
return items, nil
}
func (t *timeline) GetXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]Preparable, error) {
// getXBeforeID returns x amount of items up to the given id, from newest to oldest.
// This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER
func (t *timeline) getXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]Preparable, error) {
// make a slice of items with the length we need to return
items := make([]Preparable, 0, amount)
@@ -241,7 +261,7 @@ findMarkLoop:
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBeforeID: could not parse e as a preparedPostsEntry")
}
if entry.itemID >= beforeID {
@@ -263,7 +283,7 @@ findMarkLoop:
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBeforeID: could not parse e as a preparedPostsEntry")
}
if entry.itemID == beforeID {
@@ -283,7 +303,7 @@ findMarkLoop:
for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBeforeID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
@@ -298,7 +318,11 @@ findMarkLoop:
return items, nil
}
func (t *timeline) GetXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]Preparable, error) {
// getXBetweenID returns x amount of items from the given maxID, up to the given id, from newest to oldest.
// This will NOT include the item with the given IDs.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
func (t *timeline) getXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]Preparable, error) {
// make a slice of items with the length we need to return
items := make([]Preparable, 0, amount)
@@ -314,7 +338,7 @@ findMarkLoop:
position++
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBetweenID: could not parse e as a preparedPostsEntry")
}
if entry.itemID == behindID {
@@ -325,12 +349,12 @@ findMarkLoop:
// we didn't find it
if behindIDMark == nil {
return nil, fmt.Errorf("GetXBetweenID: couldn't find item with ID %s", behindID)
return nil, fmt.Errorf("getXBetweenID: couldn't find item with ID %s", behindID)
}
// make sure we have enough items prepared behind it to return what we're being asked for
if t.preparedItems.data.Len() < amount+position {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
if err := t.prepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
}
@@ -341,7 +365,7 @@ serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
return nil, errors.New("getXBetweenID: could not parse e as a preparedPostsEntry")
}
if entry.itemID == beforeID {