From 1b36e858406ff6b15217229d1abaaabdbeec24e8 Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Sat, 5 Feb 2022 12:47:38 +0100
Subject: [PATCH] [feature] Rework timeline code to make it useful for more
than just statuses (#373)
* add preparable and timelineable interfaces
* initialize timeline manager within the processor
* generic renaming
* move status-specific timeline logic into the processor
* refactor timeline to make it useful for more than statuses
---
cmd/gotosocial/action/server/server.go | 4 +-
internal/api/model/status.go | 30 +++
.../api/s2s/webfinger/webfingerget_test.go | 4 +-
internal/gtsmodel/status.go | 21 ++
internal/processing/fromclientapi.go | 4 +-
internal/processing/fromcommon.go | 4 +-
internal/processing/fromfederator.go | 4 +-
internal/processing/processor.go | 6 +-
internal/processing/processor_test.go | 3 +-
.../{timeline.go => statustimeline.go} | 121 ++++++++++-
internal/timeline/get.go | 161 ++++++++-------
internal/timeline/get_test.go | 114 ++++++-----
internal/timeline/index.go | 191 +++++++++---------
internal/timeline/index_test.go | 60 +++---
.../timeline/{postindex.go => itemindex.go} | 44 ++--
internal/timeline/manager.go | 128 ++++++------
internal/timeline/manager_test.go | 17 +-
internal/timeline/preparable.go | 26 +++
internal/timeline/prepare.go | 120 +++++------
.../{preparedposts.go => prepareditems.go} | 41 ++--
internal/timeline/remove.go | 36 ++--
internal/timeline/timeline.go | 182 ++++++++++-------
internal/timeline/timeline_test.go | 6 +-
internal/timeline/timelineable.go | 27 +++
testrig/processor.go | 2 +-
testrig/timelinemanager.go | 11 -
26 files changed, 801 insertions(+), 566 deletions(-)
rename internal/processing/{timeline.go => statustimeline.go} (61%)
rename internal/timeline/{postindex.go => itemindex.go} (61%)
create mode 100644 internal/timeline/preparable.go
rename internal/timeline/{preparedposts.go => prepareditems.go} (71%)
create mode 100644 internal/timeline/timelineable.go
delete mode 100644 testrig/timelinemanager.go
diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go
index d55f38fc5..be4e2cc54 100644
--- a/cmd/gotosocial/action/server/server.go
+++ b/cmd/gotosocial/action/server/server.go
@@ -65,7 +65,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oidc"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
- timelineprocessing "github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/web"
@@ -95,7 +94,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
// build converters and util
typeConverter := typeutils.NewConverter(dbService)
- timelineManager := timelineprocessing.NewManager(dbService, typeConverter)
// Open the storage backend
storageBasePath := viper.GetString(config.Keys.StorageLocalBasePath)
@@ -128,7 +126,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
}
// create and start the message processor using the other services we've created so far
- processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, timelineManager, dbService, emailSender)
+ processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, dbService, emailSender)
if err := processor.Start(ctx); err != nil {
return fmt.Errorf("error starting processor: %s", err)
}
diff --git a/internal/api/model/status.go b/internal/api/model/status.go
index 3ff3f791d..fade58a49 100644
--- a/internal/api/model/status.go
+++ b/internal/api/model/status.go
@@ -96,6 +96,36 @@ type Status struct {
Text string `json:"text"`
}
+/*
+** The below functions are added onto the API model status so that it satisfies
+** the Preparable interface in internal/timeline.
+ */
+
+func (s *Status) GetID() string {
+ return s.ID
+}
+
+func (s *Status) GetAccountID() string {
+ if s.Account != nil {
+ return s.Account.ID
+ }
+ return ""
+}
+
+func (s *Status) GetBoostOfID() string {
+ if s.Reblog != nil {
+ return s.Reblog.ID
+ }
+ return ""
+}
+
+func (s *Status) GetBoostOfAccountID() string {
+ if s.Reblog != nil && s.Reblog.Account != nil {
+ return s.Reblog.Account.ID
+ }
+ return ""
+}
+
// StatusReblogged represents a reblogged status.
//
// swagger:model statusReblogged
diff --git a/internal/api/s2s/webfinger/webfingerget_test.go b/internal/api/s2s/webfinger/webfingerget_test.go
index 4b27ada42..d3b0c32e8 100644
--- a/internal/api/s2s/webfinger/webfingerget_test.go
+++ b/internal/api/s2s/webfinger/webfingerget_test.go
@@ -69,7 +69,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() {
func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() {
viper.Set(config.Keys.Host, "gts.example.org")
viper.Set(config.Keys.AccountDomain, "example.org")
- suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender)
+ suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
targetAccount := accountDomainAccount()
@@ -103,7 +103,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo
func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() {
viper.Set(config.Keys.Host, "gts.example.org")
viper.Set(config.Keys.AccountDomain, "example.org")
- suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender)
+ suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
targetAccount := accountDomainAccount()
diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go
index e80924ca3..e798ea41b 100644
--- a/internal/gtsmodel/status.go
+++ b/internal/gtsmodel/status.go
@@ -66,6 +66,27 @@ type Status struct {
Likeable bool `validate:"-" bun:",notnull"` // This status can be liked/faved
}
+/*
+ The below functions are added onto the gtsmodel status so that it satisfies
+ the Timelineable interface in internal/timeline.
+*/
+
+func (s *Status) GetID() string {
+ return s.ID
+}
+
+func (s *Status) GetAccountID() string {
+ return s.AccountID
+}
+
+func (s *Status) GetBoostOfID() string {
+ return s.BoostOfID
+}
+
+func (s *Status) GetBoostOfAccountID() string {
+ return s.BoostOfAccountID
+}
+
// StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags.
type StatusToTag struct {
StatusID string `validate:"ulid,required" bun:"type:CHAR(26),unique:statustag,nullzero,notnull"`
diff --git a/internal/processing/fromclientapi.go b/internal/processing/fromclientapi.go
index 51c896291..11ce2215e 100644
--- a/internal/processing/fromclientapi.go
+++ b/internal/processing/fromclientapi.go
@@ -192,10 +192,10 @@ func (p *processor) processCreateBlockFromClientAPI(ctx context.Context, clientM
}
// remove any of the blocking account's statuses from the blocked account's timeline, and vice versa
- if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
+ if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
return err
}
- if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
+ if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
return err
}
diff --git a/internal/processing/fromcommon.go b/internal/processing/fromcommon.go
index 3c52cf669..8e7f20145 100644
--- a/internal/processing/fromcommon.go
+++ b/internal/processing/fromcommon.go
@@ -413,7 +413,7 @@ func (p *processor) timelineStatusForAccount(ctx context.Context, status *gtsmod
}
// stick the status in the timeline for the account and then immediately prepare it so they can see it right away
- inserted, err := p.timelineManager.IngestAndPrepare(ctx, status, timelineAccount.ID)
+ inserted, err := p.statusTimelines.IngestAndPrepare(ctx, status, timelineAccount.ID)
if err != nil {
errors <- fmt.Errorf("timelineStatusForAccount: error ingesting status %s: %s", status.ID, err)
return
@@ -436,7 +436,7 @@ func (p *processor) timelineStatusForAccount(ctx context.Context, status *gtsmod
// deleteStatusFromTimelines completely removes the given status from all timelines.
// It will also stream deletion of the status to all open streams.
func (p *processor) deleteStatusFromTimelines(ctx context.Context, status *gtsmodel.Status) error {
- if err := p.timelineManager.WipeStatusFromAllTimelines(ctx, status.ID); err != nil {
+ if err := p.statusTimelines.WipeItemFromAllTimelines(ctx, status.ID); err != nil {
return err
}
diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go
index 533d00242..3083bcc40 100644
--- a/internal/processing/fromfederator.go
+++ b/internal/processing/fromfederator.go
@@ -213,10 +213,10 @@ func (p *processor) processCreateBlockFromFederator(ctx context.Context, federat
}
// remove any of the blocking account's statuses from the blocked account's timeline, and vice versa
- if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
+ if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
return err
}
- if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
+ if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
return err
}
// TODO: same with notifications
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
index f5334a1ef..13fc08c26 100644
--- a/internal/processing/processor.go
+++ b/internal/processing/processor.go
@@ -237,7 +237,7 @@ type processor struct {
oauthServer oauth.Server
mediaHandler media.Handler
storage *kv.KVStore
- timelineManager timeline.Manager
+ statusTimelines timeline.Manager
db db.DB
filter visibility.Filter
@@ -261,7 +261,6 @@ func NewProcessor(
oauthServer oauth.Server,
mediaHandler media.Handler,
storage *kv.KVStore,
- timelineManager timeline.Manager,
db db.DB,
emailSender email.Sender) Processor {
fromClientAPI := make(chan messages.FromClientAPI, 1000)
@@ -274,6 +273,7 @@ func NewProcessor(
mediaProcessor := mediaProcessor.New(db, tc, mediaHandler, storage)
userProcessor := user.New(db, emailSender)
federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
+ filter := visibility.NewFilter(db)
return &processor{
fromClientAPI: fromClientAPI,
@@ -284,7 +284,7 @@ func NewProcessor(
oauthServer: oauthServer,
mediaHandler: mediaHandler,
storage: storage,
- timelineManager: timelineManager,
+ statusTimelines: timeline.NewManager(StatusGrabFunction(db), StatusFilterFunction(db, filter), StatusPrepareFunction(db, tc), StatusSkipInsertFunction()),
db: db,
filter: visibility.NewFilter(db),
diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go
index dc7562a2e..34b8f40bb 100644
--- a/internal/processing/processor_test.go
+++ b/internal/processing/processor_test.go
@@ -219,10 +219,9 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
- suite.timelineManager = testrig.NewTestTimelineManager(suite.db)
suite.emailSender = testrig.NewEmailSender("../../web/template/", nil)
- suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.timelineManager, suite.db, suite.emailSender)
+ suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.db, suite.emailSender)
testrig.StandardDBSetup(suite.db, suite.testAccounts)
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
diff --git a/internal/processing/timeline.go b/internal/processing/statustimeline.go
similarity index 61%
rename from internal/processing/timeline.go
rename to internal/processing/statustimeline.go
index 2e2b7d637..355825900 100644
--- a/internal/processing/timeline.go
+++ b/internal/processing/statustimeline.go
@@ -20,6 +20,7 @@ package processing
import (
"context"
+ "errors"
"fmt"
"net/url"
@@ -32,8 +33,113 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
+ "github.com/superseriousbusiness/gotosocial/internal/timeline"
+ "github.com/superseriousbusiness/gotosocial/internal/typeutils"
+ "github.com/superseriousbusiness/gotosocial/internal/visibility"
)
+const boostReinsertionDepth = 50
+
+// StatusGrabFunction returns a function that satisfies the GrabFunction interface in internal/timeline.
+func StatusGrabFunction(database db.DB) timeline.GrabFunction {
+ return func(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int) ([]timeline.Timelineable, bool, error) {
+ statuses, err := database.GetHomeTimeline(ctx, timelineAccountID, maxID, sinceID, minID, limit, false)
+ if err != nil {
+ if err == db.ErrNoEntries {
+ return nil, true, nil // we just don't have enough statuses left in the db so return stop = true
+ }
+ return nil, false, fmt.Errorf("statusGrabFunction: error getting statuses from db: %s", err)
+ }
+
+ items := []timeline.Timelineable{}
+ for _, s := range statuses {
+ items = append(items, s)
+ }
+
+ return items, false, nil
+ }
+}
+
+// StatusFilterFunction returns a function that satisfies the FilterFunction interface in internal/timeline.
+func StatusFilterFunction(database db.DB, filter visibility.Filter) timeline.FilterFunction {
+ return func(ctx context.Context, timelineAccountID string, item timeline.Timelineable) (shouldIndex bool, err error) {
+ status, ok := item.(*gtsmodel.Status)
+ if !ok {
+ return false, errors.New("statusFilterFunction: could not convert item to *gtsmodel.Status")
+ }
+
+ requestingAccount, err := database.GetAccountByID(ctx, timelineAccountID)
+ if err != nil {
+ return false, fmt.Errorf("statusFilterFunction: error getting account with id %s", timelineAccountID)
+ }
+
+ timelineable, err := filter.StatusHometimelineable(ctx, status, requestingAccount)
+ if err != nil {
+ logrus.Warnf("error checking hometimelineability of status %s for account %s: %s", status.ID, timelineAccountID, err)
+ }
+
+ return timelineable, nil // we don't return the error here because we want to just skip this item if something goes wrong
+ }
+}
+
+// StatusPrepareFunction returns a function that satisfies the PrepareFunction interface in internal/timeline.
+func StatusPrepareFunction(database db.DB, tc typeutils.TypeConverter) timeline.PrepareFunction {
+ return func(ctx context.Context, timelineAccountID string, itemID string) (timeline.Preparable, error) {
+ status, err := database.GetStatusByID(ctx, itemID)
+ if err != nil {
+ return nil, fmt.Errorf("statusPrepareFunction: error getting status with id %s", itemID)
+ }
+
+ requestingAccount, err := database.GetAccountByID(ctx, timelineAccountID)
+ if err != nil {
+ return nil, fmt.Errorf("statusPrepareFunction: error getting account with id %s", timelineAccountID)
+ }
+
+ return tc.StatusToAPIStatus(ctx, status, requestingAccount)
+ }
+}
+
+// StatusSkipInsertFunction returns a function that satisifes the SkipInsertFunction interface in internal/timeline.
+func StatusSkipInsertFunction() timeline.SkipInsertFunction {
+ return func(
+ ctx context.Context,
+ newItemID string,
+ newItemAccountID string,
+ newItemBoostOfID string,
+ newItemBoostOfAccountID string,
+ nextItemID string,
+ nextItemAccountID string,
+ nextItemBoostOfID string,
+ nextItemBoostOfAccountID string,
+ depth int) (bool, error) {
+
+ // make sure we don't insert a duplicate
+ if newItemID == nextItemID {
+ return true, nil
+ }
+
+ // check if it's a boost
+ if newItemBoostOfID != "" {
+ // skip if we've recently put another boost of this status in the timeline
+ if newItemBoostOfID == nextItemBoostOfID {
+ if depth < boostReinsertionDepth {
+ return true, nil
+ }
+ }
+
+ // skip if we've recently put the original status in the timeline
+ if newItemBoostOfID == nextItemID {
+ if depth < boostReinsertionDepth {
+ return true, nil
+ }
+ }
+ }
+
+ // insert the item
+ return false, nil
+ }
+}
+
func (p *processor) packageStatusResponse(statuses []*apimodel.Status, path string, nextMaxID string, prevMinID string, limit int) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
resp := &apimodel.StatusTimelineResponse{
Statuses: []*apimodel.Status{},
@@ -67,18 +173,27 @@ func (p *processor) packageStatusResponse(statuses []*apimodel.Status, path stri
}
func (p *processor) HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
- statuses, err := p.timelineManager.HomeTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local)
+ preparedItems, err := p.statusTimelines.GetTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
- if len(statuses) == 0 {
+ if len(preparedItems) == 0 {
return &apimodel.StatusTimelineResponse{
Statuses: []*apimodel.Status{},
}, nil
}
- return p.packageStatusResponse(statuses, "api/v1/timelines/home", statuses[len(statuses)-1].ID, statuses[0].ID, limit)
+ statuses := []*apimodel.Status{}
+ for _, i := range preparedItems {
+ status, ok := i.(*apimodel.Status)
+ if !ok {
+ return nil, gtserror.NewErrorInternalError(errors.New("error converting prepared timeline entry to api status"))
+ }
+ statuses = append(statuses, status)
+ }
+
+ return p.packageStatusResponse(statuses, "api/v1/timelines/home", statuses[len(preparedItems)-1].ID, statuses[0].ID, limit)
}
func (p *processor) PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
diff --git a/internal/timeline/get.go b/internal/timeline/get.go
index da6dc6f76..a3cc8ef9b 100644
--- a/internal/timeline/get.go
+++ b/internal/timeline/get.go
@@ -25,12 +25,11 @@ import (
"fmt"
"github.com/sirupsen/logrus"
- apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
const retries = 5
-func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error) {
+func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error) {
l := logrus.WithFields(logrus.Fields{
"func": "Get",
"accountID": t.accountID,
@@ -41,16 +40,16 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
})
l.Debug("entering get")
- var statuses []*apimodel.Status
+ var items []Preparable
var err error
// no params are defined to just fetch from the top
- // this is equivalent to a user asking for the top x posts from their timeline
+ // this is equivalent to a user asking for the top x items from their timeline
if maxID == "" && sinceID == "" && minID == "" {
- statuses, err = t.GetXFromTop(ctx, amount)
+ items, err = t.GetXFromTop(ctx, amount)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
- if len(statuses) != 0 {
- nextMaxID := statuses[len(statuses)-1].ID
+ if len(items) != 0 {
+ nextMaxID := items[len(items)-1].GetID()
if prepareNext {
// already cache the next query to speed up scrolling
go func() {
@@ -64,13 +63,13 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
}
// maxID is defined but sinceID isn't so take from behind
- // this is equivalent to a user asking for the next x posts from their timeline, starting from maxID
+ // this is equivalent to a user asking for the next x items from their timeline, starting from maxID
if maxID != "" && sinceID == "" {
attempts := 0
- statuses, err = t.GetXBehindID(ctx, amount, maxID, &attempts)
+ items, err = t.GetXBehindID(ctx, amount, maxID, &attempts)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
- if len(statuses) != 0 {
- nextMaxID := statuses[len(statuses)-1].ID
+ if len(items) != 0 {
+ nextMaxID := items[len(items)-1].GetID()
if prepareNext {
// already cache the next query to speed up scrolling
go func() {
@@ -84,59 +83,59 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
}
// maxID is defined and sinceID || minID are as well, so take a slice between them
- // this is equivalent to a user asking for posts older than x but newer than y
+ // this is equivalent to a user asking for items older than x but newer than y
if maxID != "" && sinceID != "" {
- statuses, err = t.GetXBetweenID(ctx, amount, maxID, minID)
+ items, err = t.GetXBetweenID(ctx, amount, maxID, minID)
}
if maxID != "" && minID != "" {
- statuses, err = t.GetXBetweenID(ctx, amount, maxID, minID)
+ items, err = t.GetXBetweenID(ctx, amount, maxID, minID)
}
// maxID isn't defined, but sinceID || minID are, so take x before
- // this is equivalent to a user asking for posts newer than x (eg., refreshing the top of their timeline)
+ // this is equivalent to a user asking for items newer than x (eg., refreshing the top of their timeline)
if maxID == "" && sinceID != "" {
- statuses, err = t.GetXBeforeID(ctx, amount, sinceID, true)
+ items, err = t.GetXBeforeID(ctx, amount, sinceID, true)
}
if maxID == "" && minID != "" {
- statuses, err = t.GetXBeforeID(ctx, amount, minID, true)
+ items, err = t.GetXBeforeID(ctx, amount, minID, true)
}
- return statuses, err
+ return items, err
}
-func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]*apimodel.Status, error) {
- // make a slice of statuses with the length we need to return
- statuses := make([]*apimodel.Status, 0, amount)
+func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]Preparable, error) {
+ // make a slice of preparedItems with the length we need to return
+ preparedItems := make([]Preparable, 0, amount)
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
}
- // make sure we have enough posts prepared to return
- if t.preparedPosts.data.Len() < amount {
+ // make sure we have enough items prepared to return
+ if t.preparedItems.data.Len() < amount {
if err := t.PrepareFromTop(ctx, amount); err != nil {
return nil, err
}
}
- // work through the prepared posts from the top and return
+ // work through the prepared items from the top and return
var served int
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
- return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry")
+ return nil, errors.New("GetXFromTop: could not parse e as a preparedItemsEntry")
}
- statuses = append(statuses, entry.prepared)
+ preparedItems = append(preparedItems, entry.prepared)
served++
if served >= amount {
break
}
}
- return statuses, nil
+ return preparedItems, nil
}
-func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]*apimodel.Status, error) {
+func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]Preparable, error) {
l := logrus.WithFields(logrus.Fields{
"func": "GetXBehindID",
"amount": amount,
@@ -148,11 +147,11 @@ func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string
newAttempts++
attempts = &newAttempts
- // make a slice of statuses with the length we need to return
- statuses := make([]*apimodel.Status, 0, amount)
+ // make a slice of items with the length we need to return
+ items := make([]Preparable, 0, amount)
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
@@ -160,14 +159,14 @@ func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string
var behindIDMark *list.Element
findMarkLoop:
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
position++
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
- if entry.statusID <= behindID {
+ if entry.itemID <= behindID {
l.Trace("found behindID mark")
behindIDMark = e
break findMarkLoop
@@ -175,33 +174,33 @@ findMarkLoop:
}
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
- // this can happen when a user asks for really old posts
+ // this can happen when a user asks for really old items
if behindIDMark == nil {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
}
- oldestID, err := t.OldestPreparedPostID(ctx)
+ oldestID, err := t.OldestPreparedItemID(ctx)
if err != nil {
return nil, err
}
if oldestID == "" {
l.Tracef("oldestID is empty so we can't return behindID %s", behindID)
- return statuses, nil
+ return items, nil
}
if oldestID == behindID {
l.Tracef("given behindID %s is the same as oldestID %s so there's nothing to return behind it", behindID, oldestID)
- return statuses, nil
+ return items, nil
}
if *attempts > retries {
l.Tracef("exceeded retries looking for behindID %s", behindID)
- return statuses, nil
+ return items, nil
}
l.Trace("trying GetXBehindID again")
return t.GetXBehindID(ctx, amount, behindID, attempts)
}
- // make sure we have enough posts prepared behind it to return what we're being asked for
- if t.preparedPosts.data.Len() < amount+position {
+ // make sure we have enough items prepared behind it to return what we're being asked for
+ if t.preparedItems.data.Len() < amount+position {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
@@ -211,40 +210,40 @@ findMarkLoop:
var served int
serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
- statuses = append(statuses, entry.prepared)
+ items = append(items, entry.prepared)
served++
if served >= amount {
break serveloop
}
}
- return statuses, nil
+ return items, nil
}
-func (t *timeline) GetXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
- // make a slice of statuses with the length we need to return
- statuses := make([]*apimodel.Status, 0, amount)
+func (t *timeline) GetXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]Preparable, error) {
+ // make a slice of items with the length we need to return
+ items := make([]Preparable, 0, amount)
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for, or as close as possible to it
var beforeIDMark *list.Element
findMarkLoop:
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
- if entry.statusID >= beforeID {
+ if entry.itemID >= beforeID {
beforeIDMark = e
} else {
break findMarkLoop
@@ -252,26 +251,26 @@ findMarkLoop:
}
if beforeIDMark == nil {
- return statuses, nil
+ return items, nil
}
var served int
if startFromTop {
- // start serving from the front/top and keep going until we hit mark or get x amount statuses
+ // start serving from the front/top and keep going until we hit mark or get x amount items
serveloopFromTop:
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
- if entry.statusID == beforeID {
+ if entry.itemID == beforeID {
break serveloopFromTop
}
// serve up to the amount requested
- statuses = append(statuses, entry.prepared)
+ items = append(items, entry.prepared)
served++
if served >= amount {
break serveloopFromTop
@@ -281,13 +280,13 @@ findMarkLoop:
// start serving from the entry right before the mark
serveloopFromBottom:
for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
- statuses = append(statuses, entry.prepared)
+ items = append(items, entry.prepared)
served++
if served >= amount {
break serveloopFromBottom
@@ -295,29 +294,29 @@ findMarkLoop:
}
}
- return statuses, nil
+ return items, nil
}
-func (t *timeline) GetXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
- // make a slice of statuses with the length we need to return
- statuses := make([]*apimodel.Status, 0, amount)
+func (t *timeline) GetXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]Preparable, error) {
+ // make a slice of items with the length we need to return
+ items := make([]Preparable, 0, amount)
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
var position int
var behindIDMark *list.Element
findMarkLoop:
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
position++
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
}
- if entry.statusID == behindID {
+ if entry.itemID == behindID {
behindIDMark = e
break findMarkLoop
}
@@ -325,11 +324,11 @@ findMarkLoop:
// we didn't find it
if behindIDMark == nil {
- return nil, fmt.Errorf("GetXBetweenID: couldn't find status with ID %s", behindID)
+ return nil, fmt.Errorf("GetXBetweenID: couldn't find item with ID %s", behindID)
}
- // make sure we have enough posts prepared behind it to return what we're being asked for
- if t.preparedPosts.data.Len() < amount+position {
+ // make sure we have enough items prepared behind it to return what we're being asked for
+ if t.preparedItems.data.Len() < amount+position {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
@@ -339,22 +338,22 @@ findMarkLoop:
var served int
serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
}
- if entry.statusID == beforeID {
+ if entry.itemID == beforeID {
break serveloop
}
// serve up to the amount requested
- statuses = append(statuses, entry.prepared)
+ items = append(items, entry.prepared)
served++
if served >= amount {
break serveloop
}
}
- return statuses, nil
+ return items, nil
}
diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go
index b3a19f488..a1640790e 100644
--- a/internal/timeline/get_test.go
+++ b/internal/timeline/get_test.go
@@ -24,7 +24,9 @@ import (
"time"
"github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
+ "github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -43,18 +45,26 @@ func (suite *GetTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
+ suite.filter = visibility.NewFilter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner
- tl, err := timeline.NewTimeline(context.Background(), suite.testAccounts["local_account_1"].ID, suite.db, suite.tc)
+ tl, err := timeline.NewTimeline(
+ context.Background(),
+ suite.testAccounts["local_account_1"].ID,
+ processing.StatusGrabFunction(suite.db),
+ processing.StatusFilterFunction(suite.db, suite.filter),
+ processing.StatusPrepareFunction(suite.db, suite.tc),
+ processing.StatusSkipInsertFunction(),
+ )
if err != nil {
suite.FailNow(err.Error())
}
// prepare the timeline by just shoving all test statuses in it -- let's not be fussy about who sees what
for _, s := range suite.testStatuses {
- _, err := tl.IndexAndPrepareOne(context.Background(), s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID)
+ _, err := tl.IndexAndPrepareOne(context.Background(), s.GetID(), s.BoostOfID, s.AccountID, s.BoostOfAccountID)
if err != nil {
suite.FailNow(err.Error())
}
@@ -81,10 +91,10 @@ func (suite *GetTestSuite) TestGetDefault() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
}
@@ -102,10 +112,10 @@ func (suite *GetTestSuite) TestGetDefaultPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
@@ -127,10 +137,10 @@ func (suite *GetTestSuite) TestGetMaxID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
}
@@ -149,10 +159,10 @@ func (suite *GetTestSuite) TestGetMaxIDPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
@@ -174,10 +184,10 @@ func (suite *GetTestSuite) TestGetMinID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
}
@@ -196,10 +206,10 @@ func (suite *GetTestSuite) TestGetSinceID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
}
@@ -218,10 +228,10 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
@@ -243,10 +253,10 @@ func (suite *GetTestSuite) TestGetBetweenID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
}
@@ -265,10 +275,10 @@ func (suite *GetTestSuite) TestGetBetweenIDPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
@@ -289,10 +299,10 @@ func (suite *GetTestSuite) TestGetXFromTop() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
}
}
@@ -314,12 +324,12 @@ func (suite *GetTestSuite) TestGetXBehindID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
- suite.Less(s.ID, "01F8MHBQCBTDKN6X5VHGMMN4MA")
+ suite.Less(s.GetID(), "01F8MHBQCBTDKN6X5VHGMMN4MA")
}
}
@@ -353,12 +363,12 @@ func (suite *GetTestSuite) TestGetXBehindNonexistentReasonableID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
- suite.Less(s.ID, "01F8MHBCN8120SYH7D5S050MGK")
+ suite.Less(s.GetID(), "01F8MHBCN8120SYH7D5S050MGK")
}
}
@@ -380,12 +390,12 @@ func (suite *GetTestSuite) TestGetXBehindVeryHighID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
- suite.Less(s.ID, "9998MHBQCBTDKN6X5VHGMMN4MA")
+ suite.Less(s.GetID(), "9998MHBQCBTDKN6X5VHGMMN4MA")
}
}
@@ -403,12 +413,12 @@ func (suite *GetTestSuite) TestGetXBeforeID() {
var highest string
for i, s := range statuses {
if i == 0 {
- highest = s.ID
+ highest = s.GetID()
} else {
- suite.Less(s.ID, highest)
- highest = s.ID
+ suite.Less(s.GetID(), highest)
+ highest = s.GetID()
}
- suite.Greater(s.ID, "01F8MHBQCBTDKN6X5VHGMMN4MA")
+ suite.Greater(s.GetID(), "01F8MHBQCBTDKN6X5VHGMMN4MA")
}
}
@@ -426,12 +436,12 @@ func (suite *GetTestSuite) TestGetXBeforeIDNoStartFromTop() {
var lowest string
for i, s := range statuses {
if i == 0 {
- lowest = s.ID
+ lowest = s.GetID()
} else {
- suite.Greater(s.ID, lowest)
- lowest = s.ID
+ suite.Greater(s.GetID(), lowest)
+ lowest = s.GetID()
}
- suite.Greater(s.ID, "01F8MHBQCBTDKN6X5VHGMMN4MA")
+ suite.Greater(s.GetID(), "01F8MHBQCBTDKN6X5VHGMMN4MA")
}
}
diff --git a/internal/timeline/index.go b/internal/timeline/index.go
index 3d940af80..bda3a9c6c 100644
--- a/internal/timeline/index.go
+++ b/internal/timeline/index.go
@@ -23,173 +23,166 @@ import (
"context"
"errors"
"fmt"
- "time"
"github.com/sirupsen/logrus"
- "github.com/superseriousbusiness/gotosocial/internal/db"
- "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
-func (t *timeline) IndexBefore(ctx context.Context, statusID string, include bool, amount int) error {
+func (t *timeline) IndexBefore(ctx context.Context, itemID string, amount int) error {
+ l := logrus.WithFields(logrus.Fields{
+ "func": "IndexBefore",
+ "amount": amount,
+ })
+
// lazily initialize index if it hasn't been done already
- if t.postIndex.data == nil {
- t.postIndex.data = &list.List{}
- t.postIndex.data.Init()
+ if t.itemIndex.data == nil {
+ t.itemIndex.data = &list.List{}
+ t.itemIndex.data.Init()
}
- filtered := []*gtsmodel.Status{}
- offsetStatus := statusID
+ toIndex := []Timelineable{}
+ offsetID := itemID
- if include {
- // if we have the status with given statusID in the database, include it in the results set as well
- s := >smodel.Status{}
- if err := t.db.GetByID(ctx, statusID, s); err == nil {
- filtered = append(filtered, s)
- }
- }
-
- i := 0
+ l.Trace("entering grabloop")
grabloop:
- for ; len(filtered) < amount && i < 5; i++ { // try the grabloop 5 times only
- statuses, err := t.db.GetHomeTimeline(ctx, t.accountID, "", "", offsetStatus, amount, false)
+ for i := 0; len(toIndex) < amount && i < 5; i++ { // try the grabloop 5 times only
+ // first grab items using the caller-provided grab function
+ l.Trace("grabbing...")
+ items, stop, err := t.grabFunction(ctx, t.accountID, "", "", offsetID, amount)
if err != nil {
- if err == db.ErrNoEntries {
- break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
- }
- return fmt.Errorf("IndexBefore: error getting statuses from db: %s", err)
+ return err
+ }
+ if stop {
+ break grabloop
}
- for _, s := range statuses {
- timelineable, err := t.filter.StatusHometimelineable(ctx, s, t.account)
+ l.Trace("filtering...")
+ // now filter each item using the caller-provided filter function
+ for _, item := range items {
+ shouldIndex, err := t.filterFunction(ctx, t.accountID, item)
if err != nil {
- continue
+ return err
}
- if timelineable {
- filtered = append(filtered, s)
+ if shouldIndex {
+ toIndex = append(toIndex, item)
}
- offsetStatus = s.ID
+ offsetID = item.GetID()
}
}
+ l.Trace("left grabloop")
- for _, s := range filtered {
- if _, err := t.IndexOne(ctx, s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
- return fmt.Errorf("IndexBefore: error indexing status with id %s: %s", s.ID, err)
+ // index the items we got
+ for _, s := range toIndex {
+ if _, err := t.IndexOne(ctx, s.GetID(), s.GetBoostOfID(), s.GetAccountID(), s.GetBoostOfAccountID()); err != nil {
+ return fmt.Errorf("IndexBehind: error indexing item with id %s: %s", s.GetID(), err)
}
}
return nil
}
-func (t *timeline) IndexBehind(ctx context.Context, statusID string, include bool, amount int) error {
+func (t *timeline) IndexBehind(ctx context.Context, itemID string, amount int) error {
l := logrus.WithFields(logrus.Fields{
- "func": "IndexBehind",
- "include": include,
- "amount": amount,
+ "func": "IndexBehind",
+ "amount": amount,
})
// lazily initialize index if it hasn't been done already
- if t.postIndex.data == nil {
- t.postIndex.data = &list.List{}
- t.postIndex.data.Init()
+ if t.itemIndex.data == nil {
+ t.itemIndex.data = &list.List{}
+ t.itemIndex.data.Init()
}
- // If we're already indexedBehind given statusID by the required amount, we can return nil.
- // First find position of statusID (or as near as possible).
+ // If we're already indexedBehind given itemID by the required amount, we can return nil.
+ // First find position of itemID (or as near as possible).
var position int
positionLoop:
- for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*postIndexEntry)
+ for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
- return errors.New("IndexBehind: could not parse e as a postIndexEntry")
+ return errors.New("IndexBehind: could not parse e as an itemIndexEntry")
}
- if entry.statusID <= statusID {
+ if entry.itemID <= itemID {
// we've found it
break positionLoop
}
position++
}
- // now check if the length of indexed posts exceeds the amount of posts required (position of statusID, plus amount of posts requested after that)
- if t.postIndex.data.Len() > position+amount {
+
+ // now check if the length of indexed items exceeds the amount of items required (position of itemID, plus amount of posts requested after that)
+ if t.itemIndex.data.Len() > position+amount {
// we have enough indexed behind already to satisfy amount, so don't need to make db calls
- l.Trace("returning nil since we already have enough posts indexed")
+ l.Trace("returning nil since we already have enough items indexed")
return nil
}
- filtered := []*gtsmodel.Status{}
- offsetStatus := statusID
+ toIndex := []Timelineable{}
+ offsetID := itemID
- if include {
- // if we have the status with given statusID in the database, include it in the results set as well
- s := >smodel.Status{}
- if err := t.db.GetByID(ctx, statusID, s); err == nil {
- filtered = append(filtered, s)
- }
- }
-
- i := 0
+ l.Trace("entering grabloop")
grabloop:
- for ; len(filtered) < amount && i < 5; i++ { // try the grabloop 5 times only
- l.Tracef("entering grabloop; i is %d; len(filtered) is %d", i, len(filtered))
- statuses, err := t.db.GetHomeTimeline(ctx, t.accountID, offsetStatus, "", "", amount, false)
+ for i := 0; len(toIndex) < amount && i < 5; i++ { // try the grabloop 5 times only
+ // first grab items using the caller-provided grab function
+ l.Trace("grabbing...")
+ items, stop, err := t.grabFunction(ctx, t.accountID, offsetID, "", "", amount)
if err != nil {
- if err == db.ErrNoEntries {
- break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
- }
- return fmt.Errorf("IndexBehind: error getting statuses from db: %s", err)
+ return err
+ }
+ if stop {
+ break grabloop
}
- l.Tracef("got %d statuses", len(statuses))
- for _, s := range statuses {
- timelineable, err := t.filter.StatusHometimelineable(ctx, s, t.account)
+ l.Trace("filtering...")
+ // now filter each item using the caller-provided filter function
+ for _, item := range items {
+ shouldIndex, err := t.filterFunction(ctx, t.accountID, item)
if err != nil {
- l.Tracef("status was not hometimelineable: %s", err)
- continue
+ return err
}
- if timelineable {
- filtered = append(filtered, s)
+ if shouldIndex {
+ toIndex = append(toIndex, item)
}
- offsetStatus = s.ID
+ offsetID = item.GetID()
}
}
l.Trace("left grabloop")
- for _, s := range filtered {
- if _, err := t.IndexOne(ctx, s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
- return fmt.Errorf("IndexBehind: error indexing status with id %s: %s", s.ID, err)
+ // index the items we got
+ for _, s := range toIndex {
+ if _, err := t.IndexOne(ctx, s.GetID(), s.GetBoostOfID(), s.GetAccountID(), s.GetBoostOfAccountID()); err != nil {
+ return fmt.Errorf("IndexBehind: error indexing item with id %s: %s", s.GetID(), err)
}
}
- l.Trace("exiting function")
return nil
}
-func (t *timeline) IndexOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
+func (t *timeline) IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
- postIndexEntry := &postIndexEntry{
- statusID: statusID,
+ postIndexEntry := &itemIndexEntry{
+ itemID: itemID,
boostOfID: boostOfID,
accountID: accountID,
boostOfAccountID: boostOfAccountID,
}
- return t.postIndex.insertIndexed(postIndexEntry)
+ return t.itemIndex.insertIndexed(ctx, postIndexEntry)
}
-func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
+func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
- postIndexEntry := &postIndexEntry{
- statusID: statusID,
+ postIndexEntry := &itemIndexEntry{
+ itemID: statusID,
boostOfID: boostOfID,
accountID: accountID,
boostOfAccountID: boostOfAccountID,
}
- inserted, err := t.postIndex.insertIndexed(postIndexEntry)
+ inserted, err := t.itemIndex.insertIndexed(ctx, postIndexEntry)
if err != nil {
return inserted, fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
}
@@ -203,32 +196,32 @@ func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.
return inserted, nil
}
-func (t *timeline) OldestIndexedPostID(ctx context.Context) (string, error) {
+func (t *timeline) OldestIndexedItemID(ctx context.Context) (string, error) {
var id string
- if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Back() == nil {
+ if t.itemIndex == nil || t.itemIndex.data == nil || t.itemIndex.data.Back() == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
- e := t.postIndex.data.Back()
- entry, ok := e.Value.(*postIndexEntry)
+ e := t.itemIndex.data.Back()
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
- return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry")
+ return id, errors.New("OldestIndexedItemID: could not parse e as itemIndexEntry")
}
- return entry.statusID, nil
+ return entry.itemID, nil
}
-func (t *timeline) NewestIndexedPostID(ctx context.Context) (string, error) {
+func (t *timeline) NewestIndexedItemID(ctx context.Context) (string, error) {
var id string
- if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Front() == nil {
+ if t.itemIndex == nil || t.itemIndex.data == nil || t.itemIndex.data.Front() == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
- e := t.postIndex.data.Front()
- entry, ok := e.Value.(*postIndexEntry)
+ e := t.itemIndex.data.Front()
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
- return id, errors.New("NewestIndexedPostID: could not parse e as a postIndexEntry")
+ return id, errors.New("NewestIndexedItemID: could not parse e as itemIndexEntry")
}
- return entry.statusID, nil
+ return entry.itemID, nil
}
diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go
index e09421801..3f4822dcb 100644
--- a/internal/timeline/index_test.go
+++ b/internal/timeline/index_test.go
@@ -25,7 +25,9 @@ import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
+ "github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -44,11 +46,19 @@ func (suite *IndexTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
+ suite.filter = visibility.NewFilter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner, and start with an empty timeline
- tl, err := timeline.NewTimeline(context.Background(), suite.testAccounts["local_account_1"].ID, suite.db, suite.tc)
+ tl, err := timeline.NewTimeline(
+ context.Background(),
+ suite.testAccounts["local_account_1"].ID,
+ processing.StatusGrabFunction(suite.db),
+ processing.StatusFilterFunction(suite.db, suite.filter),
+ processing.StatusPrepareFunction(suite.db, suite.tc),
+ processing.StatusSkipInsertFunction(),
+ )
if err != nil {
suite.FailNow(err.Error())
}
@@ -61,82 +71,82 @@ func (suite *IndexTestSuite) TearDownTest() {
func (suite *IndexTestSuite) TestIndexBeforeLowID() {
// index 10 before the lowest status ID possible
- err := suite.timeline.IndexBefore(context.Background(), "00000000000000000000000000", true, 10)
+ err := suite.timeline.IndexBefore(context.Background(), "00000000000000000000000000", 10)
suite.NoError(err)
// the oldest indexed post should be the lowest one we have in our testrig
- postID, err := suite.timeline.OldestIndexedPostID(context.Background())
+ postID, err := suite.timeline.OldestIndexedItemID(context.Background())
suite.NoError(err)
suite.Equal("01F8MHAYFKS4KMXF8K5Y1C0KRN", postID)
- indexLength := suite.timeline.PostIndexLength(context.Background())
+ indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(10, indexLength)
}
func (suite *IndexTestSuite) TestIndexBeforeHighID() {
// index 10 before the highest status ID possible
- err := suite.timeline.IndexBefore(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
+ err := suite.timeline.IndexBefore(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", 10)
suite.NoError(err)
// the oldest indexed post should be empty
- postID, err := suite.timeline.OldestIndexedPostID(context.Background())
+ postID, err := suite.timeline.OldestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
- indexLength := suite.timeline.PostIndexLength(context.Background())
+ indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindHighID() {
// index 10 behind the highest status ID possible
- err := suite.timeline.IndexBehind(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
+ err := suite.timeline.IndexBehind(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", 10)
suite.NoError(err)
// the newest indexed post should be the highest one we have in our testrig
- postID, err := suite.timeline.NewestIndexedPostID(context.Background())
+ postID, err := suite.timeline.NewestIndexedItemID(context.Background())
suite.NoError(err)
suite.Equal("01FN3VJGFH10KR7S2PB0GFJZYG", postID)
// indexLength should be 10 because that's all this user has hometimelineable
- indexLength := suite.timeline.PostIndexLength(context.Background())
+ indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(10, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindLowID() {
// index 10 behind the lowest status ID possible
- err := suite.timeline.IndexBehind(context.Background(), "00000000000000000000000000", true, 10)
+ err := suite.timeline.IndexBehind(context.Background(), "00000000000000000000000000", 10)
suite.NoError(err)
// the newest indexed post should be empty
- postID, err := suite.timeline.NewestIndexedPostID(context.Background())
+ postID, err := suite.timeline.NewestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
- indexLength := suite.timeline.PostIndexLength(context.Background())
+ indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
-func (suite *IndexTestSuite) TestOldestIndexedPostIDEmpty() {
+func (suite *IndexTestSuite) TestOldestIndexedItemIDEmpty() {
// the oldest indexed post should be an empty string since there's nothing indexed yet
- postID, err := suite.timeline.OldestIndexedPostID(context.Background())
+ postID, err := suite.timeline.OldestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
- indexLength := suite.timeline.PostIndexLength(context.Background())
+ indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
-func (suite *IndexTestSuite) TestNewestIndexedPostIDEmpty() {
+func (suite *IndexTestSuite) TestNewestIndexedItemIDEmpty() {
// the newest indexed post should be an empty string since there's nothing indexed yet
- postID, err := suite.timeline.NewestIndexedPostID(context.Background())
+ postID, err := suite.timeline.NewestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
- indexLength := suite.timeline.PostIndexLength(context.Background())
+ indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
@@ -144,12 +154,12 @@ func (suite *IndexTestSuite) TestIndexAlreadyIndexed() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index one post -- it should be indexed
- indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
+ indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the same post again -- it should not be indexed
- indexed, err = suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
+ indexed, err = suite.timeline.IndexOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
@@ -158,12 +168,12 @@ func (suite *IndexTestSuite) TestIndexAndPrepareAlreadyIndexedAndPrepared() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index and prepare one post -- it should be indexed
- indexed, err := suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
+ indexed, err := suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index and prepare the same post again -- it should not be indexed
- indexed, err = suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
+ indexed, err = suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
@@ -179,12 +189,12 @@ func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() {
}
// index one post -- it should be indexed
- indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
+ indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the a boost of that post -- it should not be indexed
- indexed, err = suite.timeline.IndexOne(context.Background(), boostOfTestStatus.CreatedAt, boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
+ indexed, err = suite.timeline.IndexOne(context.Background(), boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
diff --git a/internal/timeline/postindex.go b/internal/timeline/itemindex.go
similarity index 61%
rename from internal/timeline/postindex.go
rename to internal/timeline/itemindex.go
index 5fe795d5a..968650e07 100644
--- a/internal/timeline/postindex.go
+++ b/internal/timeline/itemindex.go
@@ -20,21 +20,23 @@ package timeline
import (
"container/list"
+ "context"
"errors"
)
-type postIndex struct {
- data *list.List
+type itemIndex struct {
+ data *list.List
+ skipInsert SkipInsertFunction
}
-type postIndexEntry struct {
- statusID string
+type itemIndexEntry struct {
+ itemID string
boostOfID string
accountID string
boostOfAccountID string
}
-func (p *postIndex) insertIndexed(i *postIndexEntry) (bool, error) {
+func (p *itemIndex) insertIndexed(ctx context.Context, i *itemIndexEntry) (bool, error) {
if p.data == nil {
p.data = &list.List{}
}
@@ -47,36 +49,30 @@ func (p *postIndex) insertIndexed(i *postIndexEntry) (bool, error) {
var insertMark *list.Element
var position int
- // We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
- // We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
+ // We need to iterate through the index to make sure we put this item in the appropriate place according to when it was created.
+ // We also need to make sure we're not inserting a duplicate item -- this can happen sometimes and it's not nice UX (*shudder*).
for e := p.data.Front(); e != nil; e = e.Next() {
position++
- entry, ok := e.Value.(*postIndexEntry)
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
- return false, errors.New("index: could not parse e as a postIndexEntry")
+ return false, errors.New("index: could not parse e as an itemIndexEntry")
}
- // don't insert this if it's a boost of a status we've seen recently
- if i.boostOfID != "" {
- if i.boostOfID == entry.boostOfID || i.boostOfID == entry.statusID {
- if position < boostReinsertionDepth {
- return false, nil
- }
- }
+ skip, err := p.skipInsert(ctx, i.itemID, i.accountID, i.boostOfID, i.boostOfAccountID, entry.itemID, entry.accountID, entry.boostOfID, entry.boostOfAccountID, position)
+ if err != nil {
+ return false, err
+ }
+ if skip {
+ return false, nil
}
- // if the post to index is newer than e, insert it before e in the list
+ // if the item to index is newer than e, insert it before e in the list
if insertMark == nil {
- if i.statusID > entry.statusID {
+ if i.itemID > entry.itemID {
insertMark = e
}
}
-
- // make sure we don't insert a duplicate
- if entry.statusID == i.statusID {
- return false, nil
- }
}
if insertMark != nil {
@@ -84,7 +80,7 @@ func (p *postIndex) insertIndexed(i *postIndexEntry) (bool, error) {
return true, nil
}
- // if we reach this point it's the oldest post we've seen so put it at the back
+ // if we reach this point it's the oldest item we've seen so put it at the back
p.data.PushBack(i)
return true, nil
}
diff --git a/internal/timeline/manager.go b/internal/timeline/manager.go
index 5aa74ef91..02a388aba 100644
--- a/internal/timeline/manager.go
+++ b/internal/timeline/manager.go
@@ -25,10 +25,6 @@ import (
"sync"
"github.com/sirupsen/logrus"
- apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
- "github.com/superseriousbusiness/gotosocial/internal/db"
- "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
- "github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
const (
@@ -37,71 +33,75 @@ const (
// Manager abstracts functions for creating timelines for multiple accounts, and adding, removing, and fetching entries from those timelines.
//
-// By the time a status hits the manager interface, it should already have been filtered and it should be established that the status indeed
-// belongs in the home timeline of the given account ID.
+// By the time a timelineable hits the manager interface, it should already have been filtered and it should be established that the item indeed
+// belongs in the timeline of the given account ID.
//
-// The manager makes a distinction between *indexed* posts and *prepared* posts.
+// The manager makes a distinction between *indexed* items and *prepared* items.
//
-// Indexed posts consist of just that post's ID (in the database) and the time it was created. An indexed post takes up very little memory, so
-// it's not a huge priority to keep trimming the indexed posts list.
+// Indexed items consist of just that item's ID (in the database) and the time it was created. An indexed item takes up very little memory, so
+// it's not a huge priority to keep trimming the indexed items list.
//
-// Prepared posts consist of the post's database ID, the time it was created, AND the apimodel representation of that post, for quick serialization.
-// Prepared posts of course take up more memory than indexed posts, so they should be regularly pruned if they're not being actively served.
+// Prepared items consist of the item's database ID, the time it was created, AND the apimodel representation of that item, for quick serialization.
+// Prepared items of course take up more memory than indexed items, so they should be regularly pruned if they're not being actively served.
type Manager interface {
- // Ingest takes one status and indexes it into the timeline for the given account ID.
+ // Ingest takes one item and indexes it into the timeline for the given account ID.
//
- // It should already be established before calling this function that the status/post actually belongs in the timeline!
+ // It should already be established before calling this function that the item actually belongs in the timeline!
//
- // The returned bool indicates whether the status was actually put in the timeline. This could be false in cases where
- // the status is a boost, but a boost of the original post or the post itself already exists recently in the timeline.
- Ingest(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error)
- // IngestAndPrepare takes one status and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
- // This is useful in cases where we know the status will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
+ // The returned bool indicates whether the item was actually put in the timeline. This could be false in cases where
+ // the item is a boosted status, but a boost of the original status or the status itself already exists recently in the timeline.
+ Ingest(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error)
+ // IngestAndPrepare takes one timelineable and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
+ // This is useful in cases where we know the item will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
//
- // It should already be established before calling this function that the status/post actually belongs in the timeline!
+ // It should already be established before calling this function that the item actually belongs in the timeline!
//
- // The returned bool indicates whether the status was actually put in the timeline. This could be false in cases where
- // the status is a boost, but a boost of the original post or the post itself already exists recently in the timeline.
- IngestAndPrepare(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error)
- // HomeTimeline returns limit n amount of entries from the home timeline of the given account ID, in descending chronological order.
- // If maxID is provided, it will return entries from that maxID onwards, inclusive.
- HomeTimeline(ctx context.Context, accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
- // GetIndexedLength returns the amount of posts/statuses that have been *indexed* for the given account ID.
+ // The returned bool indicates whether the item was actually put in the timeline. This could be false in cases where
+ // a status is a boost, but a boost of the original status or the status itself already exists recently in the timeline.
+ IngestAndPrepare(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error)
+ // GetTimeline returns limit n amount of prepared entries from the timeline of the given account ID, in descending chronological order.
+ // If maxID is provided, it will return prepared entries from that maxID onwards, inclusive.
+ GetTimeline(ctx context.Context, accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error)
+ // GetIndexedLength returns the amount of items that have been *indexed* for the given account ID.
GetIndexedLength(ctx context.Context, timelineAccountID string) int
- // GetDesiredIndexLength returns the amount of posts that we, ideally, index for each user.
+ // GetDesiredIndexLength returns the amount of items that we, ideally, index for each user.
GetDesiredIndexLength(ctx context.Context) int
- // GetOldestIndexedID returns the status ID for the oldest post that we have indexed for the given account.
+ // GetOldestIndexedID returns the id ID for the oldest item that we have indexed for the given account.
GetOldestIndexedID(ctx context.Context, timelineAccountID string) (string, error)
- // PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
+ // PrepareXFromTop prepares limit n amount of items, based on their indexed representations, from the top of the index.
PrepareXFromTop(ctx context.Context, timelineAccountID string, limit int) error
- // Remove removes one status from the timeline of the given timelineAccountID
- Remove(ctx context.Context, timelineAccountID string, statusID string) (int, error)
- // WipeStatusFromAllTimelines removes one status from the index and prepared posts of all timelines
- WipeStatusFromAllTimelines(ctx context.Context, statusID string) error
- // WipeStatusesFromAccountID removes all statuses by the given accountID from the timelineAccountID's timelines.
- WipeStatusesFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error
+ // Remove removes one item from the timeline of the given timelineAccountID
+ Remove(ctx context.Context, timelineAccountID string, itemID string) (int, error)
+ // WipeItemFromAllTimelines removes one item from the index and prepared items of all timelines
+ WipeItemFromAllTimelines(ctx context.Context, itemID string) error
+ // WipeStatusesFromAccountID removes all items by the given accountID from the timelineAccountID's timelines.
+ WipeItemsFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error
}
-// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
-func NewManager(db db.DB, tc typeutils.TypeConverter) Manager {
+// NewManager returns a new timeline manager.
+func NewManager(grabFunction GrabFunction, filterFunction FilterFunction, prepareFunction PrepareFunction, skipInsertFunction SkipInsertFunction) Manager {
return &manager{
- accountTimelines: sync.Map{},
- db: db,
- tc: tc,
+ accountTimelines: sync.Map{},
+ grabFunction: grabFunction,
+ filterFunction: filterFunction,
+ prepareFunction: prepareFunction,
+ skipInsertFunction: skipInsertFunction,
}
}
type manager struct {
- accountTimelines sync.Map
- db db.DB
- tc typeutils.TypeConverter
+ accountTimelines sync.Map
+ grabFunction GrabFunction
+ filterFunction FilterFunction
+ prepareFunction PrepareFunction
+ skipInsertFunction SkipInsertFunction
}
-func (m *manager) Ingest(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error) {
+func (m *manager) Ingest(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error) {
l := logrus.WithFields(logrus.Fields{
"func": "Ingest",
"timelineAccountID": timelineAccountID,
- "statusID": status.ID,
+ "itemID": item.GetID(),
})
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
@@ -109,15 +109,15 @@ func (m *manager) Ingest(ctx context.Context, status *gtsmodel.Status, timelineA
return false, err
}
- l.Trace("ingesting status")
- return t.IndexOne(ctx, status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
+ l.Trace("ingesting item")
+ return t.IndexOne(ctx, item.GetID(), item.GetBoostOfID(), item.GetAccountID(), item.GetBoostOfAccountID())
}
-func (m *manager) IngestAndPrepare(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error) {
+func (m *manager) IngestAndPrepare(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error) {
l := logrus.WithFields(logrus.Fields{
"func": "IngestAndPrepare",
"timelineAccountID": timelineAccountID,
- "statusID": status.ID,
+ "itemID": item.GetID(),
})
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
@@ -125,15 +125,15 @@ func (m *manager) IngestAndPrepare(ctx context.Context, status *gtsmodel.Status,
return false, err
}
- l.Trace("ingesting status")
- return t.IndexAndPrepareOne(ctx, status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
+ l.Trace("ingesting item")
+ return t.IndexAndPrepareOne(ctx, item.GetID(), item.GetBoostOfID(), item.GetAccountID(), item.GetBoostOfAccountID())
}
-func (m *manager) Remove(ctx context.Context, timelineAccountID string, statusID string) (int, error) {
+func (m *manager) Remove(ctx context.Context, timelineAccountID string, itemID string) (int, error) {
l := logrus.WithFields(logrus.Fields{
"func": "Remove",
"timelineAccountID": timelineAccountID,
- "statusID": statusID,
+ "itemID": itemID,
})
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
@@ -141,13 +141,13 @@ func (m *manager) Remove(ctx context.Context, timelineAccountID string, statusID
return 0, err
}
- l.Trace("removing status")
- return t.Remove(ctx, statusID)
+ l.Trace("removing item")
+ return t.Remove(ctx, itemID)
}
-func (m *manager) HomeTimeline(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error) {
+func (m *manager) GetTimeline(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error) {
l := logrus.WithFields(logrus.Fields{
- "func": "HomeTimelineGet",
+ "func": "GetTimeline",
"timelineAccountID": timelineAccountID,
})
@@ -156,11 +156,11 @@ func (m *manager) HomeTimeline(ctx context.Context, timelineAccountID string, ma
return nil, err
}
- statuses, err := t.Get(ctx, limit, maxID, sinceID, minID, true)
+ items, err := t.Get(ctx, limit, maxID, sinceID, minID, true)
if err != nil {
l.Errorf("error getting statuses: %s", err)
}
- return statuses, nil
+ return items, nil
}
func (m *manager) GetIndexedLength(ctx context.Context, timelineAccountID string) int {
@@ -169,7 +169,7 @@ func (m *manager) GetIndexedLength(ctx context.Context, timelineAccountID string
return 0
}
- return t.PostIndexLength(ctx)
+ return t.ItemIndexLength(ctx)
}
func (m *manager) GetDesiredIndexLength(ctx context.Context) int {
@@ -182,7 +182,7 @@ func (m *manager) GetOldestIndexedID(ctx context.Context, timelineAccountID stri
return "", err
}
- return t.OldestIndexedPostID(ctx)
+ return t.OldestIndexedItemID(ctx)
}
func (m *manager) PrepareXFromTop(ctx context.Context, timelineAccountID string, limit int) error {
@@ -194,7 +194,7 @@ func (m *manager) PrepareXFromTop(ctx context.Context, timelineAccountID string,
return t.PrepareFromTop(ctx, limit)
}
-func (m *manager) WipeStatusFromAllTimelines(ctx context.Context, statusID string) error {
+func (m *manager) WipeItemFromAllTimelines(ctx context.Context, statusID string) error {
errors := []string{}
m.accountTimelines.Range(func(k interface{}, i interface{}) bool {
t, ok := i.(Timeline)
@@ -217,7 +217,7 @@ func (m *manager) WipeStatusFromAllTimelines(ctx context.Context, statusID strin
return err
}
-func (m *manager) WipeStatusesFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error {
+func (m *manager) WipeItemsFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error {
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return err
@@ -232,7 +232,7 @@ func (m *manager) getOrCreateTimeline(ctx context.Context, timelineAccountID str
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
var err error
- t, err = NewTimeline(ctx, timelineAccountID, m.db, m.tc)
+ t, err = NewTimeline(ctx, timelineAccountID, m.grabFunction, m.filterFunction, m.prepareFunction, m.skipInsertFunction)
if err != nil {
return nil, err
}
diff --git a/internal/timeline/manager_test.go b/internal/timeline/manager_test.go
index 4e81bfe17..11b5dfb78 100644
--- a/internal/timeline/manager_test.go
+++ b/internal/timeline/manager_test.go
@@ -23,6 +23,9 @@ import (
"testing"
"github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/timeline"
+ "github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -41,10 +44,16 @@ func (suite *ManagerTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
+ suite.filter = visibility.NewFilter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
- manager := testrig.NewTestTimelineManager(suite.db)
+ manager := timeline.NewManager(
+ processing.StatusGrabFunction(suite.db),
+ processing.StatusFilterFunction(suite.db, suite.filter),
+ processing.StatusPrepareFunction(suite.db, suite.tc),
+ processing.StatusSkipInsertFunction(),
+ )
suite.manager = manager
}
@@ -78,12 +87,12 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", oldestIndexed)
// get hometimeline
- statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
+ statuses, err := suite.manager.GetTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
suite.NoError(err)
suite.Len(statuses, 14)
// now wipe the last status from all timelines, as though it had been deleted by the owner
- err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
+ err = suite.manager.WipeItemFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
suite.NoError(err)
// timeline should be shorter
@@ -110,7 +119,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed)
// now remove all entries by local_account_2 from the timeline
- err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
+ err = suite.manager.WipeItemsFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
suite.NoError(err)
// timeline should be shorter
diff --git a/internal/timeline/preparable.go b/internal/timeline/preparable.go
new file mode 100644
index 000000000..c38acb450
--- /dev/null
+++ b/internal/timeline/preparable.go
@@ -0,0 +1,26 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package timeline
+
+type Preparable interface {
+ GetID() string
+ GetAccountID() string
+ GetBoostOfID() string
+ GetBoostOfAccountID() string
+}
diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go
index 4ddaad8a5..dae9031e5 100644
--- a/internal/timeline/prepare.go
+++ b/internal/timeline/prepare.go
@@ -26,7 +26,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
- "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID string, sinceID string, minID string) error {
@@ -59,19 +58,19 @@ func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID strin
return err
}
-func (t *timeline) PrepareBehind(ctx context.Context, statusID string, amount int) error {
- // lazily initialize prepared posts if it hasn't been done already
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
- t.preparedPosts.data.Init()
+func (t *timeline) PrepareBehind(ctx context.Context, itemID string, amount int) error {
+ // lazily initialize prepared items if it hasn't been done already
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
+ t.preparedItems.data.Init()
}
- if err := t.IndexBehind(ctx, statusID, true, amount); err != nil {
- return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", statusID, err)
+ if err := t.IndexBehind(ctx, itemID, amount); err != nil {
+ return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", itemID, err)
}
- // if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
- if t.postIndex.data == nil {
+ // if the itemindex is nil, nothing has been indexed yet so there's nothing to prepare
+ if t.itemIndex.data == nil {
return nil
}
@@ -80,25 +79,25 @@ func (t *timeline) PrepareBehind(ctx context.Context, statusID string, amount in
t.Lock()
defer t.Unlock()
prepareloop:
- for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*postIndexEntry)
+ for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
- return errors.New("PrepareBehind: could not parse e as a postIndexEntry")
+ return errors.New("PrepareBehind: could not parse e as itemIndexEntry")
}
if !preparing {
// we haven't hit the position we need to prepare from yet
- if entry.statusID == statusID {
+ if entry.itemID == itemID {
preparing = true
}
}
if preparing {
- if err := t.prepare(ctx, entry.statusID); err != nil {
+ if err := t.prepare(ctx, entry.itemID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
- return fmt.Errorf("PrepareBehind: error preparing status with id %s: %s", entry.statusID, err)
+ return fmt.Errorf("PrepareBehind: error preparing item with id %s: %s", entry.itemID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
@@ -119,28 +118,28 @@ func (t *timeline) PrepareBefore(ctx context.Context, statusID string, include b
defer t.Unlock()
// lazily initialize prepared posts if it hasn't been done already
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
- t.preparedPosts.data.Init()
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
+ t.preparedItems.data.Init()
}
// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
- if t.postIndex.data == nil {
+ if t.itemIndex.data == nil {
return nil
}
var prepared int
var preparing bool
prepareloop:
- for e := t.postIndex.data.Back(); e != nil; e = e.Prev() {
- entry, ok := e.Value.(*postIndexEntry)
+ for e := t.itemIndex.data.Back(); e != nil; e = e.Prev() {
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
}
if !preparing {
// we haven't hit the position we need to prepare from yet
- if entry.statusID == statusID {
+ if entry.itemID == statusID {
preparing = true
if !include {
continue
@@ -149,11 +148,11 @@ prepareloop:
}
if preparing {
- if err := t.prepare(ctx, entry.statusID); err != nil {
+ if err := t.prepare(ctx, entry.itemID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
- return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.statusID, err)
+ return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.itemID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
@@ -176,15 +175,15 @@ func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
})
// lazily initialize prepared posts if it hasn't been done already
- if t.preparedPosts.data == nil {
- t.preparedPosts.data = &list.List{}
- t.preparedPosts.data.Init()
+ if t.preparedItems.data == nil {
+ t.preparedItems.data = &list.List{}
+ t.preparedItems.data.Init()
}
// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
- if t.postIndex.data == nil {
+ if t.itemIndex.data == nil {
l.Debug("postindex.data was nil, indexing behind highest possible ID")
- if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", false, amount); err != nil {
+ if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", amount); err != nil {
return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err)
}
}
@@ -194,21 +193,21 @@ func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
defer t.Unlock()
var prepared int
prepareloop:
- for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+ for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
if e == nil {
continue
}
- entry, ok := e.Value.(*postIndexEntry)
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
}
- if err := t.prepare(ctx, entry.statusID); err != nil {
+ if err := t.prepare(ctx, entry.itemID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
- return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.statusID, err)
+ return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.itemID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
@@ -226,57 +225,42 @@ prepareloop:
return nil
}
-func (t *timeline) prepare(ctx context.Context, statusID string) error {
-
- // start by getting the status out of the database according to its indexed ID
- gtsStatus := >smodel.Status{}
- if err := t.db.GetByID(ctx, statusID, gtsStatus); err != nil {
- return err
- }
-
- // if the account pointer hasn't been set on this timeline already, set it lazily here
- if t.account == nil {
- timelineOwnerAccount := >smodel.Account{}
- if err := t.db.GetByID(ctx, t.accountID, timelineOwnerAccount); err != nil {
- return err
- }
- t.account = timelineOwnerAccount
- }
-
- // serialize the status (or, at least, convert it to a form that's ready to be serialized)
- apiModelStatus, err := t.tc.StatusToAPIStatus(ctx, gtsStatus, t.account)
+func (t *timeline) prepare(ctx context.Context, itemID string) error {
+ // trigger the caller-provided prepare function
+ prepared, err := t.prepareFunction(ctx, t.accountID, itemID)
if err != nil {
return err
}
- // shove it in prepared posts as a prepared posts entry
- preparedPostsEntry := &preparedPostsEntry{
- statusID: gtsStatus.ID,
- boostOfID: gtsStatus.BoostOfID,
- accountID: gtsStatus.AccountID,
- boostOfAccountID: gtsStatus.BoostOfAccountID,
- prepared: apiModelStatus,
+ // shove it in prepared items as a prepared items entry
+ preparedItemsEntry := &preparedItemsEntry{
+ itemID: prepared.GetID(),
+ boostOfID: prepared.GetBoostOfID(),
+ accountID: prepared.GetAccountID(),
+ boostOfAccountID: prepared.GetBoostOfAccountID(),
+ prepared: prepared,
}
- return t.preparedPosts.insertPrepared(preparedPostsEntry)
+ return t.preparedItems.insertPrepared(ctx, preparedItemsEntry)
}
-func (t *timeline) OldestPreparedPostID(ctx context.Context) (string, error) {
+func (t *timeline) OldestPreparedItemID(ctx context.Context) (string, error) {
var id string
- if t.preparedPosts == nil || t.preparedPosts.data == nil {
- // return an empty string if prepared posts hasn't been initialized yet
+ if t.preparedItems == nil || t.preparedItems.data == nil {
+ // return an empty string if prepared items hasn't been initialized yet
return id, nil
}
- e := t.preparedPosts.data.Back()
+ e := t.preparedItems.data.Back()
if e == nil {
// return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
return id, nil
}
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
- return id, errors.New("OldestPreparedPostID: could not parse e as a preparedPostsEntry")
+ return id, errors.New("OldestPreparedItemID: could not parse e as a preparedItemsEntry")
}
- return entry.statusID, nil
+
+ return entry.itemID, nil
}
diff --git a/internal/timeline/preparedposts.go b/internal/timeline/prepareditems.go
similarity index 71%
rename from internal/timeline/preparedposts.go
rename to internal/timeline/prepareditems.go
index 54e0e61f3..07a8c69ee 100644
--- a/internal/timeline/preparedposts.go
+++ b/internal/timeline/prepareditems.go
@@ -20,24 +20,24 @@ package timeline
import (
"container/list"
+ "context"
"errors"
-
- apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
-type preparedPosts struct {
- data *list.List
+type preparedItems struct {
+ data *list.List
+ skipInsert SkipInsertFunction
}
-type preparedPostsEntry struct {
- statusID string
+type preparedItemsEntry struct {
+ itemID string
boostOfID string
accountID string
boostOfAccountID string
- prepared *apimodel.Status
+ prepared Preparable
}
-func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
+func (p *preparedItems) insertPrepared(ctx context.Context, i *preparedItemsEntry) error {
if p.data == nil {
p.data = &list.List{}
}
@@ -55,35 +55,28 @@ func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
for e := p.data.Front(); e != nil; e = e.Next() {
position++
- entry, ok := e.Value.(*preparedPostsEntry)
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return errors.New("index: could not parse e as a preparedPostsEntry")
}
- // don't insert this if it's a boost of a status we've seen recently
- if i.prepared.Reblog != nil {
- if entry.prepared.Reblog != nil && i.prepared.Reblog.ID == entry.prepared.Reblog.ID {
- if position < boostReinsertionDepth {
- return nil
- }
- }
-
- if i.prepared.Reblog.ID == entry.statusID {
- if position < boostReinsertionDepth {
- return nil
- }
- }
+ skip, err := p.skipInsert(ctx, i.itemID, i.accountID, i.boostOfID, i.boostOfAccountID, entry.itemID, entry.accountID, entry.boostOfID, entry.boostOfAccountID, position)
+ if err != nil {
+ return err
+ }
+ if skip {
+ return nil
}
// if the post to index is newer than e, insert it before e in the list
if insertMark == nil {
- if i.statusID > entry.statusID {
+ if i.itemID > entry.itemID {
insertMark = e
}
}
// make sure we don't insert a duplicate
- if entry.statusID == i.statusID {
+ if entry.itemID == i.itemID {
return nil
}
}
diff --git a/internal/timeline/remove.go b/internal/timeline/remove.go
index 833d7126a..60d8108ec 100644
--- a/internal/timeline/remove.go
+++ b/internal/timeline/remove.go
@@ -38,39 +38,39 @@ func (t *timeline) Remove(ctx context.Context, statusID string) (int, error) {
// remove entr(ies) from the post index
removeIndexes := []*list.Element{}
- if t.postIndex != nil && t.postIndex.data != nil {
- for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*postIndexEntry)
+ if t.itemIndex != nil && t.itemIndex.data != nil {
+ for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a postIndexEntry")
}
- if entry.statusID == statusID {
+ if entry.itemID == statusID {
l.Debug("found status in postIndex")
removeIndexes = append(removeIndexes, e)
}
}
}
for _, e := range removeIndexes {
- t.postIndex.data.Remove(e)
+ t.itemIndex.data.Remove(e)
removed++
}
// remove entr(ies) from prepared posts
removePrepared := []*list.Element{}
- if t.preparedPosts != nil && t.preparedPosts.data != nil {
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ if t.preparedItems != nil && t.preparedItems.data != nil {
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
}
- if entry.statusID == statusID {
+ if entry.itemID == statusID {
l.Debug("found status in preparedPosts")
removePrepared = append(removePrepared, e)
}
}
}
for _, e := range removePrepared {
- t.preparedPosts.data.Remove(e)
+ t.preparedItems.data.Remove(e)
removed++
}
@@ -90,9 +90,9 @@ func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, erro
// remove entr(ies) from the post index
removeIndexes := []*list.Element{}
- if t.postIndex != nil && t.postIndex.data != nil {
- for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*postIndexEntry)
+ if t.itemIndex != nil && t.itemIndex.data != nil {
+ for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a postIndexEntry")
}
@@ -103,15 +103,15 @@ func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, erro
}
}
for _, e := range removeIndexes {
- t.postIndex.data.Remove(e)
+ t.itemIndex.data.Remove(e)
removed++
}
// remove entr(ies) from prepared posts
removePrepared := []*list.Element{}
- if t.preparedPosts != nil && t.preparedPosts.data != nil {
- for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedPostsEntry)
+ if t.preparedItems != nil && t.preparedItems.data != nil {
+ for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
}
@@ -122,7 +122,7 @@ func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, erro
}
}
for _, e := range removePrepared {
- t.preparedPosts.data.Remove(e)
+ t.preparedItems.data.Remove(e)
removed++
}
diff --git a/internal/timeline/timeline.go b/internal/timeline/timeline.go
index fc4875d1a..1d82914f8 100644
--- a/internal/timeline/timeline.go
+++ b/internal/timeline/timeline.go
@@ -21,104 +21,135 @@ package timeline
import (
"context"
"sync"
- "time"
-
- apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
- "github.com/superseriousbusiness/gotosocial/internal/db"
- "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
- "github.com/superseriousbusiness/gotosocial/internal/typeutils"
- "github.com/superseriousbusiness/gotosocial/internal/visibility"
)
-const boostReinsertionDepth = 50
+// GrabFunction is used by a Timeline to grab more items to index.
+//
+// It should be provided to NewTimeline when the caller is creating a timeline
+// (of statuses, notifications, etc).
+//
+// timelineAccountID: the owner of the timeline
+// maxID: the maximum item ID desired.
+// sinceID: the minimum item ID desired.
+// minID: see sinceID
+// limit: the maximum amount of items to be returned
+//
+// If an error is returned, the timeline will stop processing whatever request called GrabFunction,
+// and return the error. If no error is returned, but stop = true, this indicates to the caller of GrabFunction
+// that there are no more items to return, and processing should continue with the items already grabbed.
+type GrabFunction func(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int) (items []Timelineable, stop bool, err error)
-// Timeline represents a timeline for one account, and contains indexed and prepared posts.
+// FilterFunction is used by a Timeline to filter whether or not a grabbed item should be indexed.
+type FilterFunction func(ctx context.Context, timelineAccountID string, item Timelineable) (shouldIndex bool, err error)
+
+// PrepareFunction converts a Timelineable into a Preparable.
+//
+// For example, this might result in the converstion of a *gtsmodel.Status with the given itemID into a serializable *apimodel.Status.
+type PrepareFunction func(ctx context.Context, timelineAccountID string, itemID string) (Preparable, error)
+
+// SkipInsertFunction indicates whether a new item about to be inserted in the prepared list should be skipped,
+// based on the item itself, the next item in the timeline, and the depth at which nextItem has been found in the list.
+//
+// This will be called for every item found while iterating through a timeline, so callers should be very careful
+// not to do anything expensive here.
+type SkipInsertFunction func(ctx context.Context,
+ newItemID string,
+ newItemAccountID string,
+ newItemBoostOfID string,
+ newItemBoostOfAccountID string,
+ nextItemID string,
+ nextItemAccountID string,
+ nextItemBoostOfID string,
+ nextItemBoostOfAccountID string,
+ depth int) (bool, error)
+
+// Timeline represents a timeline for one account, and contains indexed and prepared items.
type Timeline interface {
/*
RETRIEVAL FUNCTIONS
*/
- // Get returns an amount of statuses with the given parameters.
+ // Get returns an amount of prepared items with the given parameters.
// If prepareNext is true, then the next predicted query will be prepared already in a goroutine,
// to make the next call to Get faster.
- Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error)
- // GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest.
- GetXFromTop(ctx context.Context, amount int) ([]*apimodel.Status, error)
- // GetXBehindID returns x amount of posts from the given id onwards, from newest to oldest.
- // This will NOT include the status with the given ID.
+ Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error)
+ // GetXFromTop returns x amount of items from the top of the timeline, from newest to oldest.
+ GetXFromTop(ctx context.Context, amount int) ([]Preparable, error)
+ // GetXBehindID returns x amount of items from the given id onwards, from newest to oldest.
+ // This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?max_id=WHATEVER
- GetXBehindID(ctx context.Context, amount int, fromID string, attempts *int) ([]*apimodel.Status, error)
- // GetXBeforeID returns x amount of posts up to the given id, from newest to oldest.
- // This will NOT include the status with the given ID.
+ GetXBehindID(ctx context.Context, amount int, fromID string, attempts *int) ([]Preparable, error)
+ // GetXBeforeID returns x amount of items up to the given id, from newest to oldest.
+ // This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER
- GetXBeforeID(ctx context.Context, amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
- // GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest.
- // This will NOT include the status with the given IDs.
+ GetXBeforeID(ctx context.Context, amount int, sinceID string, startFromTop bool) ([]Preparable, error)
+ // GetXBetweenID returns x amount of items from the given maxID, up to the given id, from newest to oldest.
+ // This will NOT include the item with the given IDs.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
- GetXBetweenID(ctx context.Context, amount int, maxID string, sinceID string) ([]*apimodel.Status, error)
+ GetXBetweenID(ctx context.Context, amount int, maxID string, sinceID string) ([]Preparable, error)
/*
INDEXING FUNCTIONS
*/
- // IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property.
+ // IndexOne puts a item into the timeline at the appropriate place according to its 'createdAt' property.
//
- // The returned bool indicates whether or not the status was actually inserted into the timeline. This will be false
- // if the status is a boost and the original post or another boost of it already exists < boostReinsertionDepth back in the timeline.
- IndexOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
+ // The returned bool indicates whether or not the item was actually inserted into the timeline. This will be false
+ // if the item is a boost and the original item or another boost of it already exists < boostReinsertionDepth back in the timeline.
+ IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
- // OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
- // If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
- OldestIndexedPostID(ctx context.Context) (string, error)
- // NewestIndexedPostID returns the id of the frontmost (ie., the newest) indexed post, or an error if something goes wrong.
- // If nothing goes wrong but there's no newest post, an empty string will be returned so make sure to check for this.
- NewestIndexedPostID(ctx context.Context) (string, error)
+ // OldestIndexedItemID returns the id of the rearmost (ie., the oldest) indexed item, or an error if something goes wrong.
+ // If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
+ OldestIndexedItemID(ctx context.Context) (string, error)
+ // NewestIndexedItemID returns the id of the frontmost (ie., the newest) indexed item, or an error if something goes wrong.
+ // If nothing goes wrong but there's no newest item, an empty string will be returned so make sure to check for this.
+ NewestIndexedItemID(ctx context.Context) (string, error)
- IndexBefore(ctx context.Context, statusID string, include bool, amount int) error
- IndexBehind(ctx context.Context, statusID string, include bool, amount int) error
+ IndexBefore(ctx context.Context, itemID string, amount int) error
+ IndexBehind(ctx context.Context, itemID string, amount int) error
/*
PREPARATION FUNCTIONS
*/
- // PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline.
+ // PrepareXFromTop instructs the timeline to prepare x amount of items from the top of the timeline.
PrepareFromTop(ctx context.Context, amount int) error
// PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
- // If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared.
- PrepareBehind(ctx context.Context, statusID string, amount int) error
- // IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property,
+ // If include is true, then the given item ID will also be prepared, otherwise only entries behind it will be prepared.
+ PrepareBehind(ctx context.Context, itemID string, amount int) error
+ // IndexOne puts a item into the timeline at the appropriate place according to its 'createdAt' property,
// and then immediately prepares it.
//
- // The returned bool indicates whether or not the status was actually inserted into the timeline. This will be false
- // if the status is a boost and the original post or another boost of it already exists < boostReinsertionDepth back in the timeline.
- IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
- // OldestPreparedPostID returns the id of the rearmost (ie., the oldest) prepared post, or an error if something goes wrong.
- // If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
- OldestPreparedPostID(ctx context.Context) (string, error)
+ // The returned bool indicates whether or not the item was actually inserted into the timeline. This will be false
+ // if the item is a boost and the original item or another boost of it already exists < boostReinsertionDepth back in the timeline.
+ IndexAndPrepareOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
+ // OldestPreparedItemID returns the id of the rearmost (ie., the oldest) prepared item, or an error if something goes wrong.
+ // If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
+ OldestPreparedItemID(ctx context.Context) (string, error)
/*
INFO FUNCTIONS
*/
- // ActualPostIndexLength returns the actual length of the post index at this point in time.
- PostIndexLength(ctx context.Context) int
+ // ActualPostIndexLength returns the actual length of the item index at this point in time.
+ ItemIndexLength(ctx context.Context) int
/*
UTILITY FUNCTIONS
*/
- // Reset instructs the timeline to reset to its base state -- cache only the minimum amount of posts.
+ // Reset instructs the timeline to reset to its base state -- cache only the minimum amount of items.
Reset() error
- // Remove removes a status from both the index and prepared posts.
+ // Remove removes a item from both the index and prepared items.
//
- // If a status has multiple entries in a timeline, they will all be removed.
+ // If a item has multiple entries in a timeline, they will all be removed.
//
// The returned int indicates the amount of entries that were removed.
- Remove(ctx context.Context, statusID string) (int, error)
- // RemoveAllBy removes all statuses by the given accountID, from both the index and prepared posts.
+ Remove(ctx context.Context, itemID string) (int, error)
+ // RemoveAllBy removes all items by the given accountID, from both the index and prepared items.
//
// The returned int indicates the amount of entries that were removed.
RemoveAllBy(ctx context.Context, accountID string) (int, error)
@@ -126,31 +157,34 @@ type Timeline interface {
// timeline fulfils the Timeline interface
type timeline struct {
- postIndex *postIndex
- preparedPosts *preparedPosts
- accountID string
- account *gtsmodel.Account
- db db.DB
- filter visibility.Filter
- tc typeutils.TypeConverter
+ itemIndex *itemIndex
+ preparedItems *preparedItems
+ grabFunction GrabFunction
+ filterFunction FilterFunction
+ prepareFunction PrepareFunction
+ accountID string
sync.Mutex
}
// NewTimeline returns a new Timeline for the given account ID
-func NewTimeline(ctx context.Context, accountID string, db db.DB, typeConverter typeutils.TypeConverter) (Timeline, error) {
- timelineOwnerAccount := >smodel.Account{}
- if err := db.GetByID(ctx, accountID, timelineOwnerAccount); err != nil {
- return nil, err
- }
-
+func NewTimeline(
+ ctx context.Context,
+ timelineAccountID string,
+ grabFunction GrabFunction,
+ filterFunction FilterFunction,
+ prepareFunction PrepareFunction,
+ skipInsertFunction SkipInsertFunction) (Timeline, error) {
return &timeline{
- postIndex: &postIndex{},
- preparedPosts: &preparedPosts{},
- accountID: accountID,
- account: timelineOwnerAccount,
- db: db,
- filter: visibility.NewFilter(db),
- tc: typeConverter,
+ itemIndex: &itemIndex{
+ skipInsert: skipInsertFunction,
+ },
+ preparedItems: &preparedItems{
+ skipInsert: skipInsertFunction,
+ },
+ grabFunction: grabFunction,
+ filterFunction: filterFunction,
+ prepareFunction: prepareFunction,
+ accountID: timelineAccountID,
}, nil
}
@@ -158,10 +192,10 @@ func (t *timeline) Reset() error {
return nil
}
-func (t *timeline) PostIndexLength(ctx context.Context) int {
- if t.postIndex == nil || t.postIndex.data == nil {
+func (t *timeline) ItemIndexLength(ctx context.Context) int {
+ if t.itemIndex == nil || t.itemIndex.data == nil {
return 0
}
- return t.postIndex.data.Len()
+ return t.itemIndex.data.Len()
}
diff --git a/internal/timeline/timeline_test.go b/internal/timeline/timeline_test.go
index 96a938b07..ef6b66535 100644
--- a/internal/timeline/timeline_test.go
+++ b/internal/timeline/timeline_test.go
@@ -24,12 +24,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+ "github.com/superseriousbusiness/gotosocial/internal/visibility"
)
type TimelineStandardTestSuite struct {
suite.Suite
- db db.DB
- tc typeutils.TypeConverter
+ db db.DB
+ tc typeutils.TypeConverter
+ filter visibility.Filter
testAccounts map[string]*gtsmodel.Account
testStatuses map[string]*gtsmodel.Status
diff --git a/internal/timeline/timelineable.go b/internal/timeline/timelineable.go
new file mode 100644
index 000000000..cf2d06775
--- /dev/null
+++ b/internal/timeline/timelineable.go
@@ -0,0 +1,27 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package timeline
+
+// Timelineable represents any item that can be put in a timeline.
+type Timelineable interface {
+ GetID() string
+ GetAccountID() string
+ GetBoostOfID() string
+ GetBoostOfAccountID() string
+}
diff --git a/testrig/processor.go b/testrig/processor.go
index 0baffd35b..cfcaa612b 100644
--- a/testrig/processor.go
+++ b/testrig/processor.go
@@ -28,5 +28,5 @@ import (
// NewTestProcessor returns a Processor suitable for testing purposes
func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender) processing.Processor {
- return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, NewTestTimelineManager(db), db, emailSender)
+ return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, db, emailSender)
}
diff --git a/testrig/timelinemanager.go b/testrig/timelinemanager.go
deleted file mode 100644
index 4b316a52e..000000000
--- a/testrig/timelinemanager.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package testrig
-
-import (
- "github.com/superseriousbusiness/gotosocial/internal/db"
- "github.com/superseriousbusiness/gotosocial/internal/timeline"
-)
-
-// NewTestTimelineManager retuts a new timeline.Manager, suitable for testing, using the given db.
-func NewTestTimelineManager(db db.DB) timeline.Manager {
- return timeline.NewManager(db, NewTestTypeConverter(db))
-}