SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

Timeline loop fix (#140)

Browse Source 
* uwu we made a fucky wucky

* uwu we made a fucky wucky

* work on timeline fixes a little

* fiddle with tests some more

* bleep bloop more tests

* more tests

* update drone yml

* update some sturf

* make the timeline code a bit lazier

* go fmt

* fix drone.yml
This commit is contained in:
Tobi Smethurst
2021-08-15 18:43:08 +02:00
committed by GitHub
parent a4a33b9ad9
commit ff406be68f
13 changed files with 1035 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/processing/processor.go
View File

@@ -304,7 +304,7 @@ func (p *processor) Start() error {
}
}
}()
return p.initTimelines()
return nil
}
// Stop stops the processor cleanly, finishing handling any remaining messages before closing down.

102
internal/processing/timeline.go
View File

@@ -21,9 +21,7 @@ package processing
import (
"fmt"
"net/url"
"sync"
"github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -186,103 +184,3 @@ func (p *processor) filterFavedStatuses(authed *oauth.Auth, statuses []*gtsmodel
return apiStatuses, nil
}
func (p *processor) initTimelines() error {
// get all local accounts (ie., domain = nil) that aren't suspended (suspended_at = nil)
localAccounts := []*gtsmodel.Account{}
where := []db.Where{
{
Key: "domain", Value: nil,
},
{
Key: "suspended_at", Value: nil,
},
}
if err := p.db.GetWhere(where, &localAccounts); err != nil {
if _, ok := err.(db.ErrNoEntries); ok {
return nil
}
return fmt.Errorf("initTimelines: db error initializing timelines: %s", err)
}
// we want to wait until all timelines are populated so created a waitgroup here
wg := &sync.WaitGroup{}
wg.Add(len(localAccounts))
for _, localAccount := range localAccounts {
// to save time we can populate the timelines asynchronously
// this will go heavy on the database, but since we're not actually serving yet it doesn't really matter
go p.initTimelineFor(localAccount, wg)
}
// wait for all timelines to be populated before we exit
wg.Wait()
return nil
}
func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGroup) {
defer wg.Done()
l := p.log.WithFields(logrus.Fields{
"func": "initTimelineFor",
"accountID": account.ID,
})
desiredIndexLength := p.timelineManager.GetDesiredIndexLength()
statuses, err := p.db.GetHomeTimelineForAccount(account.ID, "", "", "", desiredIndexLength, false)
if err != nil {
if _, ok := err.(db.ErrNoEntries); !ok {
l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err))
}
return
}
p.indexAndIngest(statuses, account, desiredIndexLength)
lengthNow := p.timelineManager.GetIndexedLength(account.ID)
if lengthNow < desiredIndexLength {
// try and get more posts from the last ID onwards
rearmostStatusID, err := p.timelineManager.GetOldestIndexedID(account.ID)
if err != nil {
l.Error(fmt.Errorf("initTimelineFor: error getting id of rearmost status: %s", err))
return
}
if rearmostStatusID != "" {
moreStatuses, err := p.db.GetHomeTimelineForAccount(account.ID, rearmostStatusID, "", "", desiredIndexLength/2, false)
if err != nil {
l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err))
return
}
p.indexAndIngest(moreStatuses, account, desiredIndexLength)
}
}
l.Debugf("prepared timeline of length %d for account %s", lengthNow, account.ID)
}
func (p *processor) indexAndIngest(statuses []*gtsmodel.Status, timelineAccount *gtsmodel.Account, desiredIndexLength int) {
l := p.log.WithFields(logrus.Fields{
"func": "indexAndIngest",
"accountID": timelineAccount.ID,
})
for _, s := range statuses {
timelineable, err := p.filter.StatusHometimelineable(s, timelineAccount)
if err != nil {
l.Error(fmt.Errorf("initTimelineFor: error checking home timelineability of status %s: %s", s.ID, err))
continue
}
if timelineable {
if _, err := p.timelineManager.Ingest(s, timelineAccount.ID); err != nil {
l.Error(fmt.Errorf("initTimelineFor: error ingesting status %s: %s", s.ID, err))
continue
}
// check if we have enough posts now and return if we do
if p.timelineManager.GetIndexedLength(timelineAccount.ID) >= desiredIndexLength {
return
}
}
}
}