SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

Timeline manager (#40)

Browse Source 
* start messing about with timeline manager

* i have no idea what i'm doing

* i continue to not know what i'm doing

* it's coming along

* bit more progress

* update timeline with new posts as they come in

* lint and fmt

* Select accounts where empty string

* restructure a bunch, get unfaves working

* moving stuff around

* federate status deletes properly

* mention regex better but not 100% there

* fix regex

* some more hacking away at the timeline code phew

* fix up some little things

* i can't even

* more timeline stuff

* move to ulid

* fiddley

* some lil fixes for kibou compatibility

* timelines working pretty alright!

* tidy + lint
This commit is contained in:
Tobi Smethurst
2021-06-13 18:42:28 +02:00
committed by GitHub
parent 6ac6f8d614
commit b4288f3c47
96 changed files with 3458 additions and 1679 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
internal/timeline/get.go Normal file
View File

@@ -0,0 +1,309 @@
package timeline
import (
"container/list"
"errors"
"fmt"
"github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
func (t *timeline) Get(amount int, maxID string, sinceID string, minID string) ([]*apimodel.Status, error) {
l := t.log.WithFields(logrus.Fields{
"func": "Get",
"accountID": t.accountID,
})
var statuses []*apimodel.Status
var err error
// no params are defined to just fetch from the top
if maxID == "" && sinceID == "" && minID == "" {
statuses, err = t.GetXFromTop(amount)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(statuses) != 0 {
nextMaxID := statuses[len(statuses)-1].ID
go func() {
if err := t.prepareNextQuery(amount, nextMaxID, "", ""); err != nil {
l.Errorf("error preparing next query: %s", err)
}
}()
}
}
// maxID is defined but sinceID isn't so take from behind
if maxID != "" && sinceID == "" {
statuses, err = t.GetXBehindID(amount, maxID)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(statuses) != 0 {
nextMaxID := statuses[len(statuses)-1].ID
go func() {
if err := t.prepareNextQuery(amount, nextMaxID, "", ""); err != nil {
l.Errorf("error preparing next query: %s", err)
}
}()
}
}
// maxID is defined and sinceID || minID are as well, so take a slice between them
if maxID != "" && sinceID != "" {
statuses, err = t.GetXBetweenID(amount, maxID, minID)
}
if maxID != "" && minID != "" {
statuses, err = t.GetXBetweenID(amount, maxID, minID)
}
// maxID isn't defined, but sinceID || minID are, so take x before
if maxID == "" && sinceID != "" {
statuses, err = t.GetXBeforeID(amount, sinceID, true)
}
if maxID == "" && minID != "" {
statuses, err = t.GetXBeforeID(amount, minID, true)
}
return statuses, err
}
func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
}
// make sure we have enough posts prepared to return
if t.preparedPosts.data.Len() < amount {
if err := t.PrepareFromTop(amount); err != nil {
return nil, err
}
}
// work through the prepared posts from the top and return
var served int
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry")
}
statuses = append(statuses, entry.prepared)
served = served + 1
if served >= amount {
break
}
}
return statuses, nil
}
func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
var position int
var behindIDMark *list.Element
findMarkLoop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
position = position + 1
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == behindID {
behindIDMark = e
break findMarkLoop
}
}
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
if behindIDMark == nil {
if err := t.IndexBehind(behindID, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID)
}
if err := t.PrepareBehind(behindID, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
}
oldestID, err := t.OldestPreparedPostID()
if err != nil {
return nil, err
}
if oldestID == "" || oldestID == behindID {
// there is no oldest prepared post, or the oldest prepared post is still the post we're looking for entries after
// this means we should just return the empty statuses slice since we don't have any more posts to offer
return statuses, nil
}
return t.GetXBehindID(amount, behindID)
}
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
if err := t.PrepareBehind(behindID, amount); err != nil {
return nil, err
}
}
// start serving from the entry right after the mark
var served int
serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
served = served + 1
if served >= amount {
break serveloop
}
}
return statuses, nil
}
func (t *timeline) GetXBeforeID(amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
var beforeIDMark *list.Element
findMarkLoop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == beforeID {
beforeIDMark = e
break findMarkLoop
}
}
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
if beforeIDMark == nil {
if err := t.IndexBefore(beforeID, true, amount); err != nil {
return nil, fmt.Errorf("GetXBeforeID: error indexing before and including ID %s", beforeID)
}
if err := t.PrepareBefore(beforeID, true, amount); err != nil {
return nil, fmt.Errorf("GetXBeforeID: error preparing before and including ID %s", beforeID)
}
return t.GetXBeforeID(amount, beforeID, startFromTop)
}
var served int
if startFromTop {
// start serving from the front/top and keep going until we hit mark or get x amount statuses
serveloopFromTop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == beforeID {
break serveloopFromTop
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
served = served + 1
if served >= amount {
break serveloopFromTop
}
}
} else if !startFromTop {
// start serving from the entry right before the mark
serveloopFromBottom:
for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
served = served + 1
if served >= amount {
break serveloopFromBottom
}
}
}
return statuses, nil
}
func (t *timeline) GetXBetweenID(amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
var position int
var behindIDMark *list.Element
findMarkLoop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
position = position + 1
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == behindID {
behindIDMark = e
break findMarkLoop
}
}
// we didn't find it
if behindIDMark == nil {
return nil, fmt.Errorf("GetXBetweenID: couldn't find status with ID %s", behindID)
}
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
if err := t.PrepareBehind(behindID, amount); err != nil {
return nil, err
}
}
// start serving from the entry right after the mark
var served int
serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == beforeID {
break serveloop
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
served = served + 1
if served >= amount {
break serveloop
}
}
return statuses, nil
}

143
internal/timeline/index.go Normal file
View File

@@ -0,0 +1,143 @@
package timeline
import (
"errors"
"fmt"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) IndexBefore(statusID string, include bool, amount int) error {
// filtered := []*gtsmodel.Status{}
// offsetStatus := statusID
// grabloop:
// for len(filtered) < amount {
// statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, amount, offsetStatus, include, true)
// if err != nil {
// if _, ok := err.(db.ErrNoEntries); !ok {
// return fmt.Errorf("IndexBeforeAndIncluding: error getting statuses from db: %s", err)
// }
// break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
// }
// for _, s := range statuses {
// relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s)
// if err != nil {
// continue
// }
// visible, err := t.db.StatusVisible(s, t.account, relevantAccounts)
// if err != nil {
// continue
// }
// if visible {
// filtered = append(filtered, s)
// }
// offsetStatus = s.ID
// }
// }
// for _, s := range filtered {
// if err := t.IndexOne(s.CreatedAt, s.ID); err != nil {
// return fmt.Errorf("IndexBeforeAndIncluding: error indexing status with id %s: %s", s.ID, err)
// }
// }
return nil
}
func (t *timeline) IndexBehind(statusID string, amount int) error {
filtered := []*gtsmodel.Status{}
offsetStatus := statusID
grabloop:
for len(filtered) < amount {
statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, offsetStatus, "", "", amount, false)
if err != nil {
if _, ok := err.(db.ErrNoEntries); ok {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
}
return fmt.Errorf("IndexBehindAndIncluding: error getting statuses from db: %s", err)
}
for _, s := range statuses {
relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s)
if err != nil {
continue
}
visible, err := t.db.StatusVisible(s, t.account, relevantAccounts)
if err != nil {
continue
}
if visible {
filtered = append(filtered, s)
}
offsetStatus = s.ID
}
}
for _, s := range filtered {
if err := t.IndexOne(s.CreatedAt, s.ID); err != nil {
return fmt.Errorf("IndexBehindAndIncluding: error indexing status with id %s: %s", s.ID, err)
}
}
return nil
}
func (t *timeline) IndexOneByID(statusID string) error {
return nil
}
func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error {
t.Lock()
defer t.Unlock()
postIndexEntry := &postIndexEntry{
statusID: statusID,
}
return t.postIndex.insertIndexed(postIndexEntry)
}
func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error {
t.Lock()
defer t.Unlock()
postIndexEntry := &postIndexEntry{
statusID: statusID,
}
if err := t.postIndex.insertIndexed(postIndexEntry); err != nil {
return fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
}
if err := t.prepare(statusID); err != nil {
return fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
}
return nil
}
func (t *timeline) OldestIndexedPostID() (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
e := t.postIndex.data.Back()
if e == nil {
// return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
return id, nil
}
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry")
}
return entry.statusID, nil
}

217
internal/timeline/manager.go Normal file
View File

@@ -0,0 +1,217 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
import (
"fmt"
"strings"
"sync"
"github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
const (
desiredPostIndexLength = 400
)
// Manager abstracts functions for creating timelines for multiple accounts, and adding, removing, and fetching entries from those timelines.
//
// By the time a status hits the manager interface, it should already have been filtered and it should be established that the status indeed
// belongs in the home timeline of the given account ID.
//
// The manager makes a distinction between *indexed* posts and *prepared* posts.
//
// Indexed posts consist of just that post's ID (in the database) and the time it was created. An indexed post takes up very little memory, so
// it's not a huge priority to keep trimming the indexed posts list.
//
// Prepared posts consist of the post's database ID, the time it was created, AND the apimodel representation of that post, for quick serialization.
// Prepared posts of course take up more memory than indexed posts, so they should be regularly pruned if they're not being actively served.
type Manager interface {
// Ingest takes one status and indexes it into the timeline for the given account ID.
//
// It should already be established before calling this function that the status/post actually belongs in the timeline!
Ingest(status *gtsmodel.Status, timelineAccountID string) error
// IngestAndPrepare takes one status and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
// This is useful in cases where we know the status will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
//
// It should already be established before calling this function that the status/post actually belongs in the timeline!
IngestAndPrepare(status *gtsmodel.Status, timelineAccountID string) error
// HomeTimeline returns limit n amount of entries from the home timeline of the given account ID, in descending chronological order.
// If maxID is provided, it will return entries from that maxID onwards, inclusive.
HomeTimeline(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
// GetIndexedLength returns the amount of posts/statuses that have been *indexed* for the given account ID.
GetIndexedLength(timelineAccountID string) int
// GetDesiredIndexLength returns the amount of posts that we, ideally, index for each user.
GetDesiredIndexLength() int
// GetOldestIndexedID returns the status ID for the oldest post that we have indexed for the given account.
GetOldestIndexedID(timelineAccountID string) (string, error)
// PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
PrepareXFromTop(timelineAccountID string, limit int) error
// WipeStatusFromTimeline completely removes a status and from the index and prepared posts of the given account ID
//
// The returned int indicates how many entries were removed.
WipeStatusFromTimeline(timelineAccountID string, statusID string) (int, error)
// WipeStatusFromAllTimelines removes the status from the index and prepared posts of all timelines
WipeStatusFromAllTimelines(statusID string) error
}
// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
func NewManager(db db.DB, tc typeutils.TypeConverter, config *config.Config, log *logrus.Logger) Manager {
return &manager{
accountTimelines: sync.Map{},
db: db,
tc: tc,
config: config,
log: log,
}
}
type manager struct {
accountTimelines sync.Map
db db.DB
tc typeutils.TypeConverter
config *config.Config
log *logrus.Logger
}
func (m *manager) Ingest(status *gtsmodel.Status, timelineAccountID string) error {
l := m.log.WithFields(logrus.Fields{
"func": "Ingest",
"timelineAccountID": timelineAccountID,
"statusID": status.ID,
})
t := m.getOrCreateTimeline(timelineAccountID)
l.Trace("ingesting status")
return t.IndexOne(status.CreatedAt, status.ID)
}
func (m *manager) IngestAndPrepare(status *gtsmodel.Status, timelineAccountID string) error {
l := m.log.WithFields(logrus.Fields{
"func": "IngestAndPrepare",
"timelineAccountID": timelineAccountID,
"statusID": status.ID,
})
t := m.getOrCreateTimeline(timelineAccountID)
l.Trace("ingesting status")
return t.IndexAndPrepareOne(status.CreatedAt, status.ID)
}
func (m *manager) Remove(statusID string, timelineAccountID string) (int, error) {
l := m.log.WithFields(logrus.Fields{
"func": "Remove",
"timelineAccountID": timelineAccountID,
"statusID": statusID,
})
t := m.getOrCreateTimeline(timelineAccountID)
l.Trace("removing status")
return t.Remove(statusID)
}
func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error) {
l := m.log.WithFields(logrus.Fields{
"func": "HomeTimelineGet",
"timelineAccountID": timelineAccountID,
})
t := m.getOrCreateTimeline(timelineAccountID)
statuses, err := t.Get(limit, maxID, sinceID, minID)
if err != nil {
l.Errorf("error getting statuses: %s", err)
}
return statuses, nil
}
func (m *manager) GetIndexedLength(timelineAccountID string) int {
t := m.getOrCreateTimeline(timelineAccountID)
return t.PostIndexLength()
}
func (m *manager) GetDesiredIndexLength() int {
return desiredPostIndexLength
}
func (m *manager) GetOldestIndexedID(timelineAccountID string) (string, error) {
t := m.getOrCreateTimeline(timelineAccountID)
return t.OldestIndexedPostID()
}
func (m *manager) PrepareXFromTop(timelineAccountID string, limit int) error {
t := m.getOrCreateTimeline(timelineAccountID)
return t.PrepareFromTop(limit)
}
func (m *manager) WipeStatusFromTimeline(timelineAccountID string, statusID string) (int, error) {
t := m.getOrCreateTimeline(timelineAccountID)
return t.Remove(statusID)
}
func (m *manager) WipeStatusFromAllTimelines(statusID string) error {
errors := []string{}
m.accountTimelines.Range(func(k interface{}, i interface{}) bool {
t, ok := i.(Timeline)
if !ok {
panic("couldn't parse entry as Timeline, this should never happen so panic")
}
if _, err := t.Remove(statusID); err != nil {
errors = append(errors, err.Error())
}
return false
})
var err error
if len(errors) > 0 {
err = fmt.Errorf("one or more errors removing status %s from all timelines: %s", statusID, strings.Join(errors, ";"))
}
return err
}
func (m *manager) getOrCreateTimeline(timelineAccountID string) Timeline {
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc, m.log)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t, ok = i.(Timeline)
if !ok {
panic("couldn't parse entry as Timeline, this should never happen so panic")
}
}
return t
}

57
internal/timeline/postindex.go Normal file
View File

@@ -0,0 +1,57 @@
package timeline
import (
"container/list"
"errors"
)
type postIndex struct {
data *list.List
}
type postIndexEntry struct {
statusID string
}
func (p *postIndex) insertIndexed(i *postIndexEntry) error {
if p.data == nil {
p.data = &list.List{}
}
// if we have no entries yet, this is both the newest and oldest entry, so just put it in the front
if p.data.Len() == 0 {
p.data.PushFront(i)
return nil
}
var insertMark *list.Element
// We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
// We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
for e := p.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return errors.New("index: could not parse e as a postIndexEntry")
}
// if the post to index is newer than e, insert it before e in the list
if insertMark == nil {
if i.statusID > entry.statusID {
insertMark = e
}
}
// make sure we don't insert a duplicate
if entry.statusID == i.statusID {
return nil
}
}
if insertMark != nil {
p.data.InsertBefore(i, insertMark)
return nil
}
// if we reach this point it's the oldest post we've seen so put it at the back
p.data.PushBack(i)
return nil
}

215
internal/timeline/prepare.go Normal file
View File

@@ -0,0 +1,215 @@
package timeline
import (
"errors"
"fmt"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) error {
var err error
// maxID is defined but sinceID isn't so take from behind
if maxID != "" && sinceID == "" {
err = t.PrepareBehind(maxID, amount)
}
// maxID isn't defined, but sinceID || minID are, so take x before
if maxID == "" && sinceID != "" {
err = t.PrepareBefore(sinceID, false, amount)
}
if maxID == "" && minID != "" {
err = t.PrepareBefore(minID, false, amount)
}
return err
}
func (t *timeline) PrepareBehind(statusID string, amount int) error {
t.Lock()
defer t.Unlock()
var prepared int
var preparing bool
prepareloop:
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return errors.New("PrepareBehind: could not parse e as a postIndexEntry")
}
if !preparing {
// we haven't hit the position we need to prepare from yet
if entry.statusID == statusID {
preparing = true
}
}
if preparing {
if err := t.prepare(entry.statusID); err != nil {
// there's been an error
if _, ok := err.(db.ErrNoEntries); !ok {
// it's a real error
return fmt.Errorf("PrepareBehind: error preparing status with id %s: %s", entry.statusID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
}
if prepared == amount {
// we're done
break prepareloop
}
prepared = prepared + 1
}
}
return nil
}
func (t *timeline) PrepareBefore(statusID string, include bool, amount int) error {
t.Lock()
defer t.Unlock()
var prepared int
var preparing bool
prepareloop:
for e := t.postIndex.data.Back(); e != nil; e = e.Prev() {
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
}
if !preparing {
// we haven't hit the position we need to prepare from yet
if entry.statusID == statusID {
preparing = true
if !include {
continue
}
}
}
if preparing {
if err := t.prepare(entry.statusID); err != nil {
// there's been an error
if _, ok := err.(db.ErrNoEntries); !ok {
// it's a real error
return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.statusID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
}
if prepared == amount {
// we're done
break prepareloop
}
prepared = prepared + 1
}
}
return nil
}
func (t *timeline) PrepareFromTop(amount int) error {
t.Lock()
defer t.Unlock()
t.preparedPosts.data.Init()
var prepared int
prepareloop:
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
}
if err := t.prepare(entry.statusID); err != nil {
// there's been an error
if _, ok := err.(db.ErrNoEntries); !ok {
// it's a real error
return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.statusID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
}
prepared = prepared + 1
if prepared == amount {
// we're done
break prepareloop
}
}
return nil
}
func (t *timeline) prepare(statusID string) error {
// start by getting the status out of the database according to its indexed ID
gtsStatus := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, gtsStatus); err != nil {
return err
}
// if the account pointer hasn't been set on this timeline already, set it lazily here
if t.account == nil {
timelineOwnerAccount := &gtsmodel.Account{}
if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil {
return err
}
t.account = timelineOwnerAccount
}
// to convert the status we need relevant accounts from it, so pull them out here
relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus)
if err != nil {
return err
}
// check if this is a boost...
var reblogOfStatus *gtsmodel.Status
if gtsStatus.BoostOfID != "" {
s := &gtsmodel.Status{}
if err := t.db.GetByID(gtsStatus.BoostOfID, s); err != nil {
return err
}
reblogOfStatus = s
}
// serialize the status (or, at least, convert it to a form that's ready to be serialized)
apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus)
if err != nil {
return err
}
// shove it in prepared posts as a prepared posts entry
preparedPostsEntry := &preparedPostsEntry{
statusID: statusID,
prepared: apiModelStatus,
}
return t.preparedPosts.insertPrepared(preparedPostsEntry)
}
func (t *timeline) OldestPreparedPostID() (string, error) {
var id string
if t.preparedPosts == nil || t.preparedPosts.data == nil {
// return an empty string if prepared posts hasn't been initialized yet
return id, nil
}
e := t.preparedPosts.data.Back()
if e == nil {
// return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
return id, nil
}
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return id, errors.New("OldestPreparedPostID: could not parse e as a preparedPostsEntry")
}
return entry.statusID, nil
}

60
internal/timeline/preparedposts.go Normal file
View File

@@ -0,0 +1,60 @@
package timeline
import (
"container/list"
"errors"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
type preparedPosts struct {
data *list.List
}
type preparedPostsEntry struct {
statusID string
prepared *apimodel.Status
}
func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
if p.data == nil {
p.data = &list.List{}
}
// if we have no entries yet, this is both the newest and oldest entry, so just put it in the front
if p.data.Len() == 0 {
p.data.PushFront(i)
return nil
}
var insertMark *list.Element
// We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
// We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
for e := p.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return errors.New("index: could not parse e as a preparedPostsEntry")
}
// if the post to index is newer than e, insert it before e in the list
if insertMark == nil {
if i.statusID > entry.statusID {
insertMark = e
}
}
// make sure we don't insert a duplicate
if entry.statusID == i.statusID {
return nil
}
}
if insertMark != nil {
p.data.InsertBefore(i, insertMark)
return nil
}
// if we reach this point it's the oldest post we've seen so put it at the back
p.data.PushBack(i)
return nil
}

50
internal/timeline/remove.go Normal file
View File

@@ -0,0 +1,50 @@
package timeline
import (
"container/list"
"errors"
)
func (t *timeline) Remove(statusID string) (int, error) {
t.Lock()
defer t.Unlock()
var removed int
// remove entr(ies) from the post index
removeIndexes := []*list.Element{}
if t.postIndex != nil && t.postIndex.data != nil {
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a postIndexEntry")
}
if entry.statusID == statusID {
removeIndexes = append(removeIndexes, e)
}
}
}
for _, e := range removeIndexes {
t.postIndex.data.Remove(e)
removed = removed + 1
}
// remove entr(ies) from prepared posts
removePrepared := []*list.Element{}
if t.preparedPosts != nil && t.preparedPosts.data != nil {
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
}
if entry.statusID == statusID {
removePrepared = append(removePrepared, e)
}
}
}
for _, e := range removePrepared {
t.preparedPosts.data.Remove(e)
removed = removed + 1
}
return removed, nil
}

139
internal/timeline/timeline.go Normal file
View File

@@ -0,0 +1,139 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
import (
"sync"
"time"
"github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// Timeline represents a timeline for one account, and contains indexed and prepared posts.
type Timeline interface {
/*
RETRIEVAL FUNCTIONS
*/
Get(amount int, maxID string, sinceID string, minID string) ([]*apimodel.Status, error)
// GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest.
GetXFromTop(amount int) ([]*apimodel.Status, error)
// GetXBehindID returns x amount of posts from the given id onwards, from newest to oldest.
// This will NOT include the status with the given ID.
//
// This corresponds to an api call to /timelines/home?max_id=WHATEVER
GetXBehindID(amount int, fromID string) ([]*apimodel.Status, error)
// GetXBeforeID returns x amount of posts up to the given id, from newest to oldest.
// This will NOT include the status with the given ID.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER
GetXBeforeID(amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
// GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest.
// This will NOT include the status with the given IDs.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error)
/*
INDEXING FUNCTIONS
*/
// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property.
IndexOne(statusCreatedAt time.Time, statusID string) error
// OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestIndexedPostID() (string, error)
/*
PREPARATION FUNCTIONS
*/
// PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline.
PrepareFromTop(amount int) error
// PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
// If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared.
PrepareBehind(statusID string, amount int) error
// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property,
// and then immediately prepares it.
IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error
// OldestPreparedPostID returns the id of the rearmost (ie., the oldest) prepared post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestPreparedPostID() (string, error)
/*
INFO FUNCTIONS
*/
// ActualPostIndexLength returns the actual length of the post index at this point in time.
PostIndexLength() int
/*
UTILITY FUNCTIONS
*/
// Reset instructs the timeline to reset to its base state -- cache only the minimum amount of posts.
Reset() error
// Remove removes a status from both the index and prepared posts.
//
// If a status has multiple entries in a timeline, they will all be removed.
//
// The returned int indicates the amount of entries that were removed.
Remove(statusID string) (int, error)
}
// timeline fulfils the Timeline interface
type timeline struct {
postIndex *postIndex
preparedPosts *preparedPosts
accountID string
account *gtsmodel.Account
db db.DB
tc typeutils.TypeConverter
log *logrus.Logger
sync.Mutex
}
// NewTimeline returns a new Timeline for the given account ID
func NewTimeline(accountID string, db db.DB, typeConverter typeutils.TypeConverter, log *logrus.Logger) Timeline {
return &timeline{
postIndex: &postIndex{},
preparedPosts: &preparedPosts{},
accountID: accountID,
db: db,
tc: typeConverter,
log: log,
}
}
func (t *timeline) Reset() error {
return nil
}
func (t *timeline) PostIndexLength() int {
if t.postIndex == nil || t.postIndex.data == nil {
return 0
}
return t.postIndex.data.Len()
}