feature: filters v2 server-side warning/hiding (#2793)

* Remove dead code

* Filter statuses when converting to frontend representation

* status.filtered is an array

* Make matching case-insensitive

* Remove TODOs that don't need to be done now

* Add missing filter check for notification

* lint: rename ErrHideStatus

* APIFilterActionToFilterAction not used yet

* swaggerino docseroni

* Address review comments

* Add apimodel.FilterActionNone

---------

Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
Co-authored-by: tobi <tobi.smethurst@protonmail.com>
This commit is contained in:
Vyr Cossont
2024-05-06 04:49:08 -07:00
committed by GitHub
parent a0d066844f
commit 45f4afe60e
24 changed files with 855 additions and 130 deletions

View File

@@ -23,6 +23,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -74,7 +75,7 @@ func (p *Processor) BookmarksGet(ctx context.Context, requestingAccount *gtsmode
}
// Convert the status.
item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount)
item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextNone, nil)
if err != nil {
log.Errorf(ctx, "error converting bookmarked status to api: %s", err)
continue

View File

@@ -24,6 +24,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -96,9 +97,15 @@ func (p *Processor) StatusesGet(
return nil, gtserror.NewErrorInternalError(err)
}
filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
for _, s := range filtered {
// Convert filtered statuses to API statuses.
item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount)
item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, statusfilter.FilterContextAccount, filters)
if err != nil {
log.Errorf(ctx, "error convering to api status: %v", err)
continue

View File

@@ -24,6 +24,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -184,7 +185,7 @@ func (p *Processor) GetAPIStatus(
apiStatus *apimodel.Status,
errWithCode gtserror.WithCode,
) {
apiStatus, err := p.converter.StatusToAPIStatus(ctx, target, requester)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, target, requester, statusfilter.FilterContextNone, nil)
if err != nil {
err = gtserror.Newf("error converting status: %w", err)
return nil, gtserror.NewErrorInternalError(err)
@@ -192,87 +193,6 @@ func (p *Processor) GetAPIStatus(
return apiStatus, nil
}
// GetVisibleAPIStatuses converts an array of gtsmodel.Status (inputted by next function) into
// API model statuses, checking first for visibility. Please note that all errors will be
// logged at ERROR level, but will not be returned. Callers are likely to run into show-stopping
// errors in the lead-up to this function, whereas calling this should not be a show-stopper.
func (p *Processor) GetVisibleAPIStatuses(
ctx context.Context,
requester *gtsmodel.Account,
next func(int) *gtsmodel.Status,
length int,
) []*apimodel.Status {
return p.getVisibleAPIStatuses(ctx, 3, requester, next, length)
}
// GetVisibleAPIStatusesPaged is functionally equivalent to GetVisibleAPIStatuses(),
// except the statuses are returned as a converted slice of statuses as interface{}.
func (p *Processor) GetVisibleAPIStatusesPaged(
ctx context.Context,
requester *gtsmodel.Account,
next func(int) *gtsmodel.Status,
length int,
) []interface{} {
statuses := p.getVisibleAPIStatuses(ctx, 3, requester, next, length)
if len(statuses) == 0 {
return nil
}
items := make([]interface{}, len(statuses))
for i, status := range statuses {
items[i] = status
}
return items
}
func (p *Processor) getVisibleAPIStatuses(
ctx context.Context,
calldepth int, // used to skip wrapping func above these's names
requester *gtsmodel.Account,
next func(int) *gtsmodel.Status,
length int,
) []*apimodel.Status {
// Start new log entry with
// the above calling func's name.
l := log.
WithContext(ctx).
WithField("caller", log.Caller(calldepth+1))
// Preallocate slice according to expected length.
statuses := make([]*apimodel.Status, 0, length)
for i := 0; i < length; i++ {
// Get next status.
status := next(i)
if status == nil {
continue
}
// Check whether this status is visible to requesting account.
visible, err := p.filter.StatusVisible(ctx, requester, status)
if err != nil {
l.Errorf("error checking status visibility: %v", err)
continue
}
if !visible {
// Not visible to requester.
continue
}
// Convert the status to an API model representation.
apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requester)
if err != nil {
l.Errorf("error converting status: %v", err)
continue
}
// Append API model to return slice.
statuses = append(statuses, apiStatus)
}
return statuses
}
// InvalidateTimelinedStatus is a shortcut function for invalidating the cached
// representation one status in the home timeline and all list timelines of the
// given accountID. It should only be called in cases where a status update

View File

@@ -21,6 +21,7 @@ import (
"context"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -113,7 +114,7 @@ func (p *Processor) packageStatuses(
continue
}
apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextNone, nil)
if err != nil {
log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", status.ID, err)
continue

View File

@@ -23,6 +23,7 @@ import (
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -280,7 +281,15 @@ func TopoSort(apiStatuses []*apimodel.Status, targetAccountID string) {
// ContextGet returns the context (previous and following posts) from the given status ID.
func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode) {
return p.contextGet(ctx, requestingAccount, targetStatusID, p.converter.StatusToAPIStatus)
filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
convert := func(ctx context.Context, status *gtsmodel.Status, requestingAccount *gtsmodel.Account) (*apimodel.Status, error) {
return p.converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextThread, filters)
}
return p.contextGet(ctx, requestingAccount, targetStatusID, convert)
}
// WebContextGet is like ContextGet, but is explicitly

View File

@@ -24,6 +24,7 @@ import (
"testing"
"github.com/stretchr/testify/suite"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/stream"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@@ -39,7 +40,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() {
suite.NoError(errWithCode)
editedStatus := suite.testStatuses["remote_account_1_status_1"]
apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(context.Background(), editedStatus, account)
apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(context.Background(), editedStatus, account, statusfilter.FilterContextNotifications, nil)
suite.NoError(err)
suite.streamProcessor.StatusUpdate(context.Background(), account, apiStatus, stream.TimelineHome)

View File

@@ -24,6 +24,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -54,7 +55,7 @@ func (p *Processor) FavedTimelineGet(ctx context.Context, authed *oauth.Auth, ma
continue
}
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, statusfilter.FilterContextNone, nil)
if err != nil {
log.Errorf(ctx, "error convering to api status: %v", err)
continue

View File

@@ -23,6 +23,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -98,7 +99,13 @@ func HomeTimelineStatusPrepare(state *state.State, converter *typeutils.Converte
return nil, err
}
return converter.StatusToAPIStatus(ctx, status, requestingAccount)
filters, err := state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err)
return nil, err
}
return converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextHome, filters)
}
}

View File

@@ -23,6 +23,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -110,7 +111,13 @@ func ListTimelineStatusPrepare(state *state.State, converter *typeutils.Converte
return nil, err
}
return converter.StatusToAPIStatus(ctx, status, requestingAccount)
filters, err := state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err)
return nil, err
}
return converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextHome, filters)
}
}

View File

@@ -43,6 +43,12 @@ func (p *Processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, ma
return util.EmptyPageableResponse(), nil
}
filters, err := p.state.DB.GetFiltersForAccountID(ctx, authed.Account.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", authed.Account.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
var (
items = make([]interface{}, 0, count)
nextMaxIDValue string
@@ -70,7 +76,7 @@ func (p *Processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, ma
continue
}
item, err := p.converter.NotificationToAPINotification(ctx, n)
item, err := p.converter.NotificationToAPINotification(ctx, n, filters)
if err != nil {
log.Debugf(ctx, "skipping notification %s because it couldn't be converted to its api representation: %s", n.ID, err)
continue
@@ -104,7 +110,13 @@ func (p *Processor) NotificationGet(ctx context.Context, account *gtsmodel.Accou
return nil, gtserror.NewErrorNotFound(err)
}
apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif)
filters, err := p.state.DB.GetFiltersForAccountID(ctx, account.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", account.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif, filters)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.NewErrorNotFound(err)

View File

@@ -24,6 +24,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -46,6 +47,16 @@ func (p *Processor) PublicTimelineGet(
items = make([]any, 0, limit)
)
var filters []*gtsmodel.Filter
if requester != nil {
var err error
filters, err = p.state.DB.GetFiltersForAccountID(ctx, requester.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requester.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
}
// Try a few times to select appropriate public
// statuses from the db, paging up or down to
// reattempt if nothing suitable is found.
@@ -87,7 +98,10 @@ outer:
continue inner
}
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester, statusfilter.FilterContextPublic, filters)
if errors.Is(err, statusfilter.ErrHideStatus) {
continue
}
if err != nil {
log.Errorf(ctx, "error converting to api status: %v", err)
continue inner

View File

@@ -24,6 +24,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -111,6 +112,12 @@ func (p *Processor) packageTagResponse(
prevMinIDValue = statuses[0].ID
)
filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAcct.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAcct.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
for _, s := range statuses {
timelineable, err := p.filter.StatusTagTimelineable(ctx, requestingAcct, s)
if err != nil {
@@ -122,7 +129,10 @@ func (p *Processor) packageTagResponse(
continue
}
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct, statusfilter.FilterContextPublic, filters)
if errors.Is(err, statusfilter.ErrHideStatus) {
continue
}
if err != nil {
log.Errorf(ctx, "error converting to api status: %v", err)
continue

View File

@@ -28,6 +28,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/messages"
@@ -154,6 +155,8 @@ func (suite *FromClientAPITestSuite) statusJSON(
ctx,
status,
requestingAccount,
statusfilter.FilterContextNone,
nil,
)
if err != nil {
suite.FailNow(err.Error())
@@ -258,7 +261,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() {
suite.FailNow("timed out waiting for new status notification")
}
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif)
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil)
if err != nil {
suite.FailNow(err.Error())
}

View File

@@ -467,7 +467,12 @@ func (s *Surface) Notify(
unlock()
// Stream notification to the user.
apiNotif, err := s.Converter.NotificationToAPINotification(ctx, notif)
filters, err := s.State.DB.GetFiltersForAccountID(ctx, targetAccount.ID)
if err != nil {
return gtserror.Newf("couldn't retrieve filters for account %s: %w", targetAccount.ID, err)
}
apiNotif, err := s.Converter.NotificationToAPINotification(ctx, notif, filters)
if err != nil {
return gtserror.Newf("error converting notification to api representation: %w", err)
}

View File

@@ -22,6 +22,7 @@ import (
"errors"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -111,6 +112,11 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
continue
}
filters, err := s.State.DB.GetFiltersForAccountID(ctx, follow.AccountID)
if err != nil {
return gtserror.Newf("couldn't retrieve filters for account %s: %w", follow.AccountID, err)
}
// Add status to any relevant lists
// for this follow, if applicable.
s.listTimelineStatusForFollow(
@@ -118,6 +124,7 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
status,
follow,
&errs,
filters,
)
// Add status to home timeline for owner
@@ -129,6 +136,7 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
follow.Account,
status,
stream.TimelineHome,
filters,
)
if err != nil {
errs.Appendf("error home timelining status: %w", err)
@@ -180,6 +188,7 @@ func (s *Surface) listTimelineStatusForFollow(
status *gtsmodel.Status,
follow *gtsmodel.Follow,
errs *gtserror.MultiError,
filters []*gtsmodel.Filter,
) {
// To put this status in appropriate list timelines,
// we need to get each listEntry that pertains to
@@ -222,6 +231,7 @@ func (s *Surface) listTimelineStatusForFollow(
follow.Account,
status,
stream.TimelineList+":"+listEntry.ListID, // key streamType to this specific list
filters,
); err != nil {
errs.Appendf("error adding status to timeline for list %s: %w", listEntry.ListID, err)
// implicit continue
@@ -332,6 +342,7 @@ func (s *Surface) timelineStatus(
account *gtsmodel.Account,
status *gtsmodel.Status,
streamType string,
filters []*gtsmodel.Filter,
) (bool, error) {
// Ingest status into given timeline using provided function.
if inserted, err := ingest(ctx, timelineID, status); err != nil {
@@ -343,7 +354,12 @@ func (s *Surface) timelineStatus(
}
// The status was inserted so stream it to the user.
apiStatus, err := s.Converter.StatusToAPIStatus(ctx, status, account)
apiStatus, err := s.Converter.StatusToAPIStatus(ctx,
status,
account,
statusfilter.FilterContextHome,
filters,
)
if err != nil {
err = gtserror.Newf("error converting status %s to frontend representation: %w", status.ID, err)
return true, err
@@ -457,6 +473,11 @@ func (s *Surface) timelineStatusUpdateForFollowers(
continue
}
filters, err := s.State.DB.GetFiltersForAccountID(ctx, follow.AccountID)
if err != nil {
return gtserror.Newf("couldn't retrieve filters for account %s: %w", follow.AccountID, err)
}
// Add status to any relevant lists
// for this follow, if applicable.
s.listTimelineStatusUpdateForFollow(
@@ -464,6 +485,7 @@ func (s *Surface) timelineStatusUpdateForFollowers(
status,
follow,
&errs,
filters,
)
// Add status to home timeline for owner
@@ -473,6 +495,7 @@ func (s *Surface) timelineStatusUpdateForFollowers(
follow.Account,
status,
stream.TimelineHome,
filters,
)
if err != nil {
errs.Appendf("error home timelining status: %w", err)
@@ -490,6 +513,7 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
status *gtsmodel.Status,
follow *gtsmodel.Follow,
errs *gtserror.MultiError,
filters []*gtsmodel.Filter,
) {
// To put this status in appropriate list timelines,
// we need to get each listEntry that pertains to
@@ -530,6 +554,7 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
follow.Account,
status,
stream.TimelineList+":"+listEntry.ListID, // key streamType to this specific list
filters,
); err != nil {
errs.Appendf("error adding status to timeline for list %s: %w", listEntry.ListID, err)
// implicit continue
@@ -544,8 +569,13 @@ func (s *Surface) timelineStreamStatusUpdate(
account *gtsmodel.Account,
status *gtsmodel.Status,
streamType string,
filters []*gtsmodel.Filter,
) error {
apiStatus, err := s.Converter.StatusToAPIStatus(ctx, status, account)
apiStatus, err := s.Converter.StatusToAPIStatus(ctx, status, account, statusfilter.FilterContextHome, filters)
if errors.Is(err, statusfilter.ErrHideStatus) {
// Don't put this status in the stream.
return nil
}
if err != nil {
err = gtserror.Newf("error converting status %s to frontend representation: %w", status.ID, err)
return err