Dereference remote replies (#132)

* decided where to put reply dereferencing

* fiddling with dereferencing threads

* further adventures

* tidy up some stuff

* move dereferencing functionality

* a bunch of refactoring

* go fmt

* more refactoring

* bleep bloop

* docs and linting

* start implementing replies collection on gts side

* fiddling around

* allow dereferencing our replies

* lint, fmt
This commit is contained in:
Tobi Smethurst
2021-08-10 13:32:39 +02:00
committed by GitHub
parent 0386a28b5a
commit 0f2de6394a
68 changed files with 2946 additions and 1393 deletions

View File

@@ -36,15 +36,6 @@ func (p *processor) Get(requestingAccount *gtsmodel.Account, targetAccountID str
return nil, fmt.Errorf("db error: %s", err)
}
// lazily dereference things on the account if it hasn't been done yet
var requestingUsername string
if requestingAccount != nil {
requestingUsername = requestingAccount.Username
}
if err := p.federator.DereferenceAccountFields(targetAccount, requestingUsername, false); err != nil {
p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err)
}
var blocked bool
var err error
if requestingAccount != nil {

View File

@@ -63,12 +63,6 @@ func (p *processor) FollowersGet(requestingAccount *gtsmodel.Account, targetAcco
return nil, gtserror.NewErrorInternalError(err)
}
// derefence account fields in case we haven't done it already
if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil {
// don't bail if we can't fetch them, we'll try another time
p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err)
}
account, err := p.tc.AccountToMastoPublic(a)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)

View File

@@ -63,12 +63,6 @@ func (p *processor) FollowingGet(requestingAccount *gtsmodel.Account, targetAcco
return nil, gtserror.NewErrorInternalError(err)
}
// derefence account fields in case we haven't done it already
if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil {
// don't bail if we can't fetch them, we'll try another time
p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err)
}
account, err := p.tc.AccountToMastoPublic(a)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)

View File

@@ -31,65 +31,9 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
// username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account
// for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database,
// and passing it into the processor through a channel for further asynchronous processing.
func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) {
// OK now we can do the dereferencing part
// we might already have an entry for this account so check that first
requestingAccount := &gtsmodel.Account{}
err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
if err == nil {
// we do have it yay, return it
return requestingAccount, nil
}
if _, ok := err.(db.ErrNoEntries); !ok {
// something has actually gone wrong so bail
return nil, fmt.Errorf("database error getting account with uri %s: %s", requestingAccountURI.String(), err)
}
// we just don't have an entry for this account yet
// what we do now should depend on our chosen federation method
// for now though, we'll just dereference it
// TODO: slow-fed
requestingPerson, err := p.federator.DereferenceRemoteAccount(username, requestingAccountURI)
if err != nil {
return nil, fmt.Errorf("couldn't dereference %s: %s", requestingAccountURI.String(), err)
}
// convert it to our internal account representation
requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false)
if err != nil {
return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err)
}
requestingAccountID, err := id.NewRandomULID()
if err != nil {
return nil, err
}
requestingAccount.ID = requestingAccountID
if err := p.db.Put(requestingAccount); err != nil {
return nil, fmt.Errorf("database error inserting account with uri %s: %s", requestingAccountURI.String(), err)
}
// put it in our channel to queue it for async processing
p.fromFederator <- gtsmodel.FromFederator{
APObjectType: gtsmodel.ActivityStreamsProfile,
APActivityType: gtsmodel.ActivityStreamsCreate,
GTSModel: requestingAccount,
}
return requestingAccount, nil
}
func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
// get the account the request is referring to
requestedAccount := &gtsmodel.Account{}
@@ -112,9 +56,9 @@ func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, r
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
// if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part
// if we're not already handshaking/dereferencing a remote account, dereference it now
if !p.federator.Handshaking(requestedUsername, requestingAccountURI) {
requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}
@@ -158,7 +102,7 @@ func (p *processor) GetFediFollowers(ctx context.Context, requestedUsername stri
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}
@@ -203,7 +147,7 @@ func (p *processor) GetFediFollowing(ctx context.Context, requestedUsername stri
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}
@@ -248,7 +192,7 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string,
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}
@@ -295,6 +239,139 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string,
return data, nil
}
func (p *processor) GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
// get the account the request is referring to
requestedAccount := &gtsmodel.Account{}
if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
}
// authenticate the request
requestingAccountURI, authenticated, err := p.federator.AuthenticateFederatedRequest(ctx, requestedUsername)
if err != nil || !authenticated {
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}
// authorize the request:
// 1. check if a block exists between the requester and the requestee
blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
if blocked {
return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
}
// get the status out of the database here
s := &gtsmodel.Status{}
if err := p.db.GetWhere([]db.Where{
{Key: "id", Value: requestedStatusID},
{Key: "account_id", Value: requestedAccount.ID},
}, s); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting status with id %s and account id %s: %s", requestedStatusID, requestedAccount.ID, err))
}
visible, err := p.filter.StatusVisible(s, requestingAccount)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
if !visible {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("status with id %s not visible to user with id %s", s.ID, requestingAccount.ID))
}
var data map[string]interface{}
// now there are three scenarios:
// 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page.
// 2. we're asked for a page but only_other_accounts has not been set in the query -- so we should just return the first page of the collection, with no items.
// 3. we're asked for a page, and only_other_accounts has been set, and min_id has optionally been set -- so we need to return some actual items!
if !page {
// scenario 1
// get the collection
collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
data, err = streams.Serialize(collection)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
} else if page && requestURL.Query().Get("only_other_accounts") == "" {
// scenario 2
// get the collection
collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// but only return the first page
data, err = streams.Serialize(collection.GetActivityStreamsFirst().GetActivityStreamsCollectionPage())
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
} else {
// scenario 3
// get immediate children
replies, err := p.db.StatusChildren(s, true, minID)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// filter children and extract URIs
replyURIs := map[string]*url.URL{}
for _, r := range replies {
// only show public or unlocked statuses as replies
if r.Visibility != gtsmodel.VisibilityPublic && r.Visibility != gtsmodel.VisibilityUnlocked {
continue
}
// respect onlyOtherAccounts parameter
if onlyOtherAccounts && r.AccountID == requestedAccount.ID {
continue
}
// only show replies that the status owner can see
visibleToStatusOwner, err := p.filter.StatusVisible(r, requestedAccount)
if err != nil || !visibleToStatusOwner {
continue
}
// only show replies that the requester can see
visibleToRequester, err := p.filter.StatusVisible(r, requestingAccount)
if err != nil || !visibleToRequester {
continue
}
rURI, err := url.Parse(r.URI)
if err != nil {
continue
}
replyURIs[r.ID] = rURI
}
repliesPage, err := p.tc.StatusURIsToASRepliesPage(s, onlyOtherAccounts, minID, replyURIs)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
data, err = streams.Serialize(repliesPage)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
}
return data, nil
}
func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) {
// get the account the request is referring to
requestedAccount := &gtsmodel.Account{}

View File

@@ -21,6 +21,7 @@ package processing
import (
"errors"
"fmt"
"net/url"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -47,36 +48,21 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
return errors.New("note was not parseable as *gtsmodel.Status")
}
l.Trace("will now derefence incoming status")
if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
return fmt.Errorf("error dereferencing status from federator: %s", err)
}
if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
return fmt.Errorf("error updating dereferenced status in the db: %s", err)
}
if err := p.timelineStatus(incomingStatus); err != nil {
status, err := p.federator.EnrichRemoteStatus(federatorMsg.ReceivingAccount.Username, incomingStatus)
if err != nil {
return err
}
if err := p.notifyStatus(incomingStatus); err != nil {
if err := p.timelineStatus(status); err != nil {
return err
}
if err := p.notifyStatus(status); err != nil {
return err
}
case gtsmodel.ActivityStreamsProfile:
// CREATE AN ACCOUNT
incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account)
if !ok {
return errors.New("profile was not parseable as *gtsmodel.Account")
}
l.Trace("will now derefence incoming account")
if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil {
return fmt.Errorf("error dereferencing account from federator: %s", err)
}
if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
return fmt.Errorf("error updating dereferenced account in the db: %s", err)
}
// nothing to do here
case gtsmodel.ActivityStreamsLike:
// CREATE A FAVE
incomingFave, ok := federatorMsg.GTSModel.(*gtsmodel.StatusFave)
@@ -154,12 +140,13 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
return errors.New("profile was not parseable as *gtsmodel.Account")
}
l.Trace("will now derefence incoming account")
if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
return fmt.Errorf("error dereferencing account from federator: %s", err)
incomingAccountURI, err := url.Parse(incomingAccount.URI)
if err != nil {
return err
}
if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
return fmt.Errorf("error updating dereferenced account in the db: %s", err)
if _, _, err := p.federator.GetRemoteAccount(federatorMsg.ReceivingAccount.Username, incomingAccountURI, true); err != nil {
return fmt.Errorf("error dereferencing account from federator: %s", err)
}
}
case gtsmodel.ActivityStreamsDelete:

View File

@@ -191,6 +191,10 @@ type Processor interface {
// authentication before returning a JSON serializable interface to the caller.
GetFediStatus(ctx context.Context, requestedUsername string, requestedStatusID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
// GetFediStatus handles the getting of a fedi/activitypub representation of replies to a status, performing appropriate
// authentication before returning a JSON serializable interface to the caller.
GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode)

View File

@@ -19,7 +19,6 @@
package processing
import (
"errors"
"fmt"
"net/url"
"strings"
@@ -29,7 +28,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@@ -122,6 +120,11 @@ func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQu
}
func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) {
l := p.log.WithFields(logrus.Fields{
"func": "searchStatusByURI",
"uri": uri.String(),
"resolve": resolve,
})
maybeStatus := &gtsmodel.Status{}
if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
@@ -134,57 +137,12 @@ func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve
// we don't have it locally so dereference it if we're allowed to
if resolve {
statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri)
status, _, _, err := p.federator.GetRemoteStatus(authed.Account.Username, uri, true)
if err == nil {
// it IS a status!
// extract the status owner's IRI from the statusable
var statusOwnerURI *url.URL
statusAttributedTo := statusable.GetActivityStreamsAttributedTo()
for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() {
if i.IsIRI() {
statusOwnerURI = i.GetIRI()
break
}
if err := p.federator.DereferenceRemoteThread(authed.Account.Username, uri); err != nil {
// try to deref the thread while we're here
l.Debugf("searchStatusByURI: error dereferencing remote thread: %s", err)
}
if statusOwnerURI == nil {
return nil, errors.New("couldn't extract ownerAccountURI from statusable")
}
// make sure the status owner exists in the db by searching for it
_, err := p.searchAccountByURI(authed, statusOwnerURI, resolve)
if err != nil {
return nil, err
}
// we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly
// first turn it into a gtsmodel.Status
status, err := p.tc.ASStatusToStatus(statusable)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
statusID, err := id.NewULIDFromTime(status.CreatedAt)
if err != nil {
return nil, err
}
status.ID = statusID
if err := p.db.Put(status); err != nil {
return nil, fmt.Errorf("error putting status in the db: %s", err)
}
// properly dereference everything in the status (media attachments etc)
if err := p.federator.DereferenceStatusFields(status, authed.Account.Username); err != nil {
return nil, fmt.Errorf("error dereferencing status fields: %s", err)
}
// update with the nicely dereferenced status
if err := p.db.UpdateByID(status.ID, status); err != nil {
return nil, fmt.Errorf("error updating status in the db: %s", err)
}
return status, nil
}
}
@@ -202,31 +160,10 @@ func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve
}
if resolve {
// we don't have it locally so try and dereference it
accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri)
account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, uri, true)
if err != nil {
return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
}
// it IS an account!
account, err := p.tc.ASRepresentationToAccount(accountable, false)
if err != nil {
return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
}
accountID, err := id.NewRandomULID()
if err != nil {
return nil, err
}
account.ID = accountID
if err := p.db.Put(account); err != nil {
return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err)
}
if err := p.federator.DereferenceAccountFields(account, authed.Account.Username, false); err != nil {
return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err)
}
return account, nil
}
return nil, nil
@@ -275,35 +212,12 @@ func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, r
return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err)
}
// dereference the account based on the URI we retrieved from the webfinger lookup
accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI)
// we don't have it locally so try and dereference it
account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, acctURI, true)
if err != nil {
// something went wrong doing the dereferencing so we can't process the request
return nil, fmt.Errorf("searchAccountByMention: error dereferencing remote account with uri %s: %s", acctURI.String(), err)
}
// convert the dereferenced account to the gts model of that account
foundAccount, err := p.tc.ASRepresentationToAccount(accountable, false)
if err != nil {
// something went wrong doing the conversion to a gtsmodel.Account so we can't process the request
return nil, fmt.Errorf("searchAccountByMention: error converting account with uri %s: %s", acctURI.String(), err)
}
foundAccountID, err := id.NewULID()
if err != nil {
return nil, err
}
foundAccount.ID = foundAccountID
// put this new account in our database
if err := p.db.Put(foundAccount); err != nil {
return nil, fmt.Errorf("searchAccountByMention: error inserting account with uri %s: %s", acctURI.String(), err)
}
// properly dereference all the fields on the account immediately
if err := p.federator.DereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err)
return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err)
}
return account, nil
}
return nil, nil

View File

@@ -33,7 +33,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (*
return nil, gtserror.NewErrorForbidden(fmt.Errorf("account with id %s does not have permission to view status %s", account.ID, targetStatusID))
}
parents, err := p.db.StatusParents(targetStatus)
parents, err := p.db.StatusParents(targetStatus, false)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
@@ -51,7 +51,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (*
return context.Ancestors[i].ID < context.Ancestors[j].ID
})
children, err := p.db.StatusChildren(targetStatus)
children, err := p.db.StatusChildren(targetStatus, false, "")
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}