From e63b6531994adcf976d8e15c2b791682b8531e7d Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 15 Mar 2022 15:01:19 +0100 Subject: [PATCH] [performance] Add dereference shortcuts to avoid making http calls to self (#430) * update transport (controller) to allow shortcuts * go fmt * expose underlying sig transport to allow test sigs --- cmd/gotosocial/action/server/server.go | 2 +- internal/transport/controller.go | 80 +++++++++++++++++++++----- internal/transport/deliver.go | 7 +++ internal/transport/dereference.go | 19 ++++++ internal/transport/transport.go | 18 +++++- testrig/testmodels.go | 8 +-- testrig/transportcontroller.go | 2 +- 7 files changed, 114 insertions(+), 22 deletions(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 8d447c59f..edd6fc1a7 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -117,7 +117,7 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error creating media manager: %s", err) } oauthServer := oauth.New(ctx, dbService) - transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient) + transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, http.DefaultClient) federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) // decide whether to create a noop email sender (won't send emails) or a real one diff --git a/internal/transport/controller.go b/internal/transport/controller.go index b16f5a1b0..56a922a8b 100644 --- a/internal/transport/controller.go +++ b/internal/transport/controller.go @@ -21,14 +21,18 @@ package transport import ( "context" "crypto" + "encoding/json" "fmt" + "net/url" "sync" "github.com/go-fed/httpsig" "github.com/spf13/viper" "github.com/superseriousbusiness/activity/pub" + "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" ) // Controller generates transports for use in making federation requests to other servers. @@ -42,19 +46,65 @@ type controller struct { clock pub.Clock client pub.HttpClient appAgent string + + // dereferenceFollowersShortcut is a shortcut to dereference followers of an + // account on this instance, without making any external api/http calls. + // + // It is passed to new transports, and should only be invoked when the iri.Host == this host. + dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error) + + // dereferenceUserShortcut is a shortcut to dereference followers an account on + // this instance, without making any external api/http calls. + // + // It is passed to new transports, and should only be invoked when the iri.Host == this host. + dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error) +} + +func dereferenceFollowersShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) { + return func(ctx context.Context, iri *url.URL) ([]byte, error) { + followers, err := federatingDB.Followers(ctx, iri) + if err != nil { + return nil, err + } + + i, err := streams.Serialize(followers) + if err != nil { + return nil, err + } + + return json.Marshal(i) + } +} + +func dereferenceUserShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) { + return func(ctx context.Context, iri *url.URL) ([]byte, error) { + user, err := federatingDB.Get(ctx, iri) + if err != nil { + return nil, err + } + + i, err := streams.Serialize(user) + if err != nil { + return nil, err + } + + return json.Marshal(i) + } } // NewController returns an implementation of the Controller interface for creating new transports -func NewController(db db.DB, clock pub.Clock, client pub.HttpClient) Controller { +func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller { applicationName := viper.GetString(config.Keys.ApplicationName) host := viper.GetString(config.Keys.Host) appAgent := fmt.Sprintf("%s %s", applicationName, host) return &controller{ - db: db, - clock: clock, - client: client, - appAgent: appAgent, + db: db, + clock: clock, + client: client, + appAgent: appAgent, + dereferenceFollowersShortcut: dereferenceFollowersShortcut(federatingDB), + dereferenceUserShortcut: dereferenceUserShortcut(federatingDB), } } @@ -78,15 +128,17 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T sigTransport := pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey) return &transport{ - client: c.client, - appAgent: c.appAgent, - gofedAgent: "(go-fed/activity v1.0.0)", - clock: c.clock, - pubKeyID: pubKeyID, - privkey: privkey, - sigTransport: sigTransport, - getSigner: getSigner, - getSignerMu: &sync.Mutex{}, + client: c.client, + appAgent: c.appAgent, + gofedAgent: "(go-fed/activity v1.0.0)", + clock: c.clock, + pubKeyID: pubKeyID, + privkey: privkey, + sigTransport: sigTransport, + getSigner: getSigner, + getSignerMu: &sync.Mutex{}, + dereferenceFollowersShortcut: c.dereferenceFollowersShortcut, + dereferenceUserShortcut: c.dereferenceUserShortcut, }, nil } diff --git a/internal/transport/deliver.go b/internal/transport/deliver.go index 80e84802c..cbba8f080 100644 --- a/internal/transport/deliver.go +++ b/internal/transport/deliver.go @@ -23,6 +23,8 @@ import ( "net/url" "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/superseriousbusiness/gotosocial/internal/config" ) func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*url.URL) error { @@ -30,6 +32,11 @@ func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*ur } func (t *transport) Deliver(ctx context.Context, b []byte, to *url.URL) error { + // if the 'to' host is our own, just skip this delivery since we by definition already have the message! + if to.Host == viper.GetString(config.Keys.Host) { + return nil + } + l := logrus.WithField("func", "Deliver") l.Debugf("performing POST to %s", to.String()) return t.sigTransport.Deliver(ctx, b, to) diff --git a/internal/transport/dereference.go b/internal/transport/dereference.go index 5ef6db9a7..61d99c5c5 100644 --- a/internal/transport/dereference.go +++ b/internal/transport/dereference.go @@ -23,10 +23,29 @@ import ( "net/url" "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, error) { l := logrus.WithField("func", "Dereference") + + // if the request is to us, we can shortcut for certain URIs rather than going through + // the normal request flow, thereby saving time and energy + if iri.Host == viper.GetString(config.Keys.Host) { + if uris.IsFollowersPath(iri) { + // the request is for followers of one of our accounts, which we can shortcut + return t.dereferenceFollowersShortcut(ctx, iri) + } + + if uris.IsUserPath(iri) { + // the request is for one of our accounts, which we can shortcut + return t.dereferenceUserShortcut(ctx, iri) + } + } + + // the request is either for a remote host or for us but we don't have a shortcut, so continue as normal l.Debugf("performing GET to %s", iri.String()) return t.sigTransport.Dereference(ctx, iri) } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 9e8cd8213..40c11ca17 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -30,8 +30,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -// Transport wraps the pub.Transport interface with some additional -// functionality for fetching remote media. +// Transport wraps the pub.Transport interface with some additional functionality for fetching remote media. +// +// Since the transport has the concept of 'shortcuts' for fetching data locally rather than remotely, it is +// not *always* the case that calling a Transport function does an http call, but it usually will for remote +// hosts or resources for which a shortcut isn't provided by the transport controller (also in this package). type Transport interface { pub.Transport // DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize. @@ -40,6 +43,8 @@ type Transport interface { DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. Finger(ctx context.Context, targetUsername string, targetDomains string) ([]byte, error) + // SigTransport returns the underlying http signature transport wrapped by the GoToSocial transport. + SigTransport() pub.Transport } // transport implements the Transport interface @@ -53,4 +58,13 @@ type transport struct { sigTransport *pub.HttpSigTransport getSigner httpsig.Signer getSignerMu *sync.Mutex + + // shortcuts for dereferencing things that exist on our instance without making an http call to ourself + + dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error) + dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error) +} + +func (t *transport) SigTransport() pub.Transport { + return t.sigTransport } diff --git a/testrig/testmodels.go b/testrig/testmodels.go index b95f5fbb6..a959bac51 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -1749,8 +1749,8 @@ func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry panic(err) } - // trigger the delivery function, which will trigger the 'do' function of the recorder above - if err := tp.Deliver(context.Background(), bytes, destination); err != nil { + // trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above + if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil { panic(err) } @@ -1781,8 +1781,8 @@ func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest panic(err) } - // trigger the delivery function, which will trigger the 'do' function of the recorder above - if _, err := tp.Dereference(context.Background(), destination); err != nil { + // trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above + if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil { panic(err) } diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go index 7408de967..90eab5ab3 100644 --- a/testrig/transportcontroller.go +++ b/testrig/transportcontroller.go @@ -39,7 +39,7 @@ import ( // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular) // basis. func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller { - return transport.NewController(db, &federation.Clock{}, client) + return transport.NewController(db, NewTestFederatingDB(db), &federation.Clock{}, client) } // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,