From 1bad36b63703c705d73a6b62de9145f13edae133 Mon Sep 17 00:00:00 2001 From: tobi Date: Wed, 30 Apr 2025 18:19:01 +0000 Subject: [PATCH] [bugfix] Fix wrong URI used in dereferenceLocal (#4097) # Description > If this is a code change, please include a summary of what you've coded, and link to the issue(s) it closes/implements. > > If this is a documentation change, please briefly describe what you've changed and why. Previously we were passing a *followers* URI into the federatingDB `Followers` function, which expects an *account* URI. This led to a nil body when doing internal dereferences (ie., while sending out messages). I think I probably wrote this bug when I refactored stuff a while ago. This PR fixes that up by making the dereferenceLocal function more consistent, and falling back to an http dereference if nothing found. Clarifies behavior of the federatingdb.Get function as well, by commenting the shit out of it. Also adds tests, precious tests. Closes https://codeberg.org/superseriousbusiness/gotosocial/issues/4066 ## Checklist Please put an x inside each checkbox to indicate that you've read and followed it: `[ ]` -> `[x]` If this is a documentation change, only the first checkbox must be filled (you can delete the others if you want). - [x] I/we have read the [GoToSocial contribution guidelines](https://codeberg.org/superseriousbusiness/gotosocial/src/branch/main/CONTRIBUTING.md). - [x] I/we have discussed the proposed changes already, either in an issue on the repository, or in the Matrix chat. - [x] I/we have not leveraged AI to create the proposed changes. - [x] I/we have performed a self-review of added code. - [x] I/we have written code that is legible and maintainable by others. - [x] I/we have commented the added code, particularly in hard-to-understand areas. - [x] I/we have made any necessary changes to documentation. - [x] I/we have added tests that cover new code. - [x] I/we have run tests and they pass locally with the changes. - [x] I/we have run `go fmt ./...` and `golangci-lint run`. Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4097 Co-authored-by: tobi Co-committed-by: tobi --- internal/federation/federatingdb/get.go | 73 +++++-- internal/transport/controller.go | 130 ++++-------- internal/transport/dereference.go | 44 ++-- internal/transport/dereference_test.go | 258 ++++++++++++++++++++++++ internal/transport/transport_test.go | 2 + 5 files changed, 385 insertions(+), 122 deletions(-) create mode 100644 internal/transport/dereference_test.go diff --git a/internal/federation/federatingdb/get.go b/internal/federation/federatingdb/get.go index 10e7dae8c..92c2d1d8d 100644 --- a/internal/federation/federatingdb/get.go +++ b/internal/federation/federatingdb/get.go @@ -19,10 +19,12 @@ package federatingdb import ( "context" - "fmt" "net/url" "code.superseriousbusiness.org/activity/streams/vocab" + "code.superseriousbusiness.org/gotosocial/internal/config" + "code.superseriousbusiness.org/gotosocial/internal/db" + "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/log" "code.superseriousbusiness.org/gotosocial/internal/uris" ) @@ -30,35 +32,78 @@ import ( // Get returns the database entry for the specified id. // // The library makes this call only after acquiring a lock first. +// +// Implementation notes: in GoToSocial this function should *only* +// be used for internal dereference calls. Everything coming from the +// outside goes via the handlers defined in internal/api/activitypub. +// +// Normally with go-fed this function would get used in lots of +// places for the side effect callback handlers, but since we override +// everything and handle side effects ourselves, the only two places +// this function actually ends up getting called are: +// +// - vendor/code.superseriousbusiness.org/activity/pub/side_effect_actor.go +// to get outbox actor inside the prepare function. +// - internal/transport/controller.go to try to shortcut deref a local item. +// +// It may be useful in future to add more matching here so that more +// stuff can be shortcutted by the dereferencer, saving HTTP calls. func (f *federatingDB) Get(ctx context.Context, id *url.URL) (value vocab.Type, err error) { log.DebugKV(ctx, "id", id) - switch { + // Ensure our host, for safety. + if id.Host != config.GetHost() { + return nil, gtserror.Newf("%s was not for our host", id.String()) + } - case uris.IsUserPath(id): - acct, err := f.state.DB.GetAccountByURI(ctx, id.String()) + if username, err := uris.ParseUserPath(id); err == nil && username != "" { + acct, err := f.state.DB.GetAccountByUsernameDomain(ctx, username, "") if err != nil { return nil, err } return f.converter.AccountToAS(ctx, acct) - case uris.IsStatusesPath(id): - status, err := f.state.DB.GetStatusByURI(ctx, id.String()) + } else if _, statusID, err := uris.ParseStatusesPath(id); err == nil && statusID != "" { + status, err := f.state.DB.GetStatusByID(ctx, statusID) if err != nil { return nil, err } return f.converter.StatusToAS(ctx, status) - case uris.IsFollowersPath(id): - return f.Followers(ctx, id) + } else if username, err := uris.ParseFollowersPath(id); err == nil && username != "" { + acct, err := f.state.DB.GetAccountByUsernameDomain(ctx, username, "") + if err != nil { + return nil, err + } - case uris.IsFollowingPath(id): - return f.Following(ctx, id) + acctURI, err := url.Parse(acct.URI) + if err != nil { + return nil, err + } - case uris.IsAcceptsPath(id): + return f.Followers(ctx, acctURI) + + } else if username, err := uris.ParseFollowingPath(id); err == nil && username != "" { + acct, err := f.state.DB.GetAccountByUsernameDomain(ctx, username, "") + if err != nil { + return nil, err + } + + acctURI, err := url.Parse(acct.URI) + if err != nil { + return nil, err + } + + return f.Following(ctx, acctURI) + + } else if uris.IsAcceptsPath(id) { return f.GetAccept(ctx, id) - - default: - return nil, fmt.Errorf("federatingDB: could not Get %s", id.String()) } + + // Nothing found, the caller + // will have to deal with this. + return nil, gtserror.Newf( + "not implemented for %s: %w", + id.String(), db.ErrNoEntries, + ) } diff --git a/internal/transport/controller.go b/internal/transport/controller.go index 7229c216d..0f3c1c9b0 100644 --- a/internal/transport/controller.go +++ b/internal/transport/controller.go @@ -23,18 +23,20 @@ import ( "crypto/rsa" "crypto/x509" "encoding/json" - "errors" "fmt" "io" "net/http" "net/url" + "strconv" "code.superseriousbusiness.org/activity/pub" + "code.superseriousbusiness.org/activity/streams/vocab" "code.superseriousbusiness.org/gotosocial/internal/ap" + apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util" "code.superseriousbusiness.org/gotosocial/internal/config" - "code.superseriousbusiness.org/gotosocial/internal/db" "code.superseriousbusiness.org/gotosocial/internal/federation/federatingdb" "code.superseriousbusiness.org/gotosocial/internal/state" + "code.superseriousbusiness.org/gotosocial/internal/util" "codeberg.org/gruf/go-byteutil" "codeberg.org/gruf/go-cache/v3" ) @@ -140,23 +142,37 @@ func (c *controller) NewTransportForUsername(ctx context.Context, username strin return transport, nil } -// dereferenceLocalFollowers is a shortcut to dereference followers of an -// account on this instance, without making any external api/http calls. +// dereferenceLocal is a shortcut to try dereferencing +// something on this instance without making any http calls. // -// It is passed to new transports, and should only be invoked when the iri.Host == this host. -func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL) (*http.Response, error) { - followers, err := c.fedDB.Followers(ctx, iri) - if err != nil && !errors.Is(err, db.ErrNoEntries) { +// Will return an error if nothing could be found, indicating that +// the calling transport should continue with an http call anyway. +// +// It should only be invoked when the iri.Host == this host. +func (c *controller) dereferenceLocal( + ctx context.Context, + uri *url.URL, +) (*http.Response, error) { + var ( + t vocab.Type + err error + ) + + t, err = c.fedDB.Get(ctx, uri) + if err != nil { + // Don't check especially for + // db.ErrNoEntries, as we *want* + // to pass this back to the caller + // if we didn't get anything. return nil, err } - if followers == nil { - // Return a generic 404 not found response. - rsp := craftResponse(iri, http.StatusNotFound) - return rsp, nil + if util.IsNil(t) { + // This should never happen. + panic("nil vocab.Type after successful c.fedDB.Get call") } - i, err := ap.Serialize(followers) + i, err := ap.Serialize(t) if err != nil { return nil, err } @@ -165,86 +181,24 @@ func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL if err != nil { return nil, err } + contentLength := len(b) // Return a response with AS data as body. - rsp := craftResponse(iri, http.StatusOK) - rsp.Body = io.NopCloser(bytes.NewReader(b)) + rsp := &http.Response{ + Request: &http.Request{URL: uri}, + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(b)), + ContentLength: int64(contentLength), + Header: map[string][]string{ + "Content-Type": {apiutil.AppActivityLDJSON}, + "Content-Length": {strconv.Itoa(contentLength)}, + }, + } + return rsp, nil } -// dereferenceLocalUser is a shortcut to dereference followers an account on -// this instance, without making any external api/http calls. -// -// It is passed to new transports, and should only be invoked when the iri.Host == this host. -func (c *controller) dereferenceLocalUser(ctx context.Context, iri *url.URL) (*http.Response, error) { - user, err := c.fedDB.Get(ctx, iri) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return nil, err - } - - if user == nil { - // Return a generic 404 not found response. - rsp := craftResponse(iri, http.StatusNotFound) - return rsp, nil - } - - i, err := ap.Serialize(user) - if err != nil { - return nil, err - } - - b, err := json.Marshal(i) - if err != nil { - return nil, err - } - - // Return a response with AS data as body. - rsp := craftResponse(iri, http.StatusOK) - rsp.Body = io.NopCloser(bytes.NewReader(b)) - return rsp, nil -} - -// dereferenceLocalAccept is a shortcut to dereference an accept created -// by an account on this instance, without making any external api/http calls. -// -// It is passed to new transports, and should only be invoked when the iri.Host == this host. -func (c *controller) dereferenceLocalAccept(ctx context.Context, iri *url.URL) (*http.Response, error) { - accept, err := c.fedDB.GetAccept(ctx, iri) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return nil, err - } - - if accept == nil { - // Return a generic 404 not found response. - rsp := craftResponse(iri, http.StatusNotFound) - return rsp, nil - } - - i, err := ap.Serialize(accept) - if err != nil { - return nil, err - } - - b, err := json.Marshal(i) - if err != nil { - return nil, err - } - - // Return a response with AS data as body. - rsp := craftResponse(iri, http.StatusOK) - rsp.Body = io.NopCloser(bytes.NewReader(b)) - return rsp, nil -} - -func craftResponse(url *url.URL, code int) *http.Response { - rsp := new(http.Response) - rsp.Request = new(http.Request) - rsp.Request.URL = url - rsp.Status = http.StatusText(code) - rsp.StatusCode = code - return rsp -} - // privkeyToPublicStr will create a string representation of RSA public key from private. func privkeyToPublicStr(privkey *rsa.PrivateKey) string { b := x509.MarshalPKCS1PublicKey(&privkey.PublicKey) diff --git a/internal/transport/dereference.go b/internal/transport/dereference.go index 85a19efea..a7ef83d3e 100644 --- a/internal/transport/dereference.go +++ b/internal/transport/dereference.go @@ -19,37 +19,41 @@ package transport import ( "context" + "errors" "net/http" "net/url" apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util" "code.superseriousbusiness.org/gotosocial/internal/config" + "code.superseriousbusiness.org/gotosocial/internal/db" "code.superseriousbusiness.org/gotosocial/internal/gtserror" - "code.superseriousbusiness.org/gotosocial/internal/uris" + "code.superseriousbusiness.org/gotosocial/internal/log" ) func (t *transport) Dereference(ctx context.Context, iri *url.URL) (*http.Response, error) { - // If the request is to us, we can shortcut for - // certain URIs rather than going through the normal - // request flow, thereby saving time and energy. + // If the request is to us, we can try to shortcut + // rather than going through the normal request flow. + // + // Only bail on a real error, otherwise continue + // to just make a normal http request to ourself. if iri.Host == config.GetHost() { - switch { - - case uris.IsFollowersPath(iri): - // The request is for followers of one of - // our accounts, which we can shortcut. - return t.controller.dereferenceLocalFollowers(ctx, iri) - - case uris.IsUserPath(iri): - // The request is for one of our - // accounts, which we can shortcut. - return t.controller.dereferenceLocalUser(ctx, iri) - - case uris.IsAcceptsPath(iri): - // The request is for an Accept on - // our instance, which we can shortcut. - return t.controller.dereferenceLocalAccept(ctx, iri) + rsp, err := t.controller.dereferenceLocal(ctx, iri) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real error. + err := gtserror.Newf("error trying dereferenceLocal: %w", err) + return nil, err } + + if rsp != nil { + // Got something! + // + // No need for + // further business. + return rsp, nil + } + + // Blast out a cheeky warning so we can keep track of this. + log.Warnf(ctx, "about to perform request to self: GET %s", iri) } // Build IRI just once diff --git a/internal/transport/dereference_test.go b/internal/transport/dereference_test.go new file mode 100644 index 000000000..c1d6fb952 --- /dev/null +++ b/internal/transport/dereference_test.go @@ -0,0 +1,258 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transport_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "testing" + + apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util" + "code.superseriousbusiness.org/gotosocial/testrig" + "github.com/stretchr/testify/suite" +) + +type DereferenceTestSuite struct { + TransportTestSuite +} + +func (suite *DereferenceTestSuite) TestDerefLocalUser() { + iri := testrig.URLMustParse(suite.testAccounts["local_account_1"].URI) + + resp, err := suite.transport.Dereference(context.Background(), iri) + if err != nil { + suite.FailNow(err.Error()) + } + defer resp.Body.Close() + + suite.Equal(http.StatusOK, resp.StatusCode) + suite.EqualValues(1887, resp.ContentLength) + suite.Equal("1887", resp.Header.Get("Content-Length")) + suite.Equal(apiutil.AppActivityLDJSON, resp.Header.Get("Content-Type")) + + b, err := io.ReadAll(resp.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + dst := bytes.Buffer{} + if err := json.Indent(&dst, b, "", " "); err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal(`{ + "@context": [ + "https://w3id.org/security/v1", + "https://www.w3.org/ns/activitystreams", + { + "discoverable": "toot:discoverable", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "toot": "http://joinmastodon.org/ns#" + } + ], + "discoverable": true, + "featured": "http://localhost:8080/users/the_mighty_zork/collections/featured", + "followers": "http://localhost:8080/users/the_mighty_zork/followers", + "following": "http://localhost:8080/users/the_mighty_zork/following", + "icon": { + "mediaType": "image/jpeg", + "type": "Image", + "url": "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg" + }, + "id": "http://localhost:8080/users/the_mighty_zork", + "image": { + "mediaType": "image/jpeg", + "type": "Image", + "url": "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg" + }, + "inbox": "http://localhost:8080/users/the_mighty_zork/inbox", + "manuallyApprovesFollowers": false, + "name": "original zork (he/they)", + "outbox": "http://localhost:8080/users/the_mighty_zork/outbox", + "preferredUsername": "the_mighty_zork", + "publicKey": { + "id": "http://localhost:8080/users/the_mighty_zork/main-key", + "owner": "http://localhost:8080/users/the_mighty_zork", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqtQQjwFLHPez+7uF9AX7\nuvLFHm3SyNIozhhVmGhxHIs0xdgRnZKmzmZkFdrFuXddBTAglU4C2u3dw10jJO1a\nWIFQF8bGkRHZG7Pd25/XmWWBRPmOJxNLeWBqpj0G+2zTMgnAV72hALSDFY2/QDsx\nUthenKw0Srpj1LUwvRbyVQQ8fGu4v0HACFnlOX2hCQwhfAnGrb0V70Y2IJu++MP7\n6i49md0vR0Mv3WbsEJUNp1fTIUzkgWB31icvfrNmaaAxw5FkAE+KfkkylhRxi5i5\nRR1XQUINWc2Kj2Kro+CJarKG+9zasMyN7+D230gpESi8rXv1SwRu865FR3gANdDS\nMwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "published": "2022-05-20T11:09:18Z", + "summary": "\u003cp\u003ehey yo this is my profile!\u003c/p\u003e", + "tag": [], + "type": "Person", + "url": "http://localhost:8080/@the_mighty_zork" +}`, dst.String()) +} + +func (suite *DereferenceTestSuite) TestDerefLocalStatus() { + iri := testrig.URLMustParse(suite.testStatuses["local_account_1_status_1"].URI) + + resp, err := suite.transport.Dereference(context.Background(), iri) + if err != nil { + suite.FailNow(err.Error()) + } + defer resp.Body.Close() + + suite.Equal(http.StatusOK, resp.StatusCode) + suite.EqualValues(1502, resp.ContentLength) + suite.Equal("1502", resp.Header.Get("Content-Length")) + suite.Equal(apiutil.AppActivityLDJSON, resp.Header.Get("Content-Type")) + + b, err := io.ReadAll(resp.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + dst := bytes.Buffer{} + if err := json.Indent(&dst, b, "", " "); err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal(`{ + "@context": [ + "https://gotosocial.org/ns", + "https://www.w3.org/ns/activitystreams", + { + "sensitive": "as:sensitive" + } + ], + "attachment": [], + "attributedTo": "http://localhost:8080/users/the_mighty_zork", + "cc": "http://localhost:8080/users/the_mighty_zork/followers", + "content": "\u003cp\u003ehello everyone!\u003c/p\u003e", + "contentMap": { + "en": "\u003cp\u003ehello everyone!\u003c/p\u003e" + }, + "id": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + "interactionPolicy": { + "canAnnounce": { + "always": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "approvalRequired": [] + }, + "canLike": { + "always": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "approvalRequired": [] + }, + "canReply": { + "always": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "approvalRequired": [] + } + }, + "published": "2021-10-20T10:40:37Z", + "replies": { + "first": { + "id": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true", + "next": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true", + "partOf": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies", + "type": "CollectionPage" + }, + "id": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies", + "type": "Collection" + }, + "sensitive": true, + "summary": "introduction post", + "tag": [], + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY" +}`, dst.String()) +} + +func (suite *DereferenceTestSuite) TestDerefLocalFollowers() { + iri := testrig.URLMustParse(suite.testAccounts["local_account_1"].FollowersURI) + + resp, err := suite.transport.Dereference(context.Background(), iri) + if err != nil { + suite.FailNow(err.Error()) + } + defer resp.Body.Close() + + suite.Equal(http.StatusOK, resp.StatusCode) + suite.EqualValues(161, resp.ContentLength) + suite.Equal("161", resp.Header.Get("Content-Length")) + suite.Equal(apiutil.AppActivityLDJSON, resp.Header.Get("Content-Type")) + + b, err := io.ReadAll(resp.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + dst := bytes.Buffer{} + if err := json.Indent(&dst, b, "", " "); err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "items": [ + "http://localhost:8080/users/1happyturtle", + "http://localhost:8080/users/admin" + ], + "type": "Collection" +}`, dst.String()) +} + +func (suite *DereferenceTestSuite) TestDerefLocalFollowing() { + iri := testrig.URLMustParse(suite.testAccounts["local_account_1"].FollowingURI) + + resp, err := suite.transport.Dereference(context.Background(), iri) + if err != nil { + suite.FailNow(err.Error()) + } + defer resp.Body.Close() + + suite.Equal(http.StatusOK, resp.StatusCode) + suite.EqualValues(161, resp.ContentLength) + suite.Equal("161", resp.Header.Get("Content-Length")) + suite.Equal(apiutil.AppActivityLDJSON, resp.Header.Get("Content-Type")) + + b, err := io.ReadAll(resp.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + dst := bytes.Buffer{} + if err := json.Indent(&dst, b, "", " "); err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "items": [ + "http://localhost:8080/users/admin", + "http://localhost:8080/users/1happyturtle" + ], + "type": "Collection" +}`, dst.String()) +} + +func TestDereferenceTestSuite(t *testing.T) { + suite.Run(t, new(DereferenceTestSuite)) +} diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index 864e3b6e4..50a4d772e 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -51,6 +51,7 @@ type TransportTestSuite struct { testApplications map[string]*gtsmodel.Application testUsers map[string]*gtsmodel.User testAccounts map[string]*gtsmodel.Account + testStatuses map[string]*gtsmodel.Status transport transport.Transport } @@ -60,6 +61,7 @@ func (suite *TransportTestSuite) SetupSuite() { suite.testApplications = testrig.NewTestApplications() suite.testUsers = testrig.NewTestUsers() suite.testAccounts = testrig.NewTestAccounts() + suite.testStatuses = testrig.NewTestStatuses() } func (suite *TransportTestSuite) SetupTest() {