[chore] reformat GetAccount() functionality, support updating accounts based on last_fetch (#1411)

* reformat GetAccount() functionality, and add UpdateAccount() function.

* use fetched_at instead of last_webfingered_at

* catch local "not found" errors. small formatting / error string changes

* remove now unused error type

* return nil when wrapping nil error

* update expected error messages

* return correct url for foss satan webfinger

* add AP model for Some_User

* normalize local domain

* return notretrievable where appropriate

* expose NewErrNotRetrievable

* ensure webfinger for new accounts searched by uri

* update local account short circuit

* allow enrich to fail for already-known accounts

* remove unused LastWebfingeredAt

* expose test maps on mock http client

* update Update test

* reformat GetAccount() functionality, and add UpdateAccount() function.

* use fetched_at instead of last_webfingered_at

* catch local "not found" errors. small formatting / error string changes

* remove nil error checks (we shouldn't be passing nil errors to newError() initializers)

* remove mutex unlock on transport init fail (it hasn't yet been locked!)

* woops add back the error wrapping to use ErrNotRetrievable

* caches were never being started... 🙈

---------

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

cmd/gotosocial/action/server/server.go

@@ -71,6 +71,8 @@ var Start action.GTSAction = func(ctx context.Context) error {
 
 	// Initialize caches
 	state.Caches.Init()
+	state.Caches.Start()
+	defer state.Caches.Stop()
 
 	// Open connection to the database
 	dbService, err := bundb.NewBunDBService(ctx, &state)

internal/api/activitypub/users/inboxpost_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -237,10 +238,11 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
 }
 
 func (suite *InboxPostTestSuite) TestPostUpdate() {
+	receivingAccount := suite.testAccounts["local_account_1"]
 	updatedAccount := *suite.testAccounts["remote_account_1"]
 	updatedAccount.DisplayName = "updated display name!"
 
-	// ad an emoji to the account; because we're serializing this remote
+	// add an emoji to the account; because we're serializing this remote
 	// account from our own instance, we need to cheat a bit to get the emoji
 	// to work properly, just for this test
 	testEmoji := &gtsmodel.Emoji{}
@@ -251,8 +253,6 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	asAccount, err := suite.tc.AccountToAS(context.Background(), &updatedAccount)
 	suite.NoError(err)
 
-	receivingAccount := suite.testAccounts["local_account_1"]
-
 	// create an update
 	update := streams.NewActivityStreamsUpdate()
 
@@ -294,7 +294,14 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
+	// use a different version of the mock http client which serves the updated
+	// version of the remote account, as though it had been updated there too;
+	// this is needed so it can be dereferenced + updated properly
+	mockHTTPClient := testrig.NewMockHTTPClient(nil, "../../../../testrig/media")
+	mockHTTPClient.TestRemotePeople = map[string]vocab.ActivityStreamsPerson{
+		updatedAccount.URI: asAccount,
+	}
+	tc := testrig.NewTestTransportController(mockHTTPClient, suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -346,8 +353,8 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	// emojis should be updated
 	suite.Contains(dbUpdatedAccount.EmojiIDs, testEmoji.ID)
 
-	// account should be freshly webfingered
+	// account should be freshly fetched
-	suite.WithinDuration(time.Now(), dbUpdatedAccount.LastWebfingeredAt, 10*time.Second)
+	suite.WithinDuration(time.Now(), dbUpdatedAccount.FetchedAt, 10*time.Second)
 
 	// everything else should be the same as it was before
 	suite.EqualValues(updatedAccount.Username, dbUpdatedAccount.Username)

internal/api/activitypub/users/userget_test.go

@@ -89,8 +89,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
 	suite.True(ok)
 
 	// convert person to account
-	// since this account is already known, we should get a pretty full model of it from the conversion
+	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "")
-	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false)
 	suite.NoError(err)
 	suite.EqualValues(targetAccount.Username, a.Username)
 }
@@ -167,7 +166,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() {
 	suite.True(ok)
 
 	// convert person to account
-	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false)
+	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "")
 	suite.NoError(err)
 	suite.EqualValues(targetAccount.Username, a.Username)
 }

internal/db/bundb/account.go

@@ -21,7 +21,6 @@ package bundb
 import (
 	"context"
 	"errors"
-	"fmt"
 	"strings"
 	"time"
 
@@ -145,11 +144,27 @@ func (a *accountDB) getAccount(ctx context.Context, lookup string, dbQuery func(
 		return nil, err
 	}
 
+	if account.AvatarMediaAttachmentID != "" {
+		// Set the account's related avatar
+		account.AvatarMediaAttachment, err = a.state.DB.GetAttachmentByID(ctx, account.AvatarMediaAttachmentID)
+		if err != nil {
+			log.Errorf("error getting account %s avatar: %v", account.ID, err)
+		}
+	}
+
+	if account.HeaderMediaAttachmentID != "" {
+		// Set the account's related header
+		account.HeaderMediaAttachment, err = a.state.DB.GetAttachmentByID(ctx, account.HeaderMediaAttachmentID)
+		if err != nil {
+			log.Errorf("error getting account %s header: %v", account.ID, err)
+		}
+	}
+
 	if len(account.EmojiIDs) > 0 {
 		// Set the account's related emojis
 		account.Emojis, err = a.state.DB.GetEmojisByIDs(ctx, account.EmojiIDs)
 		if err != nil {
-			return nil, fmt.Errorf("error getting account emojis: %w", err)
+			log.Errorf("error getting account %s emojis: %v", account.ID, err)
 		}
 	}
 

internal/db/bundb/basic_test.go

@@ -92,7 +92,7 @@ func (suite *BasicTestSuite) TestPutAccountWithBunDefaultFields() {
 	suite.Empty(a.StatusFormat)
 	suite.Equal(testAccount.URI, a.URI)
 	suite.Equal(testAccount.URL, a.URL)
-	suite.Zero(testAccount.LastWebfingeredAt)
+	suite.Zero(testAccount.FetchedAt)
 	suite.Equal(testAccount.InboxURI, a.InboxURI)
 	suite.Equal(testAccount.OutboxURI, a.OutboxURI)
 	suite.Empty(a.FollowingURI)

internal/db/bundb/migrations/20230202212700_rename_account_webfingered_to_fetched.go

@@ -0,0 +1,44 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package migrations
+
+import (
+	"context"
+
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	up := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			_, err := tx.ExecContext(ctx, "ALTER TABLE ? RENAME COLUMN ? TO ?", bun.Ident("accounts"), bun.Ident("last_webfingered_at"), bun.Ident("fetched_at"))
+			return err
+		})
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/db/bundb/notification_test.go

@@ -39,10 +39,7 @@ func (suite *NotificationTestSuite) spamNotifs() {
 	zork := suite.testAccounts["local_account_1"]
 
 	for i := 0; i < notifCount; i++ {
-		notifID, err := id.NewULID()
+		notifID := id.NewULID()
-		if err != nil {
-			panic(err)
-		}
 
 		var targetAccountID string
 		if i%2 == 0 {

internal/db/bundb/session.go

@@ -63,13 +63,8 @@ func (s *sessionDB) createSession(ctx context.Context) (*gtsmodel.RouterSession,
 		return nil, err
 	}
 
-	id, err := id.NewULID()
-	if err != nil {
-		return nil, err
-	}
-
 	rs := &gtsmodel.RouterSession{
-		ID:    id,
+		ID:    id.NewULID(),
 		Auth:  auth,
 		Crypt: crypt,
 	}

internal/federation/dereference.go

@@ -1,52 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package federation
-
-import (
-	"context"
-	"net/url"
-
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-)
-
-func (f *federator) GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error) {
-	return f.dereferencer.GetAccount(ctx, params)
-}
-
-func (f *federator) GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) {
-	return f.dereferencer.GetStatus(ctx, username, remoteStatusID, refetch, includeParent)
-}
-
-func (f *federator) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) {
-	return f.dereferencer.EnrichRemoteStatus(ctx, username, status, includeParent)
-}
-
-func (f *federator) DereferenceRemoteThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) {
-	f.dereferencer.DereferenceThread(ctx, username, statusIRI, status, statusable)
-}
-
-func (f *federator) GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
-	return f.dereferencer.GetRemoteInstance(ctx, username, remoteInstanceURI)
-}
-
-func (f *federator) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error {
-	return f.dereferencer.DereferenceAnnounce(ctx, announce, requestingUsername)
-}

internal/federation/dereferencing/account.go

@@ -25,10 +25,8 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-	"strings"
 	"time"
 
-	"github.com/miekg/dns"
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
@@ -38,375 +36,222 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
-var webfingerInterval = -48 * time.Hour // 2 days in the past
+func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) {
+	var (
+		account *gtsmodel.Account
+		uriStr  = uri.String()
+		err     error
+	)
 
-func instanceAccount(account *gtsmodel.Account) bool {
+	// Search the database for existing account with ID URI.
-	return strings.EqualFold(account.Username, account.Domain) ||
+	account, err = d.db.GetAccountByURI(ctx, uriStr)
-		account.FollowersURI == "" ||
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		account.FollowingURI == "" ||
+		return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
-		(account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor"))
-}
-
-// GetAccountParams wraps parameters for an account lookup.
-type GetAccountParams struct {
-	// The username of the user doing the lookup request (optional).
-	// If not set, then the GtS instance account will be used to do the lookup.
-	RequestingUsername string
-	// The ActivityPub URI of the account (optional).
-	// If not set (nil), the ActivityPub URI of the account will be discovered
-	// via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost
-	// if this parameter is not set.
-	RemoteAccountID *url.URL
-	// The username of the account (optional).
-	// If RemoteAccountID is not set, then this value must be set.
-	RemoteAccountUsername string
-	// The host of the account (optional).
-	// If RemoteAccountID is not set, then this value must be set.
-	RemoteAccountHost string
-	// Whether to do a blocking call to the remote instance. If true,
-	// then the account's media and other fields will be fully dereferenced before it is returned.
-	// If false, then the account's media and other fields will be dereferenced in the background,
-	// so only a minimal account representation will be returned by GetRemoteAccount.
-	Blocking bool
-	// Whether to skip making calls to remote instances. This is useful when you want to
-	// quickly fetch a remote account from the database or fail, and don't want to cause
-	// http requests to go flying around.
-	SkipResolve bool
-	// PartialAccount can be used if the GetRemoteAccount call results from a federated/ap
-	// account update. In this case, we will already have a partial representation of the account,
-	// derived from converting the AP representation to a gtsmodel representation. If this field
-	// is provided, then GetRemoteAccount will use this as a basis for building the full account.
-	PartialAccount *gtsmodel.Account
-}
-
-type lookupType int
-
-const (
-	lookupPartialLocal lookupType = iota
-	lookupPartial
-	lookupURILocal
-	lookupURI
-	lookupMentionLocal
-	lookupMention
-	lookupBad
-)
-
-func getLookupType(params GetAccountParams) lookupType {
-	switch {
-	case params.PartialAccount != nil:
-		if params.PartialAccount.Domain == "" || params.PartialAccount.Domain == config.GetHost() || params.PartialAccount.Domain == config.GetAccountDomain() {
-			return lookupPartialLocal
-		}
-		return lookupPartial
-	case params.RemoteAccountID != nil:
-		if host := params.RemoteAccountID.Host; host == config.GetHost() || host == config.GetAccountDomain() {
-			return lookupURILocal
-		}
-		return lookupURI
-	case params.RemoteAccountUsername != "":
-		if params.RemoteAccountHost == "" || params.RemoteAccountHost == config.GetHost() || params.RemoteAccountHost == config.GetAccountDomain() {
-			return lookupMentionLocal
-		}
-		return lookupMention
-	default:
-		return lookupBad
-	}
-}
-
-// GetAccount completely dereferences an account, converts it to a GtS model account,
-// puts or updates it in the database (if necessary), and returns it to a caller.
-//
-// GetAccount will guard against trying to do http calls to fetch an account that belongs to this instance.
-// Instead of making calls, it will just return the account early if it finds it, or return an error.
-//
-// Even if a fastfail context is used, and something goes wrong, an account might still be returned instead
-// of an error, if we already had the account in our database (in other words, if we just needed to try
-// fingering/refreshing the account again). The rationale for this is that it's more useful to be able
-// to provide *something* to the caller, even if that something is not necessarily 100% up to date.
-func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundAccount *gtsmodel.Account, err error) {
-	/*
-		In this function we want to retrieve a gtsmodel representation of a remote account, with its proper
-		accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth.
-
-		There are a few different paths through this function, and the path taken depends on how much
-		initial information we are provided with via parameters, how much information we already have stored,
-		and what we're allowed to do according to the parameters we've been passed.
-
-		Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the
-		            account username + host, so we can check in our database and return if possible.
-
-		Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host.
-		            In this case, we can use the URI to resolve the remote account and find the username,
-					and then we can webfinger the account to discover the accountDomain if necessary.
-
-		Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI.
-		            In this case, we can webfinger the account to discover the URI, and then dereference
-					from that.
-	*/
-
-	// this first step checks if we have the
-	// account in the database somewhere already,
-	// or if we've been provided it as a partial
-	switch getLookupType(params) {
-	case lookupPartialLocal:
-		params.SkipResolve = true
-		fallthrough
-	case lookupPartial:
-		foundAccount = params.PartialAccount
-	case lookupURILocal:
-		params.SkipResolve = true
-		fallthrough
-	case lookupURI:
-		// see if we have this in the db already with this uri/url
-		uri := params.RemoteAccountID.String()
-
-		if a, dbErr := d.db.GetAccountByURI(ctx, uri); dbErr == nil {
-			// got it, break here to leave early
-			foundAccount = a
-			break
-		} else if !errors.Is(dbErr, db.ErrNoEntries) {
-			// a real error
-			err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with uri %s: %w", uri, dbErr))
-			break
-		}
-
-		// dbErr was just db.ErrNoEntries so search by url instead
-		if a, dbErr := d.db.GetAccountByURL(ctx, uri); dbErr == nil {
-			// got it
-			foundAccount = a
-			break
-		} else if !errors.Is(dbErr, db.ErrNoEntries) {
-			// a real error
-			err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with url %s: %w", uri, dbErr))
-			break
-		}
-	case lookupMentionLocal:
-		params.SkipResolve = true
-		params.RemoteAccountHost = ""
-		fallthrough
-	case lookupMention:
-		// see if we have this in the db already with this username/host
-		if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, params.RemoteAccountHost); dbErr == nil {
-			foundAccount = a
-		} else if !errors.Is(dbErr, db.ErrNoEntries) {
-			// a real error
-			err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr))
-		}
-	default:
-		err = newErrBadRequest(errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account"))
 	}
 
-	// bail if we've set a real error, and not just no entries in the db
+	if account == nil {
+		// Else, search the database for existing by ID URL.
+		account, err = d.db.GetAccountByURL(ctx, uriStr)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
+		}
+	}
+
+	if account == nil {
+		// Ensure that this is isn't a search for a local account.
+		if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
+			return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
+		}
+
+		// Create and pass-through a new bare-bones model for dereferencing.
+		return d.enrichAccount(ctx, requestUser, uri, &gtsmodel.Account{
+			ID:     id.NewULID(),
+			Domain: uri.Host,
+			URI:    uriStr,
+		}, false, true)
+	}
+
+	// Try to update existing account model
+	enriched, err := d.enrichAccount(ctx, requestUser, uri, account, false, block)
 	if err != nil {
-		return
+		log.Errorf("error enriching remote account: %v", err)
+		return account, nil // fall back to returning existing
 	}
 
-	if params.SkipResolve {
+	return enriched, nil
-		// if we can't resolve, return already since there's nothing more we can do
+}
-		if foundAccount == nil {
+
-			err = newErrNotRetrievable(errors.New("GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it"))
+func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) {
+	if domain == config.GetHost() || domain == config.GetAccountDomain() {
+		// We do local lookups using an empty domain,
+		// else it will fail the db search below.
+		domain = ""
+	}
+
+	// Search the database for existing account with USERNAME@DOMAIN
+	account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
+	}
+
+	if account == nil {
+		// Check for failed local lookup.
+		if domain == "" {
+			return nil, NewErrNotRetrievable(err) // will be db.ErrNoEntries
 		}
-		return
+
+		// Create and pass-through a new bare-bones model for dereferencing.
+		return d.enrichAccount(ctx, requestUser, nil, &gtsmodel.Account{
+			ID:       id.NewULID(),
+			Username: username,
+			Domain:   domain,
+		}, false, true)
 	}
 
-	// if we reach this point, we have some remote calls to make
+	// Try to update existing account model
+	enriched, err := d.enrichAccount(ctx, requestUser, nil, account, false, block)
+	if err != nil {
+		log.Errorf("GetAccountByUsernameDomain: error enriching account from remote: %v", err)
+		return account, nil // fall back to returning unchanged existing account model
+	}
 
-	var accountable ap.Accountable
+	return enriched, nil
-	if params.RemoteAccountUsername == "" && params.RemoteAccountHost == "" {
+}
-		// if we're still missing some params, try to populate them now
-		params.RemoteAccountHost = params.RemoteAccountID.Host
-		if foundAccount != nil {
-			// username is easy if we found something already
-			params.RemoteAccountUsername = foundAccount.Username
-		} else {
-			// if we didn't already have it, we have to dereference it from remote
-			var derefErr error
-			accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
-			if derefErr != nil {
-				err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable")
-				return
-			}
 
-			var apError error
+func (d *deref) UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) {
-			params.RemoteAccountUsername, apError = ap.ExtractPreferredUsername(accountable)
+	return d.enrichAccount(ctx, requestUser, nil, account, force, false)
-			if apError != nil {
+}
-				err = newErrOther(fmt.Errorf("GetRemoteAccount: error extracting Accountable username: %w", apError))
+
-				return
+// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary.
-			}
+func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, force, block bool) (*gtsmodel.Account, error) {
+	if account.IsLocal() {
+		// Can't update local accounts.
+		return account, nil
+	}
+
+	if !account.CreatedAt.IsZero() && account.IsInstance() {
+		// Existing instance account. No need for update.
+		return account, nil
+	}
+
+	if !force {
+		const interval = time.Hour * 48
+
+		// If this account was updated recently (last interval), we return as-is.
+		if next := account.FetchedAt.Add(interval); time.Now().Before(next) {
+			return account, nil
 		}
 	}
 
-	// if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set
+	if account.Username != "" {
-	// params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it
+		// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
+		accDomain, accURI, err := d.fingerRemoteAccount(ctx, requestUser, account.Username, account.Domain)
 
-	// we finger to fetch the account domain but just in case we're not fingering, make a best guess
+		if err != nil && account.URI == "" {
-	// already about what the account domain might be; this var will be overwritten later if necessary
+			// this is a new account (to us) with username@domain but failed
-	var accountDomain string
+			// webfinger, there is nothing more we can do in this situation.
-	switch {
+			return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err)
-	case foundAccount != nil:
-		accountDomain = foundAccount.Domain
-	case params.RemoteAccountID != nil:
-		accountDomain = params.RemoteAccountID.Host
-	default:
-		accountDomain = params.RemoteAccountHost
-	}
-
-	// to save on remote calls, only webfinger if:
-	// - we don't know the remote account ActivityPub ID yet OR
-	// - we haven't found the account yet in some other way OR
-	// - we were passed a partial account in params OR
-	// - we haven't webfingered the account for two days AND the account isn't an instance account
-	var fingered time.Time
-	var refreshFinger bool
-	if foundAccount != nil {
-		refreshFinger = foundAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(foundAccount)
-	}
-
-	if params.RemoteAccountID == nil || foundAccount == nil || params.PartialAccount != nil || refreshFinger {
-		if ad, accountURI, fingerError := d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost); fingerError != nil {
-			if !refreshFinger {
-				// only return with an error if this wasn't just a refresh finger;
-				// that is, if we actually *needed* to finger in order to get the account,
-				// otherwise we can just continue and we'll try again in 2 days
-				err = newErrNotRetrievable(fmt.Errorf("GetRemoteAccount: error while fingering: %w", fingerError))
-				return
-			}
-			log.Infof("error doing non-vital webfinger refresh call to %s: %s", params.RemoteAccountHost, err)
-		} else {
-			accountDomain = ad
-			params.RemoteAccountID = accountURI
 		}
-		fingered = time.Now()
-	}
 
-	if !fingered.IsZero() && foundAccount == nil {
+		if err == nil {
-		// if we just fingered and now have a discovered account domain but still no account,
+			// Update account with latest info.
-		// we should do a final lookup in the database with the discovered username + accountDomain
+			account.URI = accURI.String()
-		// to make absolutely sure we don't already have this account
+			account.Domain = accDomain
-		if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, accountDomain); dbErr == nil {
+			uri = accURI
-			foundAccount = a
-		} else if !errors.Is(dbErr, db.ErrNoEntries) {
-			// a real error
-			err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr))
-			return
 		}
 	}
 
-	// we may have some extra information already, like the account we had in the db, or the
+	if uri == nil {
-	// accountable representation that we dereferenced from remote
+		var err error
-	if foundAccount == nil {
-		// if we still don't have a remoteAccountID here we're boned
-		if params.RemoteAccountID == nil {
-			err = newErrNotRetrievable(errors.New("GetRemoteAccount: could not populate find an account nor populate params.RemoteAccountID"))
-			return
-		}
 
-		// deference accountable if we didn't earlier
+		// No URI provided / found, must parse from account.
-		if accountable == nil {
+		uri, err = url.Parse(account.URI)
-			var derefErr error
-			accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
-			if derefErr != nil {
-				err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable")
-				return
-			}
-		}
-
-		// then convert
-		foundAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false)
 		if err != nil {
-			err = newErrOther(fmt.Errorf("GetRemoteAccount: error converting Accountable to account: %w", err))
+			return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
-			return
-		}
-
-		// this is a new account so we need to generate a new ID for it
-		var ulid string
-		ulid, err = id.NewRandomULID()
-		if err != nil {
-			err = newErrOther(fmt.Errorf("GetRemoteAccount: error generating new id for account: %w", err))
-			return
-		}
-		foundAccount.ID = ulid
-
-		if _, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking); populateErr != nil {
-			// it's not the end of the world if we can't populate account fields, but we do want to log it
-			log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr)
-		}
-
-		foundAccount.LastWebfingeredAt = fingered
-		foundAccount.UpdatedAt = time.Now()
-
-		if dbErr := d.db.PutAccount(ctx, foundAccount); dbErr != nil {
-			err = newErrDB(fmt.Errorf("GetRemoteAccount: error putting new account: %w", dbErr))
-			return
-		}
-
-		return // the new account
-	}
-
-	// we had the account already, but now we know the account domain, so update it if it's different
-	var accountDomainChanged bool
-	if !strings.EqualFold(foundAccount.Domain, accountDomain) {
-		accountDomainChanged = true
-		foundAccount.Domain = accountDomain
-	}
-
-	// if SharedInboxURI is nil, that means we don't know yet if this account has
-	// a shared inbox available for it, so we need to check this here
-	var sharedInboxChanged bool
-	if foundAccount.SharedInboxURI == nil {
-		// we need the accountable for this, so get it if we don't have it yet
-		if accountable == nil {
-			var derefErr error
-			accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
-			if derefErr != nil {
-				err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable")
-				return
-			}
-		}
-
-		// This can be:
-		// - an empty string (we know it doesn't have a shared inbox) OR
-		// - a string URL (we know it does a shared inbox).
-		// Set it either way!
-		var sharedInbox string
-
-		if sharedInboxURI := ap.ExtractSharedInbox(accountable); sharedInboxURI != nil {
-			// only trust shared inbox if it has at least two domains,
-			// from the right, in common with the domain of the account
-			if dns.CompareDomainName(foundAccount.Domain, sharedInboxURI.Host) >= 2 {
-				sharedInbox = sharedInboxURI.String()
-			}
-		}
-
-		sharedInboxChanged = true
-		foundAccount.SharedInboxURI = &sharedInbox
-	}
-
-	// make sure the account fields are populated before returning:
-	// the caller might want to block until everything is loaded
-	fieldsChanged, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking)
-	if populateErr != nil {
-		// it's not the end of the world if we can't populate account fields, but we do want to log it
-		log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr)
-	}
-
-	var fingeredChanged bool
-	if !fingered.IsZero() {
-		fingeredChanged = true
-		foundAccount.LastWebfingeredAt = fingered
-	}
-
-	if accountDomainChanged || sharedInboxChanged || fieldsChanged || fingeredChanged {
-		if dbErr := d.db.UpdateAccount(ctx, foundAccount); dbErr != nil {
-			err = newErrDB(fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %w", dbErr))
-			return
 		}
 	}
 
-	return // the account we already had + possibly updated
+	// Check whether this account URI is a blocked domain / subdomain
+	if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil {
+		return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err))
+	} else if blocked {
+		return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
+	}
+
+	// Mark deref+update handshake start
+	d.startHandshake(requestUser, uri)
+	defer d.stopHandshake(requestUser, uri)
+
+	// Dereference this account to get the latest available.
+	apubAcc, err := d.dereferenceAccountable(ctx, requestUser, uri)
+	if err != nil {
+		return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err)
+	}
+
+	// Convert the dereferenced AP account object to our GTS model.
+	latestAcc, err := d.typeConverter.ASRepresentationToAccount(
+		ctx, apubAcc, account.Domain,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err)
+	}
+
+	if account.Username == "" {
+		// No username was provided, so no webfinger was attempted earlier.
+		//
+		// Now we have a username we can attempt it now, this ensures up-to-date accountdomain info.
+		accDomain, _, err := d.fingerRemoteAccount(ctx, requestUser, latestAcc.Username, uri.Host)
+
+		if err == nil {
+			// Update account with latest info.
+			latestAcc.Domain = accDomain
+		}
+	}
+
+	// Ensure ID is set and update fetch time.
+	latestAcc.ID = account.ID
+	latestAcc.FetchedAt = time.Now()
+
+	// Fetch latest account media (TODO: check for changed URI to previous).
+	if err = d.fetchRemoteAccountMedia(ctx, latestAcc, requestUser, block); err != nil {
+		log.Errorf("error fetching remote media for account %s: %v", uri, err)
+	}
+
+	// Fetch the latest remote account emoji IDs used in account display name/bio.
+	_, err = d.fetchRemoteAccountEmojis(ctx, latestAcc, requestUser)
+	if err != nil {
+		log.Errorf("error fetching remote emojis for account %s: %v", uri, err)
+	}
+
+	if account.CreatedAt.IsZero() {
+		// CreatedAt will be zero if no local copy was
+		// found in one of the GetAccountBy___() functions.
+		//
+		// Set time of creation from the last-fetched date.
+		latestAcc.CreatedAt = latestAcc.FetchedAt
+		latestAcc.UpdatedAt = latestAcc.FetchedAt
+
+		// This is a new account, we need to place it in the database.
+		if err := d.db.PutAccount(ctx, latestAcc); err != nil {
+			return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
+		}
+	} else {
+		// Set time of update from the last-fetched date.
+		latestAcc.UpdatedAt = latestAcc.FetchedAt
+
+		// Use existing account values.
+		latestAcc.CreatedAt = account.CreatedAt
+		latestAcc.Language = account.Language
+
+		// This is an existing account, update the model in the database.
+		if err := d.db.UpdateAccount(ctx, latestAcc); err != nil {
+			return nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
+		}
+	}
+
+	return latestAcc, nil
 }
 
 // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
@@ -414,13 +259,6 @@ func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundA
 //
 // Will work for Person, Application, or Service models.
 func (d *deref) dereferenceAccountable(ctx context.Context, username string, remoteAccountID *url.URL) (ap.Accountable, error) {
-	d.startHandshake(username, remoteAccountID)
-	defer d.stopHandshake(username, remoteAccountID)
-
-	if blocked, err := d.db.IsDomainBlocked(ctx, remoteAccountID.Host); blocked || err != nil {
-		return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host)
-	}
-
 	transport, err := d.transportController.NewTransportForUsername(ctx, username)
 	if err != nil {
 		return nil, fmt.Errorf("DereferenceAccountable: transport err: %w", err)
@@ -441,83 +279,23 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
 		return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err)
 	}
 
+	//nolint shutup linter
 	switch t.GetTypeName() {
 	case ap.ActorApplication:
-		p, ok := t.(vocab.ActivityStreamsApplication)
+		return t.(vocab.ActivityStreamsApplication), nil
-		if !ok {
-			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application")
-		}
-		return p, nil
 	case ap.ActorGroup:
-		p, ok := t.(vocab.ActivityStreamsGroup)
+		return t.(vocab.ActivityStreamsGroup), nil
-		if !ok {
-			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams group")
-		}
-		return p, nil
 	case ap.ActorOrganization:
-		p, ok := t.(vocab.ActivityStreamsOrganization)
+		return t.(vocab.ActivityStreamsOrganization), nil
-		if !ok {
-			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams organization")
-		}
-		return p, nil
 	case ap.ActorPerson:
-		p, ok := t.(vocab.ActivityStreamsPerson)
+		return t.(vocab.ActivityStreamsPerson), nil
-		if !ok {
-			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person")
-		}
-		return p, nil
 	case ap.ActorService:
-		p, ok := t.(vocab.ActivityStreamsService)
+		return t.(vocab.ActivityStreamsService), nil
-		if !ok {
-			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service")
-		}
-		return p, nil
 	}
 
 	return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName()))
 }
 
-// populateAccountFields makes a best effort to populate fields on an account such as emojis, avatar, header.
-// Will return true if one of these things changed on the passed-in account.
-func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
-	// if we're dealing with an instance account, just bail, we don't need to do anything
-	if instanceAccount(account) {
-		return false, nil
-	}
-
-	accountURI, err := url.Parse(account.URI)
-	if err != nil {
-		return false, fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %w", account.URI, err)
-	}
-
-	blocked, dbErr := d.db.IsDomainBlocked(ctx, accountURI.Host)
-	if dbErr != nil {
-		return false, fmt.Errorf("populateAccountFields: eror checking for block of domain %s: %w", accountURI.Host, err)
-	}
-
-	if blocked {
-		return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host)
-	}
-
-	var changed bool
-
-	// fetch the header and avatar
-	if mediaChanged, err := d.fetchRemoteAccountMedia(ctx, account, requestingUsername, blocking); err != nil {
-		return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %w", err)
-	} else if mediaChanged {
-		changed = mediaChanged
-	}
-
-	// fetch any emojis used in note, fields, display name, etc
-	if emojisChanged, err := d.fetchRemoteAccountEmojis(ctx, account, requestingUsername); err != nil {
-		return false, fmt.Errorf("populateAccountFields: error fetching emojis for account: %w", err)
-	} else if emojisChanged {
-		changed = emojisChanged
-	}
-
-	return changed, nil
-}
-
 // fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account,
 // using a transport on behalf of requestingUsername.
 //
@@ -530,39 +308,30 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
 //
 // If blocking is true, then the calls to the media manager made by this function will be blocking:
 // in other words, the function won't return until the header and the avatar have been fully processed.
-func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
+func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) error {
-	var (
+	// Fetch a transport beforehand for either(or both) avatar / header dereferencing.
-		changed bool
+	tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
-		t       transport.Transport
+	if err != nil {
-	)
+		return fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
+	}
 
-	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") {
+	if targetAccount.AvatarRemoteURL != "" {
 		var processingMedia *media.ProcessingMedia
 
+		// Parse the target account's avatar URL into URL object.
+		avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
+		if err != nil {
+			return err
+		}
+
 		d.dereferencingAvatarsLock.Lock() // LOCK HERE
 		// first check if we're already processing this media
 		if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok {
 			// we're already on it, no worries
 			processingMedia = alreadyProcessing
 		} else {
-			// we're not already processing it so start now
-			avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
-			if err != nil {
-				d.dereferencingAvatarsLock.Unlock()
-				return changed, err
-			}
-
-			if t == nil {
-				var err error
-				t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername)
-				if err != nil {
-					d.dereferencingAvatarsLock.Unlock()
-					return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
-				}
-			}
-
 			data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
-				return t.DereferenceMedia(innerCtx, avatarIRI)
+				return tsport.DereferenceMedia(innerCtx, avatarIRI)
 			}
 
 			avatar := true
@@ -572,7 +341,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 			})
 			if err != nil {
 				d.dereferencingAvatarsLock.Unlock()
-				return changed, err
+				return err
 			}
 
 			// store it in our map to indicate it's in process
@@ -595,7 +364,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 		// block until loaded if required...
 		if blocking {
 			if err := loadAndCleanup(ctx, load, cleanup); err != nil {
-				return changed, err
+				return err
 			}
 		} else {
 			// ...otherwise do it async
@@ -609,36 +378,25 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 		}
 
 		targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID()
-		changed = true
 	}
 
-	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") {
+	if targetAccount.HeaderRemoteURL != "" {
 		var processingMedia *media.ProcessingMedia
 
+		// Parse the target account's header URL into URL object.
+		headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
+		if err != nil {
+			return err
+		}
+
 		d.dereferencingHeadersLock.Lock() // LOCK HERE
 		// first check if we're already processing this media
 		if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok {
 			// we're already on it, no worries
 			processingMedia = alreadyProcessing
 		} else {
-			// we're not already processing it so start now
-			headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
-			if err != nil {
-				d.dereferencingAvatarsLock.Unlock()
-				return changed, err
-			}
-
-			if t == nil {
-				var err error
-				t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername)
-				if err != nil {
-					d.dereferencingAvatarsLock.Unlock()
-					return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
-				}
-			}
-
 			data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
-				return t.DereferenceMedia(innerCtx, headerIRI)
+				return tsport.DereferenceMedia(innerCtx, headerIRI)
 			}
 
 			header := true
@@ -648,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 			})
 			if err != nil {
 				d.dereferencingAvatarsLock.Unlock()
-				return changed, err
+				return err
 			}
 
 			// store it in our map to indicate it's in process
@@ -671,7 +429,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 		// block until loaded if required...
 		if blocking {
 			if err := loadAndCleanup(ctx, load, cleanup); err != nil {
-				return changed, err
+				return err
 			}
 		} else {
 			// ...otherwise do it async
@@ -685,10 +443,9 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 		}
 
 		targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID()
-		changed = true
 	}
 
-	return changed, nil
+	return nil
 }
 
 func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {

internal/federation/dereferencing/account_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -39,17 +38,19 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
-	group, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	group, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername: fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountID:    groupURL,
+		fetchingAccount.Username,
-	})
+		groupURL,
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(group)
 
 	// group values should be set
 	suite.Equal("https://unknown-instance.com/groups/some_group", group.URI)
 	suite.Equal("https://unknown-instance.com/@some_group", group.URL)
-	suite.WithinDuration(time.Now(), group.LastWebfingeredAt, 5*time.Second)
+	suite.WithinDuration(time.Now(), group.FetchedAt, 5*time.Second)
 
 	// group should be in the database
 	dbGroup, err := suite.db.GetAccountByURI(context.Background(), group.URI)
@@ -62,17 +63,19 @@ func (suite *AccountTestSuite) TestDereferenceService() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
-	service, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	service, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername: fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountID:    serviceURL,
+		fetchingAccount.Username,
-	})
+		serviceURL,
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(service)
 
 	// service values should be set
 	suite.Equal("https://owncast.example.org/federation/user/rgh", service.URI)
 	suite.Equal("https://owncast.example.org/federation/user/rgh", service.URL)
-	suite.WithinDuration(time.Now(), service.LastWebfingeredAt, 5*time.Second)
+	suite.WithinDuration(time.Now(), service.FetchedAt, 5*time.Second)
 
 	// service should be in the database
 	dbService, err := suite.db.GetAccountByURI(context.Background(), service.URI)
@@ -93,10 +96,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername: fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountID:    testrig.URLMustParse(targetAccount.URI),
+		fetchingAccount.Username,
-	})
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -111,10 +116,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb
 		suite.FailNow(err.Error())
 	}
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername: fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountID:    testrig.URLMustParse(targetAccount.URI),
+		fetchingAccount.Username,
-	})
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -124,10 +131,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername:    fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountUsername: targetAccount.Username,
+		fetchingAccount.Username,
-	})
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -137,11 +146,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername:    fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountUsername: targetAccount.Username,
+		fetchingAccount.Username,
-		RemoteAccountHost:     config.GetHost(),
+		testrig.URLMustParse(targetAccount.URI),
-	})
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -151,12 +161,13 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
-		RequestingUsername:    fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountID:       testrig.URLMustParse(targetAccount.URI),
+		fetchingAccount.Username,
-		RemoteAccountUsername: targetAccount.Username,
+		targetAccount.Username,
-		RemoteAccountHost:     config.GetHost(),
+		config.GetHost(),
-	})
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -165,248 +176,50 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
 func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
-		RequestingUsername:    fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountUsername: "thisaccountdoesnotexist",
+		fetchingAccount.Username,
-	})
+		"thisaccountdoesnotexist",
+		config.GetHost(),
+		false,
+	)
 	var errNotRetrievable *dereferencing.ErrNotRetrievable
 	suite.ErrorAs(err, &errNotRetrievable)
-	suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
+	suite.EqualError(err, "item could not be retrieved: no entries")
 	suite.Nil(fetchedAccount)
 }
 
 func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
-		RequestingUsername:    fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountUsername: "thisaccountdoesnotexist",
+		fetchingAccount.Username,
-		RemoteAccountHost:     "localhost:8080",
+		"thisaccountdoesnotexist",
-	})
+		"localhost:8080",
+		false,
+	)
 	var errNotRetrievable *dereferencing.ErrNotRetrievable
 	suite.ErrorAs(err, &errNotRetrievable)
-	suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
+	suite.EqualError(err, "item could not be retrieved: no entries")
 	suite.Nil(fetchedAccount)
 }
 
 func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
-		RequestingUsername: fetchingAccount.Username,
+		context.Background(),
-		RemoteAccountID:    testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),
+		fetchingAccount.Username,
-	})
+		testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),
+		false,
+	)
 	var errNotRetrievable *dereferencing.ErrNotRetrievable
 	suite.ErrorAs(err, &errNotRetrievable)
-	suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
+	suite.EqualError(err, "item could not be retrieved: no entries")
 	suite.Nil(fetchedAccount)
 }
 
-func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() {
-	fetchingAccount := suite.testAccounts["local_account_1"]
-
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountPartial := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// dereference an emoji we don't have stored yet
-			{
-				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
-				Shortcode:       "kip_van_den_bos",
-				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageUpdatedAt:  testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",
-				Disabled:        testrig.FalseBool(),
-				VisibleInPicker: testrig.FalseBool(),
-				Domain:          "fossbros-anonymous.io",
-			},
-		},
-	}
-
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount)
-	suite.NotNil(fetchedAccount.EmojiIDs)
-	suite.NotNil(fetchedAccount.Emojis)
-}
-
-func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() {
-	fetchingAccount := suite.testAccounts["local_account_1"]
-
-	knownEmoji := suite.testEmojis["yell"]
-
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountPartial := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// an emoji we already have
-			{
-				URI:             knownEmoji.URI,
-				Shortcode:       knownEmoji.Shortcode,
-				UpdatedAt:       knownEmoji.UpdatedAt,
-				ImageUpdatedAt:  knownEmoji.ImageUpdatedAt,
-				ImageRemoteURL:  knownEmoji.ImageRemoteURL,
-				Disabled:        knownEmoji.Disabled,
-				VisibleInPicker: knownEmoji.VisibleInPicker,
-				Domain:          knownEmoji.Domain,
-			},
-		},
-	}
-
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount)
-	suite.NotNil(fetchedAccount.EmojiIDs)
-	suite.NotNil(fetchedAccount.Emojis)
-}
-
-func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() {
-	fetchingAccount := suite.testAccounts["local_account_1"]
-
-	knownEmoji := suite.testEmojis["yell"]
-
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountPartial := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// an emoji we already have
-			{
-				URI:             knownEmoji.URI,
-				Shortcode:       knownEmoji.Shortcode,
-				UpdatedAt:       knownEmoji.UpdatedAt,
-				ImageUpdatedAt:  knownEmoji.ImageUpdatedAt,
-				ImageRemoteURL:  knownEmoji.ImageRemoteURL,
-				Disabled:        knownEmoji.Disabled,
-				VisibleInPicker: knownEmoji.VisibleInPicker,
-				Domain:          knownEmoji.Domain,
-			},
-		},
-	}
-
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount)
-	suite.NotNil(fetchedAccount.EmojiIDs)
-	suite.NotNil(fetchedAccount.Emojis)
-	suite.Equal(knownEmoji.URI, fetchedAccount.Emojis[0].URI)
-
-	remoteAccountPartial2 := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// dereference an emoji we don't have stored yet
-			{
-				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
-				Shortcode:       "kip_van_den_bos",
-				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageUpdatedAt:  testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",
-				Disabled:        testrig.FalseBool(),
-				VisibleInPicker: testrig.FalseBool(),
-				Domain:          "fossbros-anonymous.io",
-			},
-		},
-	}
-
-	fetchedAccount2, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial2,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount2)
-	suite.NotNil(fetchedAccount2.EmojiIDs)
-	suite.NotNil(fetchedAccount2.Emojis)
-	suite.Equal("http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", fetchedAccount2.Emojis[0].URI)
-}
-
 func TestAccountTestSuite(t *testing.T) {
 	suite.Run(t, new(AccountTestSuite))
 }

internal/federation/dereferencing/dereferencer.go

@@ -33,7 +33,17 @@ import (
 
 // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
 type Dereferencer interface {
-	GetAccount(ctx context.Context, params GetAccountParams) (*gtsmodel.Account, error)
+	// GetAccountByURI will attempt to fetch an account by its URI, first checking the database and in the case of a remote account will either check the
+	// last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
+	GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error)
+
+	// GetAccountByUsernameDomain will attempt to fetch an account by username@domain, first checking the database and in the case of a remote account will either
+	// check the last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
+	GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error)
+
+	// UpdateAccount updates the given account if last_fetched is beyond fetch interval (or if force is set). An updated account model is returned, any media fetching is done async.
+	UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error)
+
 	GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
 
 	EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
@@ -44,7 +54,7 @@ type Dereferencer interface {
 	GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error)
 	GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error)
 
-	Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool
+	Handshaking(username string, remoteAccountID *url.URL) bool
 }
 
 type deref struct {
@@ -53,28 +63,25 @@ type deref struct {
 	transportController      transport.Controller
 	mediaManager             media.Manager
 	dereferencingAvatars     map[string]*media.ProcessingMedia
-	dereferencingAvatarsLock *sync.Mutex
+	dereferencingAvatarsLock sync.Mutex
 	dereferencingHeaders     map[string]*media.ProcessingMedia
-	dereferencingHeadersLock *sync.Mutex
+	dereferencingHeadersLock sync.Mutex
 	dereferencingEmojis      map[string]*media.ProcessingEmoji
-	dereferencingEmojisLock  *sync.Mutex
+	dereferencingEmojisLock  sync.Mutex
 	handshakes               map[string][]*url.URL
-	handshakeSync            *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
+	handshakeSync            sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
 }
 
 // NewDereferencer returns a Dereferencer initialized with the given parameters.
 func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
 	return &deref{
-		db:                       db,
+		db:                   db,
-		typeConverter:            typeConverter,
+		typeConverter:        typeConverter,
-		transportController:      transportController,
+		transportController:  transportController,
-		mediaManager:             mediaManager,
+		mediaManager:         mediaManager,
-		dereferencingAvatars:     make(map[string]*media.ProcessingMedia),
+		dereferencingAvatars: make(map[string]*media.ProcessingMedia),
-		dereferencingAvatarsLock: &sync.Mutex{},
+		dereferencingHeaders: make(map[string]*media.ProcessingMedia),
-		dereferencingHeaders:     make(map[string]*media.ProcessingMedia),
+		dereferencingEmojis:  make(map[string]*media.ProcessingEmoji),
-		dereferencingHeadersLock: &sync.Mutex{},
+		handshakes:           make(map[string][]*url.URL),
-		dereferencingEmojis:      make(map[string]*media.ProcessingEmoji),
-		dereferencingEmojisLock:  &sync.Mutex{},
-		handshakeSync:            &sync.Mutex{},
 	}
 }

internal/federation/dereferencing/error.go

@@ -49,24 +49,10 @@ func (err *ErrNotRetrievable) Error() string {
 	return fmt.Sprintf("item could not be retrieved: %v", err.wrapped)
 }
 
-func newErrNotRetrievable(err error) error {
+func NewErrNotRetrievable(err error) error {
 	return &ErrNotRetrievable{wrapped: err}
 }
 
-// ErrBadRequest denotes that insufficient or improperly formed parameters
-// were passed into one of the dereference functions.
-type ErrBadRequest struct {
-	wrapped error
-}
-
-func (err *ErrBadRequest) Error() string {
-	return fmt.Sprintf("bad request: %v", err.wrapped)
-}
-
-func newErrBadRequest(err error) error {
-	return &ErrBadRequest{wrapped: err}
-}
-
 // ErrTransportError indicates that something unforeseen went wrong creating
 // a transport, or while making an http call to a remote resource with a transport.
 type ErrTransportError struct {
@@ -121,7 +107,7 @@ func wrapDerefError(derefErr error, fluff string) error {
 
 	switch {
 	case errors.Is(derefErr, transport.ErrGone):
-		err = newErrNotRetrievable(err)
+		err = NewErrNotRetrievable(err)
 	case errors.As(derefErr, &errWrongType):
 		err = newErrWrongType(err)
 	default:

internal/federation/dereferencing/handshake.go

@@ -19,11 +19,10 @@
 package dereferencing
 
 import (
-	"context"
 	"net/url"
 )
 
-func (d *deref) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool {
+func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool {
 	d.handshakeSync.Lock()
 	defer d.handshakeSync.Unlock()
 
@@ -53,11 +52,6 @@ func (d *deref) startHandshake(username string, remoteAccountID *url.URL) {
 	d.handshakeSync.Lock()
 	defer d.handshakeSync.Unlock()
 
-	// lazily initialize handshakes
-	if d.handshakes == nil {
-		d.handshakes = make(map[string][]*url.URL)
-	}
-
 	remoteIDs, ok := d.handshakes[username]
 	if !ok {
 		// there was nothing in there yet, so just add this entry and return
@@ -74,13 +68,8 @@ func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) {
 	d.handshakeSync.Lock()
 	defer d.handshakeSync.Unlock()
 
-	if d.handshakes == nil {
-		return
-	}
-
 	remoteIDs, ok := d.handshakes[username]
 	if !ok {
-		// there was nothing in there yet anyway so just bail
 		return
 	}
 

internal/federation/dereferencing/status.go

@@ -100,7 +100,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U
 		if status != nil {
 			return status, nil, nil
 		}
-		return nil, nil, newErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString))
+		return nil, nil, NewErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString))
 	}
 
 	// if we got here, either we didn't have the status
@@ -123,11 +123,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U
 	}
 
 	// we need to get the author of the status else we can't serialize it properly
-	if _, err = d.GetAccount(ctx, GetAccountParams{
+	if _, err = d.GetAccountByURI(ctx, username, accountURI, true); err != nil {
-		RequestingUsername: username,
-		RemoteAccountID:    accountURI,
-		Blocking:           true,
-	}); err != nil {
 		return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err))
 	}
 
@@ -353,10 +349,7 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta
 		if targetAccount == nil {
 			// we didn't find the account in our database already
 			// check if we can get the account remotely (dereference it)
-			if a, err := d.GetAccount(ctx, GetAccountParams{
+			if a, err := d.GetAccountByURI(ctx, requestingUsername, targetAccountURI, false); err != nil {
-				RequestingUsername: requestingUsername,
-				RemoteAccountID:    targetAccountURI,
-			}); err != nil {
 				errs = append(errs, err.Error())
 			} else {
 				log.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)

internal/federation/federatingdb/create.go

@@ -100,11 +100,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec
 		return fmt.Errorf("activityBlock: could not convert Block to gts model block")
 	}
 
-	newID, err := id.NewULID()
+	block.ID = id.NewULID()
-	if err != nil {
-		return err
-	}
-	block.ID = newID
 
 	if err := f.db.PutBlock(ctx, block); err != nil {
 		return fmt.Errorf("activityBlock: database error inserting block: %s", err)
@@ -263,11 +259,7 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re
 		return fmt.Errorf("activityFollow: could not convert Follow to follow request: %s", err)
 	}
 
-	newID, err := id.NewULID()
+	followRequest.ID = id.NewULID()
-	if err != nil {
-		return err
-	}
-	followRequest.ID = newID
 
 	if err := f.db.Put(ctx, followRequest); err != nil {
 		return fmt.Errorf("activityFollow: database error inserting follow request: %s", err)
@@ -298,11 +290,7 @@ func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, rece
 		return fmt.Errorf("activityLike: could not convert Like to fave: %s", err)
 	}
 
-	newID, err := id.NewULID()
+	fave.ID = id.NewULID()
-	if err != nil {
-		return err
-	}
-	fave.ID = newID
 
 	if err := f.db.Put(ctx, fave); err != nil {
 		return fmt.Errorf("activityLike: database error inserting fave: %s", err)
@@ -333,11 +321,7 @@ func (f *federatingDB) activityFlag(ctx context.Context, asType vocab.Type, rece
 		return fmt.Errorf("activityFlag: could not convert Flag to report: %w", err)
 	}
 
-	newID, err := id.NewULID()
+	report.ID = id.NewULID()
-	if err != nil {
-		return err
-	}
-	report.ID = newID
 
 	if err := f.db.PutReport(ctx, report); err != nil {
 		return fmt.Errorf("activityFlag: database error inserting report: %w", err)

internal/federation/federatingdb/update.go

@@ -116,7 +116,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 			accountable = i
 		}
 
-		updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true)
+		updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "")
 		if err != nil {
 			return fmt.Errorf("UPDATE: error converting to account: %s", err)
 		}

internal/federation/federatingprotocol.go

@@ -31,7 +31,6 @@ import (
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
@@ -206,10 +205,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		}
 	}
 
-	requestingAccount, err := f.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	requestingAccount, err := f.GetAccountByURI(
-		RequestingUsername: username,
+		transport.WithFastfail(ctx), username, publicKeyOwnerURI, false,
-		RemoteAccountID:    publicKeyOwnerURI,
+	)
-	})
 	if err != nil {
 		return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err)
 	}

internal/federation/federator.go

@@ -23,12 +23,10 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/pub"
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -53,20 +51,9 @@ type Federator interface {
 	// If something goes wrong during authentication, nil, false, and an error will be returned.
 	AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode)
 
-	/*
-		dereferencing functions
-	*/
-	DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL, status *gtsmodel.Status, statusable ap.Statusable)
-	DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
-	GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error)
-	GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
-	EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
-	GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
-
-	// Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID.
-	Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool
 	pub.CommonBehavior
 	pub.FederatingProtocol
+	dereferencing.Dereferencer
 }
 
 type federator struct {
@@ -75,9 +62,9 @@ type federator struct {
 	clock               pub.Clock
 	typeConverter       typeutils.TypeConverter
 	transportController transport.Controller
-	dereferencer        dereferencing.Dereferencer
 	mediaManager        media.Manager
 	actor               pub.FederatingActor
+	dereferencing.Dereferencer
 }
 
 // NewFederator returns a new federator
@@ -91,8 +78,8 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
 		clock:               &Clock{},
 		typeConverter:       typeConverter,
 		transportController: transportController,
-		dereferencer:        dereferencer,
 		mediaManager:        mediaManager,
+		Dereferencer:        dereferencer,
 	}
 	actor := newFederatingActor(f, f, federatingDB, clock)
 	f.actor = actor

internal/federation/gone.go

@@ -2,12 +2,10 @@ package federation
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
 // CheckGone checks if a tombstone exists in the database for AP Actor or Object with the given uri.
@@ -17,18 +15,10 @@ func (f *federator) CheckGone(ctx context.Context, uri *url.URL) (bool, error) {
 
 // HandleGone puts a tombstone in the database, which marks an AP Actor or Object with the given uri as gone.
 func (f *federator) HandleGone(ctx context.Context, uri *url.URL) error {
-	tombstoneID, err := id.NewULID()
-	if err != nil {
-		err = fmt.Errorf("HandleGone: error generating id for new tombstone %s: %s", uri, err)
-		log.Error(err)
-		return err
-	}
-
 	tombstone := &gtsmodel.Tombstone{
-		ID:     tombstoneID,
+		ID:     id.NewULID(),
 		Domain: uri.Host,
 		URI:    uri.String(),
 	}
-
 	return f.db.PutTombstone(ctx, tombstone)
 }

internal/federation/handshake.go

@@ -17,12 +17,3 @@
 */
 
 package federation
-
-import (
-	"context"
-	"net/url"
-)
-
-func (f *federator) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool {
-	return f.dereferencer.Handshaking(ctx, username, remoteAccountID)
-}

internal/gtsmodel/account.go

@@ -24,14 +24,18 @@ package gtsmodel
 
 import (
 	"crypto/rsa"
+	"strings"
 	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 )
 
 // Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc).
 type Account struct {
 	ID                      string           `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"`                                               // id of this item in the database
-	CreatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item created
+	CreatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item created.
-	UpdatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item last updated
+	UpdatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item was last updated.
+	FetchedAt               time.Time        `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"`                                                       // when was item (remote) last fetched.
 	Username                string           `validate:"required" bun:",nullzero,notnull,unique:userdomain"`                                                         // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other
 	Domain                  string           `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"`                                                           // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username.
 	AvatarMediaAttachmentID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // Database ID of the media attachment, if present
@@ -60,7 +64,6 @@ type Account struct {
 	CustomCSS               string           `validate:"-" bun:",nullzero"`                                                                                          // Custom CSS that should be displayed for this Account's profile and statuses.
 	URI                     string           `validate:"required,url" bun:",nullzero,notnull,unique"`                                                                // ActivityPub URI for this account.
 	URL                     string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Web URL for this account's profile
-	LastWebfingeredAt       time.Time        `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"`                                                       // Last time this account was refreshed/located with webfinger.
 	InboxURI                string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Address of this account's ActivityPub inbox, for sending activity to
 	SharedInboxURI          *string          `validate:"-" bun:""`                                                                                                   // Address of this account's ActivityPub sharedInbox. Gotcha warning: this is a string pointer because it has three possible states: 1. We don't know yet if the account has a shared inbox -- null. 2. We know it doesn't have a shared inbox -- empty string. 3. We know it does have a shared inbox -- url string.
 	OutboxURI               string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Address of this account's activitypub outbox
@@ -79,6 +82,24 @@ type Account struct {
 	EnableRSS               *bool            `validate:"-" bun:",default:false"`                                                                                     // enable RSS feed subscription for this account's public posts at [URL]/feed
 }
 
+// IsLocal returns whether account is a local user account.
+func (a Account) IsLocal() bool {
+	return a.Domain == "" || a.Domain == config.GetHost() || a.Domain == config.GetAccountDomain()
+}
+
+// IsRemote returns whether account is a remote user account.
+func (a Account) IsRemote() bool {
+	return !a.IsLocal()
+}
+
+// IsInstance returns whether account is an instance internal actor account.
+func (a Account) IsInstance() bool {
+	return a.Username == a.Domain ||
+		a.FollowersURI == "" ||
+		a.FollowingURI == "" ||
+		(a.Username == "internal.fetch" && strings.Contains(a.Note, "internal service actor"))
+}
+
 // AccountToEmoji is an intermediate struct to facilitate the many2many relationship between an account and one or more emojis.
 type AccountToEmoji struct {
 	AccountID string   `validate:"ulid,required" bun:"type:CHAR(26),unique:accountemoji,nullzero,notnull"`

internal/id/ulid.go

@@ -35,13 +35,15 @@ const (
 // ULID represents a Universally Unique Lexicographically Sortable Identifier of 26 characters. See https://github.com/oklog/ulid
 type ULID string
 
-// NewULID returns a new ULID string using the current time, or an error if something goes wrong.
+// NewULID returns a new ULID string using the current time.
-func NewULID() (string, error) {
+func NewULID() string {
-	newUlid, err := ulid.New(ulid.Timestamp(time.Now()), rand.Reader)
+	ulid, err := ulid.New(
+		ulid.Timestamp(time.Now()), rand.Reader,
+	)
 	if err != nil {
-		return "", err
+		panic(err)
 	}
-	return newUlid.String(), nil
+	return ulid.String()
 }
 
 // NewULIDFromTime returns a new ULID string using the given time, or an error if something goes wrong.

internal/processing/account/createblock.go

@@ -53,10 +53,7 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
 
 	// make the block
 	block := &gtsmodel.Block{}
-	newBlockID, err := id.NewULID()
+	newBlockID := id.NewULID()
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
 	block.ID = newBlockID
 	block.AccountID = requestingAccount.ID
 	block.Account = requestingAccount

internal/processing/account/get.go

@@ -26,7 +26,6 @@ import (
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
@@ -94,13 +93,9 @@ func (p *processor) getAccountFor(ctx context.Context, requestingAccount *gtsmod
 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err))
 		}
 
-		a, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+		a, err := p.federator.GetAccountByURI(
-			RequestingUsername:    requestingAccount.Username,
+			transport.WithFastfail(ctx), requestingAccount.Username, targetAccountURI, true,
-			RemoteAccountID:       targetAccountURI,
+		)
-			RemoteAccountHost:     targetAccount.Domain,
-			RemoteAccountUsername: targetAccount.Username,
-			Blocking:              true,
-		})
 		if err == nil {
 			targetAccount = a
 		}

internal/processing/admin/accountaction.go

@@ -18,13 +18,8 @@ func (p *processor) AccountAction(ctx context.Context, account *gtsmodel.Account
 		return gtserror.NewErrorInternalError(err)
 	}
 
-	adminActionID, err := id.NewULID()
-	if err != nil {
-		return gtserror.NewErrorInternalError(err)
-	}
-
 	adminAction := &gtsmodel.AdminAccountAction{
-		ID:              adminActionID,
+		ID:              id.NewULID(),
 		AccountID:       account.ID,
 		TargetAccountID: targetAccount.ID,
 		Text:            form.Text,

internal/processing/admin/createdomainblock.go

@@ -50,14 +50,8 @@ func (p *processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc
 		}
 
 		// there's no block for this domain yet so create one
-		// note: we take a new ulid from timestamp here in case we need to sort blocks
-		blockID, err := id.NewULID()
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error creating id for new domain block %s: %s", domain, err))
-		}
-
 		newBlock := &gtsmodel.DomainBlock{
-			ID:                 blockID,
+			ID:                 id.NewULID(),
 			Domain:             domain,
 			CreatedByAccountID: account.ID,
 			PrivateComment:     text.SanitizePlaintext(privateComment),

internal/processing/federation/getfollowers.go

@@ -24,7 +24,6 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
@@ -42,10 +41,9 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string,
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	requestingAccount, err := p.federator.GetAccountByURI(
-		RequestingUsername: requestedUsername,
+		transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false,
-		RemoteAccountID:    requestingAccountURI,
+	)
-	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getfollowing.go

@@ -24,7 +24,6 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
@@ -42,10 +41,9 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string,
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	requestingAccount, err := p.federator.GetAccountByURI(
-		RequestingUsername: requestedUsername,
+		transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false,
-		RemoteAccountID:    requestingAccountURI,
+	)
-	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getoutbox.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
@@ -43,10 +42,9 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	requestingAccount, err := p.federator.GetAccountByURI(
-		RequestingUsername: requestedUsername,
+		transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false,
-		RemoteAccountID:    requestingAccountURI,
+	)
-	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getstatus.go

@@ -24,7 +24,6 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
@@ -42,10 +41,9 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	requestingAccount, err := p.federator.GetAccountByURI(
-		RequestingUsername: requestedUsername,
+		transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false,
-		RemoteAccountID:    requestingAccountURI,
+	)
-	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getstatusreplies.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
@@ -44,10 +43,9 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	requestingAccount, err := p.federator.GetAccountByURI(
-		RequestingUsername: requestedUsername,
+		transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false,
-		RemoteAccountID:    requestingAccountURI,
+	)
-	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getuser.go

@@ -25,20 +25,20 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
 func (p *processor) GetUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
-	// get the account the request is referring to
+	// Get the instance-local account the request is referring to.
 	requestedAccount, err := p.db.GetAccountByUsernameDomain(ctx, requestedUsername, "")
 	if err != nil {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
 	}
 
 	var requestedPerson vocab.ActivityStreamsPerson
+
 	if uris.IsPublicKeyPath(requestURL) {
 		// if it's a public key path, we don't need to authenticate but we'll only serve the bare minimum user profile needed for the public key
 		requestedPerson, err = p.tc.AccountToASMinimal(ctx, requestedAccount)
@@ -53,11 +53,10 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque
 		}
 
 		// if we're not already handshaking/dereferencing a remote account, dereference it now
-		if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) {
+		if !p.federator.Handshaking(requestedUsername, requestingAccountURI) {
-			requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+			requestingAccount, err := p.federator.GetAccountByURI(
-				RequestingUsername: requestedUsername,
+				transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false,
-				RemoteAccountID:    requestingAccountURI,
+			)
-			})
 			if err != nil {
 				return nil, gtserror.NewErrorUnauthorized(err)
 			}

internal/processing/fromcommon.go

@@ -76,13 +76,8 @@ func (p *processor) notifyStatus(ctx context.Context, status *gtsmodel.Status) e
 		}
 
 		// if we've reached this point we know the mention is for a local account, and the notification doesn't exist, so create it
-		notifID, err := id.NewULID()
-		if err != nil {
-			return err
-		}
-
 		notif := &gtsmodel.Notification{
-			ID:               notifID,
+			ID:               id.NewULID(),
 			NotificationType: gtsmodel.NotificationMention,
 			TargetAccountID:  m.TargetAccountID,
 			TargetAccount:    m.TargetAccount,
@@ -127,13 +122,8 @@ func (p *processor) notifyFollowRequest(ctx context.Context, followRequest *gtsm
 		return nil
 	}
 
-	notifID, err := id.NewULID()
-	if err != nil {
-		return err
-	}
-
 	notif := &gtsmodel.Notification{
-		ID:               notifID,
+		ID:               id.NewULID(),
 		NotificationType: gtsmodel.NotificationFollowRequest,
 		TargetAccountID:  followRequest.TargetAccountID,
 		OriginAccountID:  followRequest.AccountID,
@@ -172,13 +162,8 @@ func (p *processor) notifyFollow(ctx context.Context, follow *gtsmodel.Follow, t
 	}
 
 	// now create the new follow notification
-	notifID, err := id.NewULID()
-	if err != nil {
-		return err
-	}
-
 	notif := &gtsmodel.Notification{
-		ID:               notifID,
+		ID:               id.NewULID(),
 		NotificationType: gtsmodel.NotificationFollow,
 		TargetAccountID:  follow.TargetAccountID,
 		TargetAccount:    follow.TargetAccount,
@@ -222,13 +207,8 @@ func (p *processor) notifyFave(ctx context.Context, fave *gtsmodel.StatusFave) e
 		return nil
 	}
 
-	notifID, err := id.NewULID()
-	if err != nil {
-		return err
-	}
-
 	notif := &gtsmodel.Notification{
-		ID:               notifID,
+		ID:               id.NewULID(),
 		NotificationType: gtsmodel.NotificationFave,
 		TargetAccountID:  fave.TargetAccountID,
 		TargetAccount:    fave.TargetAccount,
@@ -301,13 +281,8 @@ func (p *processor) notifyAnnounce(ctx context.Context, status *gtsmodel.Status)
 	}
 
 	// now create the new reblog notification
-	notifID, err := id.NewULID()
-	if err != nil {
-		return err
-	}
-
 	notif := &gtsmodel.Notification{
-		ID:               notifID,
+		ID:               id.NewULID(),
 		NotificationType: gtsmodel.NotificationReblog,
 		TargetAccountID:  status.BoostOfAccountID,
 		TargetAccount:    status.BoostOfAccount,

internal/processing/fromfederator.go

@@ -27,7 +27,6 @@ import (
 	"codeberg.org/gruf/go-kv"
 	"codeberg.org/gruf/go-logger/v2/level"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -154,11 +153,11 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa
 			return err
 		}
 
-		a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{
+		a, err := p.federator.GetAccountByURI(ctx,
-			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			federatorMsg.ReceivingAccount.Username,
-			RemoteAccountID:    remoteAccountID,
+			remoteAccountID,
-			Blocking:           true,
+			true,
-		})
+		)
 		if err != nil {
 			return err
 		}
@@ -200,11 +199,11 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato
 			return err
 		}
 
-		a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{
+		a, err := p.federator.GetAccountByURI(ctx,
-			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			federatorMsg.ReceivingAccount.Username,
-			RemoteAccountID:    remoteAccountID,
+			remoteAccountID,
-			Blocking:           true,
+			true,
-		})
+		)
 		if err != nil {
 			return err
 		}
@@ -242,11 +241,11 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
 			return err
 		}
 
-		a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{
+		a, err := p.federator.GetAccountByURI(ctx,
-			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			federatorMsg.ReceivingAccount.Username,
-			RemoteAccountID:    remoteAccountID,
+			remoteAccountID,
-			Blocking:           true,
+			true,
-		})
+		)
 		if err != nil {
 			return err
 		}
@@ -303,11 +302,11 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede
 			return err
 		}
 
-		a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{
+		a, err := p.federator.GetAccountByURI(ctx,
-			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			federatorMsg.ReceivingAccount.Username,
-			RemoteAccountID:    remoteAccountID,
+			remoteAccountID,
-			Blocking:           true,
+			true,
-		})
+		)
 		if err != nil {
 			return err
 		}
@@ -380,14 +379,11 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder
 	}
 
 	// further database updates occur inside getremoteaccount
-	if _, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{
+	if _, err := p.federator.GetAccountByURI(ctx,
-		RequestingUsername:    federatorMsg.ReceivingAccount.Username,
+		federatorMsg.ReceivingAccount.Username,
-		RemoteAccountID:       incomingAccountURL,
+		incomingAccountURL,
-		RemoteAccountHost:     incomingAccount.Domain,
+		true,
-		RemoteAccountUsername: incomingAccount.Username,
+	); err != nil {
-		PartialAccount:        incomingAccount,
-		Blocking:              true,
-	}); err != nil {
 		return fmt.Errorf("error enriching updated account from federator: %s", err)
 	}
 

internal/processing/report/create.go

@@ -64,11 +64,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form
 		}
 	}
 
-	reportID, err := id.NewULID()
+	reportID := id.NewULID()
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
 	report := &gtsmodel.Report{
 		ID:              reportID,
 		URI:             uris.GenerateURIForReport(reportID),

internal/processing/search.go

@@ -27,6 +27,8 @@ import (
 
 	"codeberg.org/gruf/go-kv"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -85,7 +87,7 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *a
 
 	if username, domain, err := util.ExtractNamestringParts(maybeNamestring); err == nil {
 		l.Trace("search term is a mention, looking it up...")
-		foundAccount, err := p.searchAccountByMention(ctx, authed, username, domain, search.Resolve)
+		foundAccount, err := p.searchAccountByUsernameDomain(ctx, authed, username, domain, search.Resolve)
 		if err != nil {
 			var errNotRetrievable *dereferencing.ErrNotRetrievable
 			if !errors.As(err, &errNotRetrievable) {
@@ -210,27 +212,70 @@ func (p *processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, u
 
 	if !*status.Local && statusable != nil {
 		// Attempt to dereference the status thread while we are here
-		p.federator.DereferenceRemoteThread(transport.WithFastfail(ctx), authed.Account.Username, uri, status, statusable)
+		p.federator.DereferenceThread(transport.WithFastfail(ctx), authed.Account.Username, uri, status, statusable)
 	}
 
 	return status, nil
 }
 
 func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Account, error) {
-	return p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	if !resolve {
-		RequestingUsername: authed.Account.Username,
+		var (
-		RemoteAccountID:    uri,
+			account *gtsmodel.Account
-		Blocking:           true,
+			err     error
-		SkipResolve:        !resolve,
+			uriStr  = uri.String()
-	})
+		)
+
+		// Search the database for existing account with ID URI.
+		account, err = p.db.GetAccountByURI(ctx, uriStr)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return nil, fmt.Errorf("searchAccountByURI: error checking database for account %s: %w", uriStr, err)
+		}
+
+		if account == nil {
+			// Else, search the database for existing by ID URL.
+			account, err = p.db.GetAccountByURL(ctx, uriStr)
+			if err != nil {
+				if !errors.Is(err, db.ErrNoEntries) {
+					return nil, fmt.Errorf("searchAccountByURI: error checking database for account %s: %w", uriStr, err)
+				}
+				return nil, dereferencing.NewErrNotRetrievable(err)
+			}
+		}
+
+		return account, nil
+	}
+
+	return p.federator.GetAccountByURI(
+		transport.WithFastfail(ctx),
+		authed.Account.Username,
+		uri, false,
+	)
 }
 
-func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) {
+func (p *processor) searchAccountByUsernameDomain(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) {
-	return p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+	if !resolve {
-		RequestingUsername:    authed.Account.Username,
+		if domain == config.GetHost() || domain == config.GetAccountDomain() {
-		RemoteAccountUsername: username,
+			// We do local lookups using an empty domain,
-		RemoteAccountHost:     domain,
+			// else it will fail the db search below.
-		Blocking:              true,
+			domain = ""
-		SkipResolve:           !resolve,
+		}
-	})
+
+		// Search the database for existing account with USERNAME@DOMAIN
+		account, err := p.db.GetAccountByUsernameDomain(ctx, username, domain)
+		if err != nil {
+			if !errors.Is(err, db.ErrNoEntries) {
+				return nil, fmt.Errorf("searchAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
+			}
+			return nil, dereferencing.NewErrNotRetrievable(err)
+		}
+
+		return account, nil
+	}
+
+	return p.federator.GetAccountByUsernameDomain(
+		transport.WithFastfail(ctx),
+		authed.Account.Username,
+		username, domain, false,
+	)
 }

internal/processing/status/bookmark.go

@@ -55,14 +55,9 @@ func (p *processor) Bookmark(ctx context.Context, requestingAccount *gtsmodel.Ac
 	}
 
 	if newBookmark {
-		thisBookmarkID, err := id.NewULID()
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-
 		// we need to create a new bookmark in the database
 		gtsBookmark := &gtsmodel.StatusBookmark{
-			ID:              thisBookmarkID,
+			ID:              id.NewULID(),
 			AccountID:       requestingAccount.ID,
 			Account:         requestingAccount,
 			TargetAccountID: targetStatus.AccountID,

internal/processing/status/create.go

@@ -35,11 +35,7 @@ import (
 
 func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, application *gtsmodel.Application, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, gtserror.WithCode) {
 	accountURIs := uris.GenerateURIsForAccount(account.Username)
-	thisStatusID, err := id.NewULID()
+	thisStatusID := id.NewULID()
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
 	local := true
 	sensitive := form.Sensitive
 

internal/processing/status/fave.go

@@ -62,10 +62,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun
 	}
 
 	if newFave {
-		thisFaveID, err := id.NewULID()
+		thisFaveID := id.NewULID()
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
-		}
 
 		// we need to create a new fave in the database
 		gtsFave := &gtsmodel.StatusFave{

internal/processing/util.go

@@ -25,7 +25,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
-	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
@@ -54,21 +53,24 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.
 			mentionedAccount = localAccount
 		} else {
 			var requestingUsername string
+
 			if originAccount.Domain == "" {
 				requestingUsername = originAccount.Username
 			}
-			remoteAccount, err := federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
+
-				RequestingUsername:    requestingUsername,
+			remoteAccount, err := federator.GetAccountByUsernameDomain(
-				RemoteAccountUsername: username,
+				transport.WithFastfail(ctx),
-				RemoteAccountHost:     domain,
+				requestingUsername,
-			})
+				username,
+				domain,
+				false,
+			)
 			if err != nil {
-				return nil, fmt.Errorf("error dereferencing account: %s", err)
+				return nil, fmt.Errorf("parseMentionFunc: error fetching account: %s", err)
 			}
 
 			// we were able to resolve it!
 			mentionedAccount = remoteAccount
-
 		}
 
 		mentionID, err := id.NewRandomULID()

internal/trans/model/account.go

@@ -46,7 +46,6 @@ type Account struct {
 	StatusFormat          string          `json:"statusFormat,omitempty" bun:",nullzero"`
 	URI                   string          `json:"uri" bun:",nullzero"`
 	URL                   string          `json:"url" bun:",nullzero"`
-	LastWebfingeredAt     *time.Time      `json:"lastWebfingeredAt,omitempty" bun:",nullzero"`
 	InboxURI              string          `json:"inboxURI" bun:",nullzero"`
 	OutboxURI             string          `json:"outboxURI" bun:",nullzero"`
 	FollowingURI          string          `json:"followingUri" bun:",nullzero"`

internal/typeutils/astointernal.go

@@ -33,7 +33,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
-func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) {
+func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string) (*gtsmodel.Account, error) {
 	// first check if we actually already know this account
 	uriProp := accountable.GetJSONLDId()
 	if uriProp == nil || !uriProp.IsIRI() {
@@ -41,18 +41,6 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a
 	}
 	uri := uriProp.GetIRI()
 
-	if !update {
-		acct, err := c.db.GetAccountByURI(ctx, uri.String())
-		if err == nil {
-			// we already know this account so we can skip generating it
-			return acct, nil
-		}
-		if err != db.ErrNoEntries {
-			// we don't know the account and there's been a real error
-			return nil, fmt.Errorf("error getting account with uri %s from the database: %s", uri.String(), err)
-		}
-	}
-
 	// we don't know the account, or we're being told to update it, so we need to generate it from the person -- at least we already have the URI!
 	acct := &gtsmodel.Account{}
 	acct.URI = uri.String()
@@ -169,16 +157,12 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a
 		acct.InboxURI = accountable.GetActivityStreamsInbox().GetIRI().String()
 	}
 
-	// SharedInboxURI
+	// SharedInboxURI:
-	if sharedInboxURI := ap.ExtractSharedInbox(accountable); sharedInboxURI != nil {
+	// only trust shared inbox if it has at least two domains,
-		var sharedInbox string
+	// from the right, in common with the domain of the account
-
+	if sharedInboxURI := ap.ExtractSharedInbox(accountable); // nocollapse
-		// only trust shared inbox if it has at least two domains,
+	sharedInboxURI != nil && dns.CompareDomainName(acct.Domain, sharedInboxURI.Host) >= 2 {
-		// from the right, in common with the domain of the account
+		sharedInbox := sharedInboxURI.String()
-		if dns.CompareDomainName(acct.Domain, sharedInboxURI.Host) >= 2 {
-			sharedInbox = sharedInboxURI.String()
-		}
-
 		acct.SharedInboxURI = &sharedInbox
 	}
 

internal/typeutils/astointernal_test.go

@@ -52,7 +52,7 @@ func (suite *ASToInternalTestSuite) jsonToType(in string) vocab.Type {
 func (suite *ASToInternalTestSuite) TestParsePerson() {
 	testPerson := suite.testPeople["https://unknown-instance.com/users/brand_new_person"]
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "")
 	suite.NoError(err)
 
 	suite.Equal("https://unknown-instance.com/users/brand_new_person", acct.URI)
@@ -74,7 +74,7 @@ func (suite *ASToInternalTestSuite) TestParsePerson() {
 func (suite *ASToInternalTestSuite) TestParsePersonWithSharedInbox() {
 	testPerson := suite.testPeople["https://turnip.farm/users/turniplover6969"]
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "")
 	suite.NoError(err)
 
 	suite.Equal("https://turnip.farm/users/turniplover6969", acct.URI)
@@ -131,7 +131,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
 		suite.FailNow("type not coercible")
 	}
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "")
 	suite.NoError(err)
 	suite.Equal("https://mastodon.social/inbox", *acct.SharedInboxURI)
 }
@@ -183,7 +183,7 @@ func (suite *ASToInternalTestSuite) TestParseOwncastService() {
 		suite.FailNow("type not coercible")
 	}
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "")
 	suite.NoError(err)
 
 	suite.Equal("rgh", acct.Username)

internal/typeutils/converter.go

@@ -111,15 +111,10 @@ type TypeConverter interface {
 		ACTIVITYSTREAMS MODEL TO INTERNAL (gts) MODEL
 	*/
 
-	// ASPersonToAccount converts a remote account/person/application representation into a gts model account.
+	// ASRepresentationToAccount converts a remote account/person/application representation into a gts model account.
 	//
-	// If update is false, and the account is already known in the database, then the existing account entry will be returned.
+	// If accountDomain is provided then this value will be used as the account's Domain, else the AP ID host.
-	// If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account
+	ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string) (*gtsmodel.Account, error)
-	// will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile.
-	//
-	// If accountDomain is set (not an empty string) then this value will be used as the account's Domain. If not set,
-	// then the Host of the accountable's AP ID will be used instead.
-	ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error)
 	// ASStatus converts a remote activitystreams 'status' representation into a gts model status.
 	ASStatusToStatus(ctx context.Context, statusable ap.Statusable) (*gtsmodel.Status, error)
 	// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request.

internal/typeutils/internal.go

@@ -27,11 +27,7 @@ func (c *converter) FollowRequestToFollow(ctx context.Context, f *gtsmodel.Follo
 func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boostingAccount *gtsmodel.Account) (*gtsmodel.Status, error) {
 	// the wrapper won't use the same ID as the boosted status so we generate some new UUIDs
 	accountURIs := uris.GenerateURIsForAccount(boostingAccount.Username)
-	boostWrapperStatusID, err := id.NewULID()
+	boostWrapperStatusID := id.NewULID()
-	if err != nil {
-		return nil, err
-	}
-
 	boostWrapperStatusURI := accountURIs.StatusesURI + "/" + boostWrapperStatusID
 	boostWrapperStatusURL := accountURIs.StatusesURL + "/" + boostWrapperStatusID
 

internal/validate/account_test.go

@@ -66,7 +66,7 @@ func happyAccount() *gtsmodel.Account {
 		StatusFormat:            "plain",
 		URI:                     "http://localhost:8080/users/the_mighty_zork",
 		URL:                     "http://localhost:8080/@the_mighty_zork",
-		LastWebfingeredAt:       time.Time{},
+		FetchedAt:               time.Time{},
 		InboxURI:                "http://localhost:8080/users/the_mighty_zork/inbox",
 		OutboxURI:               "http://localhost:8080/users/the_mighty_zork/outbox",
 		FollowersURI:            "http://localhost:8080/users/the_mighty_zork/followers",
@@ -117,14 +117,14 @@ func (suite *AccountValidateTestSuite) TestValidateAccountNoCreatedAt() {
 	suite.NoError(err)
 }
 
-// LastWebfingeredAt must be defined if remote account
+// FetchedAt must be defined if remote account
 func (suite *AccountValidateTestSuite) TestValidateAccountNoWebfingeredAt() {
 	a := happyAccount()
 
 	a.Domain = "example.org"
-	a.LastWebfingeredAt = time.Time{}
+	a.FetchedAt = time.Time{}
 	err := validate.Struct(*a)
-	suite.EqualError(err, "Key: 'Account.LastWebfingeredAt' Error:Field validation for 'LastWebfingeredAt' failed on the 'required_with' tag")
+	suite.EqualError(err, "Key: 'Account.FetchedAt' Error:Field validation for 'FetchedAt' failed on the 'required_with' tag")
 }
 
 // Username must be set
@@ -139,7 +139,7 @@ func (suite *AccountValidateTestSuite) TestValidateAccountUsername() {
 // Domain must be either empty (for local accounts) or proper fqdn (for remote accounts)
 func (suite *AccountValidateTestSuite) TestValidateAccountDomain() {
 	a := happyAccount()
-	a.LastWebfingeredAt = time.Now()
+	a.FetchedAt = time.Now()
 
 	a.Domain = ""
 	err := validate.Struct(*a)
@@ -204,7 +204,7 @@ func (suite *AccountValidateTestSuite) TestValidateAttachmentRemoteURLs() {
 // Default privacy must be set if account is local
 func (suite *AccountValidateTestSuite) TestValidatePrivacy() {
 	a := happyAccount()
-	a.LastWebfingeredAt = time.Now()
+	a.FetchedAt = time.Now()
 
 	a.Privacy = ""
 	err := validate.Struct(*a)
@@ -261,7 +261,7 @@ func (suite *AccountValidateTestSuite) TestValidateAccountURI() {
 // ActivityPub URIs must be set on account if it's local
 func (suite *AccountValidateTestSuite) TestValidateAccountURIs() {
 	a := happyAccount()
-	a.LastWebfingeredAt = time.Now()
+	a.FetchedAt = time.Now()
 
 	a.InboxURI = "invalid-uri"
 	a.OutboxURI = "invalid-uri"
@@ -319,7 +319,7 @@ func (suite *AccountValidateTestSuite) TestValidateActorType() {
 // Private key must be set on local accounts
 func (suite *AccountValidateTestSuite) TestValidatePrivateKey() {
 	a := happyAccount()
-	a.LastWebfingeredAt = time.Now()
+	a.FetchedAt = time.Now()
 
 	a.PrivateKey = nil
 	err := validate.Struct(*a)

testrig/config.go

@@ -34,7 +34,7 @@ func InitTestConfig() {
 }
 
 var testDefaults = config.Configuration{
-	LogLevel:        "trace",
+	LogLevel:        "info",
 	LogDbQueries:    true,
 	ApplicationName: "gotosocial",
 	LandingPageUser: "",

testrig/testmodels.go

@@ -335,7 +335,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			URI:                     "http://localhost:8080/users/localhost:8080",
 			URL:                     "http://localhost:8080/@localhost:8080",
 			PublicKeyURI:            "http://localhost:8080/users/localhost:8080#main-key",
-			LastWebfingeredAt:       time.Time{},
+			FetchedAt:               time.Time{},
 			InboxURI:                "http://localhost:8080/users/localhost:8080/inbox",
 			OutboxURI:               "http://localhost:8080/users/localhost:8080/outbox",
 			FollowersURI:            "http://localhost:8080/users/localhost:8080/followers",
@@ -373,7 +373,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:                "en",
 			URI:                     "http://localhost:8080/users/weed_lord420",
 			URL:                     "http://localhost:8080/@weed_lord420",
-			LastWebfingeredAt:       time.Time{},
+			FetchedAt:               time.Time{},
 			InboxURI:                "http://localhost:8080/users/weed_lord420/inbox",
 			OutboxURI:               "http://localhost:8080/users/weed_lord420/outbox",
 			FollowersURI:            "http://localhost:8080/users/weed_lord420/followers",
@@ -413,7 +413,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			URI:                     "http://localhost:8080/users/admin",
 			URL:                     "http://localhost:8080/@admin",
 			PublicKeyURI:            "http://localhost:8080/users/admin#main-key",
-			LastWebfingeredAt:       time.Time{},
+			FetchedAt:               time.Time{},
 			InboxURI:                "http://localhost:8080/users/admin/inbox",
 			OutboxURI:               "http://localhost:8080/users/admin/outbox",
 			FollowersURI:            "http://localhost:8080/users/admin/followers",
@@ -452,7 +452,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:                "en",
 			URI:                     "http://localhost:8080/users/the_mighty_zork",
 			URL:                     "http://localhost:8080/@the_mighty_zork",
-			LastWebfingeredAt:       time.Time{},
+			FetchedAt:               time.Time{},
 			InboxURI:                "http://localhost:8080/users/the_mighty_zork/inbox",
 			OutboxURI:               "http://localhost:8080/users/the_mighty_zork/outbox",
 			FollowersURI:            "http://localhost:8080/users/the_mighty_zork/followers",
@@ -492,7 +492,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:                "en",
 			URI:                     "http://localhost:8080/users/1happyturtle",
 			URL:                     "http://localhost:8080/@1happyturtle",
-			LastWebfingeredAt:       time.Time{},
+			FetchedAt:               time.Time{},
 			InboxURI:                "http://localhost:8080/users/1happyturtle/inbox",
 			OutboxURI:               "http://localhost:8080/users/1happyturtle/outbox",
 			FollowersURI:            "http://localhost:8080/users/1happyturtle/followers",
@@ -527,7 +527,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:              "en",
 			URI:                   "http://fossbros-anonymous.io/users/foss_satan",
 			URL:                   "http://fossbros-anonymous.io/@foss_satan",
-			LastWebfingeredAt:     time.Time{},
+			FetchedAt:             time.Time{},
 			InboxURI:              "http://fossbros-anonymous.io/users/foss_satan/inbox",
 			SharedInboxURI:        StringPtr("http://fossbros-anonymous.io/inbox"),
 			OutboxURI:             "http://fossbros-anonymous.io/users/foss_satan/outbox",
@@ -563,7 +563,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:              "en",
 			URI:                   "http://example.org/users/Some_User",
 			URL:                   "http://example.org/@Some_User",
-			LastWebfingeredAt:     time.Time{},
+			FetchedAt:             time.Time{},
 			InboxURI:              "http://example.org/users/Some_User/inbox",
 			SharedInboxURI:        StringPtr(""),
 			OutboxURI:             "http://example.org/users/Some_User/outbox",
@@ -599,7 +599,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:                "en",
 			URI:                     "http://thequeenisstillalive.technology/users/her_fuckin_maj",
 			URL:                     "http://thequeenisstillalive.technology/@her_fuckin_maj",
-			LastWebfingeredAt:       time.Time{},
+			FetchedAt:               time.Time{},
 			InboxURI:                "http://thequeenisstillalive.technology/users/her_fuckin_maj/inbox",
 			SharedInboxURI:          StringPtr(""),
 			OutboxURI:               "http://thequeenisstillalive.technology/users/her_fuckin_maj/outbox",
@@ -2137,6 +2137,12 @@ func NewTestFediPeople() map[string]vocab.ActivityStreamsPerson {
 	}
 	turnipLover6969Pub := &turnipLover6969Priv.PublicKey
 
+	someUserPriv, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(err)
+	}
+	someUserPub := &someUserPriv.PublicKey
+
 	return map[string]vocab.ActivityStreamsPerson{
 		"https://unknown-instance.com/users/brand_new_person": newAPPerson(
 			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
@@ -2180,6 +2186,27 @@ func NewTestFediPeople() map[string]vocab.ActivityStreamsPerson {
 			"image/png",
 			false,
 		),
+		"https://example.org/users/Some_User": newAPPerson(
+			URLMustParse("https://example.org/users/Some_User"),
+			URLMustParse("https://example.org/users/Some_User/following"),
+			URLMustParse("https://example.org/users/Some_User/followers"),
+			URLMustParse("https://example.org/users/Some_User/inbox"),
+			URLMustParse("https://example.org/sharedInbox"),
+			URLMustParse("https://example.org/users/Some_User/outbox"),
+			URLMustParse("https://example.org/users/Some_User/collections/featured"),
+			"Some_User",
+			"just some user, don't mind me",
+			"Peepee poo poo",
+			URLMustParse("https://example.org/@Some_User"),
+			true,
+			URLMustParse("https://example.org/users/Some_User#main-key"),
+			someUserPub,
+			nil,
+			"image/jpeg",
+			nil,
+			"image/png",
+			false,
+		),
 	}
 }
 

testrig/transportcontroller.go

@@ -60,13 +60,13 @@ func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *conc
 type MockHTTPClient struct {
 	do func(req *http.Request) (*http.Response, error)
 
-	testRemoteStatuses    map[string]vocab.ActivityStreamsNote
+	TestRemoteStatuses    map[string]vocab.ActivityStreamsNote
-	testRemotePeople      map[string]vocab.ActivityStreamsPerson
+	TestRemotePeople      map[string]vocab.ActivityStreamsPerson
-	testRemoteGroups      map[string]vocab.ActivityStreamsGroup
+	TestRemoteGroups      map[string]vocab.ActivityStreamsGroup
-	testRemoteServices    map[string]vocab.ActivityStreamsService
+	TestRemoteServices    map[string]vocab.ActivityStreamsService
-	testRemoteAttachments map[string]RemoteAttachmentFile
+	TestRemoteAttachments map[string]RemoteAttachmentFile
-	testRemoteEmojis      map[string]vocab.TootEmoji
+	TestRemoteEmojis      map[string]vocab.TootEmoji
-	testTombstones        map[string]*gtsmodel.Tombstone
+	TestTombstones        map[string]*gtsmodel.Tombstone
 
 	SentMessages sync.Map
 }
@@ -88,13 +88,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 		return mockHTTPClient
 	}
 
-	mockHTTPClient.testRemoteStatuses = NewTestFediStatuses()
+	mockHTTPClient.TestRemoteStatuses = NewTestFediStatuses()
-	mockHTTPClient.testRemotePeople = NewTestFediPeople()
+	mockHTTPClient.TestRemotePeople = NewTestFediPeople()
-	mockHTTPClient.testRemoteGroups = NewTestFediGroups()
+	mockHTTPClient.TestRemoteGroups = NewTestFediGroups()
-	mockHTTPClient.testRemoteServices = NewTestFediServices()
+	mockHTTPClient.TestRemoteServices = NewTestFediServices()
-	mockHTTPClient.testRemoteAttachments = NewTestFediAttachments(relativeMediaPath)
+	mockHTTPClient.TestRemoteAttachments = NewTestFediAttachments(relativeMediaPath)
-	mockHTTPClient.testRemoteEmojis = NewTestFediEmojis()
+	mockHTTPClient.TestRemoteEmojis = NewTestFediEmojis()
-	mockHTTPClient.testTombstones = NewTestTombstones()
+	mockHTTPClient.TestTombstones = NewTestTombstones()
 
 	mockHTTPClient.do = func(req *http.Request) (*http.Response, error) {
 		responseCode := http.StatusNotFound
@@ -123,7 +123,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseContentLength = len(responseBytes)
 		} else if strings.Contains(req.URL.String(), ".well-known/webfinger") {
 			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
-		} else if note, ok := mockHTTPClient.testRemoteStatuses[req.URL.String()]; ok {
+		} else if note, ok := mockHTTPClient.TestRemoteStatuses[req.URL.String()]; ok {
 			// the request is for a note that we have stored
 			noteI, err := streams.Serialize(note)
 			if err != nil {
@@ -137,7 +137,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes = noteJSON
 			responseContentType = applicationActivityJSON
 			responseContentLength = len(noteJSON)
-		} else if person, ok := mockHTTPClient.testRemotePeople[req.URL.String()]; ok {
+		} else if person, ok := mockHTTPClient.TestRemotePeople[req.URL.String()]; ok {
 			// the request is for a person that we have stored
 			personI, err := streams.Serialize(person)
 			if err != nil {
@@ -151,7 +151,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes = personJSON
 			responseContentType = applicationActivityJSON
 			responseContentLength = len(personJSON)
-		} else if group, ok := mockHTTPClient.testRemoteGroups[req.URL.String()]; ok {
+		} else if group, ok := mockHTTPClient.TestRemoteGroups[req.URL.String()]; ok {
 			// the request is for a person that we have stored
 			groupI, err := streams.Serialize(group)
 			if err != nil {
@@ -165,7 +165,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes = groupJSON
 			responseContentType = applicationActivityJSON
 			responseContentLength = len(groupJSON)
-		} else if service, ok := mockHTTPClient.testRemoteServices[req.URL.String()]; ok {
+		} else if service, ok := mockHTTPClient.TestRemoteServices[req.URL.String()]; ok {
 			serviceI, err := streams.Serialize(service)
 			if err != nil {
 				panic(err)
@@ -178,7 +178,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes = serviceJSON
 			responseContentType = applicationActivityJSON
 			responseContentLength = len(serviceJSON)
-		} else if emoji, ok := mockHTTPClient.testRemoteEmojis[req.URL.String()]; ok {
+		} else if emoji, ok := mockHTTPClient.TestRemoteEmojis[req.URL.String()]; ok {
 			emojiI, err := streams.Serialize(emoji)
 			if err != nil {
 				panic(err)
@@ -191,12 +191,12 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes = emojiJSON
 			responseContentType = applicationActivityJSON
 			responseContentLength = len(emojiJSON)
-		} else if attachment, ok := mockHTTPClient.testRemoteAttachments[req.URL.String()]; ok {
+		} else if attachment, ok := mockHTTPClient.TestRemoteAttachments[req.URL.String()]; ok {
 			responseCode = http.StatusOK
 			responseBytes = attachment.Data
 			responseContentType = attachment.ContentType
 			responseContentLength = len(attachment.Data)
-		} else if _, ok := mockHTTPClient.testTombstones[req.URL.String()]; ok {
+		} else if _, ok := mockHTTPClient.TestTombstones[req.URL.String()]; ok {
 			responseCode = http.StatusGone
 			responseBytes = []byte{}
 			responseContentType = "text/html"
@@ -278,7 +278,7 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt
 				{
 					Rel:  "self",
 					Type: applicationActivityJSON,
-					Href: "https://fossbros-anonymous.io/users/foss_satan",
+					Href: "http://fossbros-anonymous.io/users/foss_satan",
 				},
 			},
 		}