Move a lot of stuff + tidy stuff (#37)

Lots of renaming and moving stuff, some bug fixes, more lenient parsing of notifications and home timeline.

internal/processing/account.go

@@ -0,0 +1,553 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/google/uuid"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// accountCreate does the dirty work of making an account and user in the database.
+// It then returns a token to the caller, for use with the new account, as per the
+// spec here: https://docs.joinmastodon.org/methods/accounts/
+func (p *processor) AccountCreate(authed *oauth.Auth, form *apimodel.AccountCreateRequest) (*apimodel.Token, error) {
+	l := p.log.WithField("func", "accountCreate")
+
+	if err := p.db.IsEmailAvailable(form.Email); err != nil {
+		return nil, err
+	}
+
+	if err := p.db.IsUsernameAvailable(form.Username); err != nil {
+		return nil, err
+	}
+
+	// don't store a reason if we don't require one
+	reason := form.Reason
+	if !p.config.AccountsConfig.ReasonRequired {
+		reason = ""
+	}
+
+	l.Trace("creating new username and account")
+	user, err := p.db.NewSignup(form.Username, reason, p.config.AccountsConfig.RequireApproval, form.Email, form.Password, form.IP, form.Locale, authed.Application.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error creating new signup in the database: %s", err)
+	}
+
+	l.Tracef("generating a token for user %s with account %s and application %s", user.ID, user.AccountID, authed.Application.ID)
+	accessToken, err := p.oauthServer.GenerateUserAccessToken(authed.Token, authed.Application.ClientSecret, user.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error creating new access token for user %s: %s", user.ID, err)
+	}
+
+	return &apimodel.Token{
+		AccessToken: accessToken.GetAccess(),
+		TokenType:   "Bearer",
+		Scope:       accessToken.GetScope(),
+		CreatedAt:   accessToken.GetAccessCreateAt().Unix(),
+	}, nil
+}
+
+func (p *processor) AccountGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Account, error) {
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, errors.New("account not found")
+		}
+		return nil, fmt.Errorf("db error: %s", err)
+	}
+
+	// lazily dereference things on the account if it hasn't been done yet
+	var requestingUsername string
+	if authed.Account != nil {
+		requestingUsername = authed.Account.Username
+	}
+	if err := p.dereferenceAccountFields(targetAccount, requestingUsername, false); err != nil {
+		p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err)
+	}
+
+	var mastoAccount *apimodel.Account
+	var err error
+	if authed.Account != nil && targetAccount.ID == authed.Account.ID {
+		mastoAccount, err = p.tc.AccountToMastoSensitive(targetAccount)
+	} else {
+		mastoAccount, err = p.tc.AccountToMastoPublic(targetAccount)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error converting account: %s", err)
+	}
+	return mastoAccount, nil
+}
+
+func (p *processor) AccountUpdate(authed *oauth.Auth, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error) {
+	l := p.log.WithField("func", "AccountUpdate")
+
+	if form.Discoverable != nil {
+		if err := p.db.UpdateOneByID(authed.Account.ID, "discoverable", *form.Discoverable, &gtsmodel.Account{}); err != nil {
+			return nil, fmt.Errorf("error updating discoverable: %s", err)
+		}
+	}
+
+	if form.Bot != nil {
+		if err := p.db.UpdateOneByID(authed.Account.ID, "bot", *form.Bot, &gtsmodel.Account{}); err != nil {
+			return nil, fmt.Errorf("error updating bot: %s", err)
+		}
+	}
+
+	if form.DisplayName != nil {
+		if err := util.ValidateDisplayName(*form.DisplayName); err != nil {
+			return nil, err
+		}
+		if err := p.db.UpdateOneByID(authed.Account.ID, "display_name", *form.DisplayName, &gtsmodel.Account{}); err != nil {
+			return nil, err
+		}
+	}
+
+	if form.Note != nil {
+		if err := util.ValidateNote(*form.Note); err != nil {
+			return nil, err
+		}
+		if err := p.db.UpdateOneByID(authed.Account.ID, "note", *form.Note, &gtsmodel.Account{}); err != nil {
+			return nil, err
+		}
+	}
+
+	if form.Avatar != nil && form.Avatar.Size != 0 {
+		avatarInfo, err := p.updateAccountAvatar(form.Avatar, authed.Account.ID)
+		if err != nil {
+			return nil, err
+		}
+		l.Tracef("new avatar info for account %s is %+v", authed.Account.ID, avatarInfo)
+	}
+
+	if form.Header != nil && form.Header.Size != 0 {
+		headerInfo, err := p.updateAccountHeader(form.Header, authed.Account.ID)
+		if err != nil {
+			return nil, err
+		}
+		l.Tracef("new header info for account %s is %+v", authed.Account.ID, headerInfo)
+	}
+
+	if form.Locked != nil {
+		if err := p.db.UpdateOneByID(authed.Account.ID, "locked", *form.Locked, &gtsmodel.Account{}); err != nil {
+			return nil, err
+		}
+	}
+
+	if form.Source != nil {
+		if form.Source.Language != nil {
+			if err := util.ValidateLanguage(*form.Source.Language); err != nil {
+				return nil, err
+			}
+			if err := p.db.UpdateOneByID(authed.Account.ID, "language", *form.Source.Language, &gtsmodel.Account{}); err != nil {
+				return nil, err
+			}
+		}
+
+		if form.Source.Sensitive != nil {
+			if err := p.db.UpdateOneByID(authed.Account.ID, "locked", *form.Locked, &gtsmodel.Account{}); err != nil {
+				return nil, err
+			}
+		}
+
+		if form.Source.Privacy != nil {
+			if err := util.ValidatePrivacy(*form.Source.Privacy); err != nil {
+				return nil, err
+			}
+			if err := p.db.UpdateOneByID(authed.Account.ID, "privacy", *form.Source.Privacy, &gtsmodel.Account{}); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// fetch the account with all updated values set
+	updatedAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(authed.Account.ID, updatedAccount); err != nil {
+		return nil, fmt.Errorf("could not fetch updated account %s: %s", authed.Account.ID, err)
+	}
+
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsProfile,
+		APActivityType: gtsmodel.ActivityStreamsUpdate,
+		GTSModel:       updatedAccount,
+		OriginAccount:  updatedAccount,
+	}
+
+	acctSensitive, err := p.tc.AccountToMastoSensitive(updatedAccount)
+	if err != nil {
+		return nil, fmt.Errorf("could not convert account into mastosensitive account: %s", err)
+	}
+	return acctSensitive, nil
+}
+
+func (p *processor) AccountStatusesGet(authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, ErrorWithCode) {
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, NewErrorNotFound(fmt.Errorf("no entry found for account id %s", targetAccountID))
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	statuses := []gtsmodel.Status{}
+	apiStatuses := []apimodel.Status{}
+	if err := p.db.GetStatusesByTimeDescending(targetAccountID, &statuses, limit, excludeReplies, maxID, pinned, mediaOnly); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return apiStatuses, nil
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	for _, s := range statuses {
+		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(&s)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("error getting relevant statuses: %s", err))
+		}
+
+		visible, err := p.db.StatusVisible(&s, targetAccount, authed.Account, relevantAccounts)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("error checking status visibility: %s", err))
+		}
+		if !visible {
+			continue
+		}
+
+		var boostedStatus *gtsmodel.Status
+		if s.BoostOfID != "" {
+			bs := &gtsmodel.Status{}
+			if err := p.db.GetByID(s.BoostOfID, bs); err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("error getting boosted status: %s", err))
+			}
+			boostedRelevantAccounts, err := p.db.PullRelevantAccountsFromStatus(bs)
+			if err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("error getting relevant accounts from boosted status: %s", err))
+			}
+
+			boostedVisible, err := p.db.StatusVisible(bs, relevantAccounts.BoostedAccount, authed.Account, boostedRelevantAccounts)
+			if err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("error checking boosted status visibility: %s", err))
+			}
+
+			if boostedVisible {
+				boostedStatus = bs
+			}
+		}
+
+		apiStatus, err := p.tc.StatusToMasto(&s, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostedStatus)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("error converting status to masto: %s", err))
+		}
+
+		apiStatuses = append(apiStatuses, *apiStatus)
+	}
+
+	return apiStatuses, nil
+}
+
+func (p *processor) AccountFollowersGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, ErrorWithCode) {
+	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotFound(fmt.Errorf("block exists between accounts"))
+	}
+
+	followers := []gtsmodel.Follow{}
+	accounts := []apimodel.Account{}
+	if err := p.db.GetFollowersByAccountID(targetAccountID, &followers); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return accounts, nil
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	for _, f := range followers {
+		blocked, err := p.db.Blocked(authed.Account.ID, f.AccountID)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		if blocked {
+			continue
+		}
+
+		a := &gtsmodel.Account{}
+		if err := p.db.GetByID(f.AccountID, a); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				continue
+			}
+			return nil, NewErrorInternalError(err)
+		}
+
+		// derefence account fields in case we haven't done it already
+		if err := p.dereferenceAccountFields(a, authed.Account.Username, false); err != nil {
+			// don't bail if we can't fetch them, we'll try another time
+			p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err)
+		}
+
+		account, err := p.tc.AccountToMastoPublic(a)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		accounts = append(accounts, *account)
+	}
+	return accounts, nil
+}
+
+func (p *processor) AccountFollowingGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, ErrorWithCode) {
+	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotFound(fmt.Errorf("block exists between accounts"))
+	}
+
+	following := []gtsmodel.Follow{}
+	accounts := []apimodel.Account{}
+	if err := p.db.GetFollowingByAccountID(targetAccountID, &following); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return accounts, nil
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	for _, f := range following {
+		blocked, err := p.db.Blocked(authed.Account.ID, f.AccountID)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		if blocked {
+			continue
+		}
+
+		a := &gtsmodel.Account{}
+		if err := p.db.GetByID(f.TargetAccountID, a); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				continue
+			}
+			return nil, NewErrorInternalError(err)
+		}
+
+		// derefence account fields in case we haven't done it already
+		if err := p.dereferenceAccountFields(a, authed.Account.Username, false); err != nil {
+			// don't bail if we can't fetch them, we'll try another time
+			p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err)
+		}
+
+		account, err := p.tc.AccountToMastoPublic(a)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		accounts = append(accounts, *account)
+	}
+	return accounts, nil
+}
+
+func (p *processor) AccountRelationshipGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, ErrorWithCode) {
+	if authed == nil || authed.Account == nil {
+		return nil, NewErrorForbidden(errors.New("not authed"))
+	}
+
+	gtsR, err := p.db.GetRelationship(authed.Account.ID, targetAccountID)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error getting relationship: %s", err))
+	}
+
+	r, err := p.tc.RelationshipToMasto(gtsR)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error converting relationship: %s", err))
+	}
+
+	return r, nil
+}
+
+func (p *processor) AccountFollowCreate(authed *oauth.Auth, form *apimodel.AccountFollowRequest) (*apimodel.Relationship, ErrorWithCode) {
+	// if there's a block between the accounts we shouldn't create the request ofc
+	blocked, err := p.db.Blocked(authed.Account.ID, form.TargetAccountID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+	if blocked {
+		return nil, NewErrorNotFound(fmt.Errorf("accountfollowcreate: block exists between accounts"))
+	}
+
+	// make sure the target account actually exists in our db
+	targetAcct := &gtsmodel.Account{}
+	if err := p.db.GetByID(form.TargetAccountID, targetAcct); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, NewErrorNotFound(fmt.Errorf("accountfollowcreate: account %s not found in the db: %s", form.TargetAccountID, err))
+		}
+	}
+
+	// check if a follow exists already
+	follows, err := p.db.Follows(authed.Account, targetAcct)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("accountfollowcreate: error checking follow in db: %s", err))
+	}
+	if follows {
+		// already follows so just return the relationship
+		return p.AccountRelationshipGet(authed, form.TargetAccountID)
+	}
+
+	// check if a follow exists already
+	followRequested, err := p.db.FollowRequested(authed.Account, targetAcct)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("accountfollowcreate: error checking follow request in db: %s", err))
+	}
+	if followRequested {
+		// already follow requested so just return the relationship
+		return p.AccountRelationshipGet(authed, form.TargetAccountID)
+	}
+
+	// make the follow request
+
+	newFollowID := uuid.NewString()
+
+	fr := &gtsmodel.FollowRequest{
+		ID:              newFollowID,
+		AccountID:       authed.Account.ID,
+		TargetAccountID: form.TargetAccountID,
+		ShowReblogs:     true,
+		URI:             util.GenerateURIForFollow(authed.Account.Username, p.config.Protocol, p.config.Host, newFollowID),
+		Notify:          false,
+	}
+	if form.Reblogs != nil {
+		fr.ShowReblogs = *form.Reblogs
+	}
+	if form.Notify != nil {
+		fr.Notify = *form.Notify
+	}
+
+	// whack it in the database
+	if err := p.db.Put(fr); err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("accountfollowcreate: error creating follow request in db: %s", err))
+	}
+
+	// if it's a local account that's not locked we can just straight up accept the follow request
+	if !targetAcct.Locked && targetAcct.Domain == "" {
+		if _, err := p.db.AcceptFollowRequest(authed.Account.ID, form.TargetAccountID); err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("accountfollowcreate: error accepting folow request for local unlocked account: %s", err))
+		}
+		// return the new relationship
+		return p.AccountRelationshipGet(authed, form.TargetAccountID)
+	}
+
+	// otherwise we leave the follow request as it is and we handle the rest of the process asynchronously
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsFollow,
+		APActivityType: gtsmodel.ActivityStreamsCreate,
+		GTSModel:       fr,
+		OriginAccount:  authed.Account,
+		TargetAccount:  targetAcct,
+	}
+
+	// return whatever relationship results from this
+	return p.AccountRelationshipGet(authed, form.TargetAccountID)
+}
+
+func (p *processor) AccountFollowRemove(authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, ErrorWithCode) {
+	// if there's a block between the accounts we shouldn't do anything
+	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+	if blocked {
+		return nil, NewErrorNotFound(fmt.Errorf("AccountFollowRemove: block exists between accounts"))
+	}
+
+	// make sure the target account actually exists in our db
+	targetAcct := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAcct); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, NewErrorNotFound(fmt.Errorf("AccountFollowRemove: account %s not found in the db: %s", targetAccountID, err))
+		}
+	}
+
+	// check if a follow request exists, and remove it if it does (storing the URI for later)
+	var frChanged bool
+	var frURI string
+	fr := &gtsmodel.FollowRequest{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "account_id", Value: authed.Account.ID},
+		{Key: "target_account_id", Value: targetAccountID},
+	}, fr); err == nil {
+		frURI = fr.URI
+		if err := p.db.DeleteByID(fr.ID, fr); err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("AccountFollowRemove: error removing follow request from db: %s", err))
+		}
+		frChanged = true
+	}
+
+	// now do the same thing for any existing follow
+	var fChanged bool
+	var fURI string
+	f := &gtsmodel.Follow{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "account_id", Value: authed.Account.ID},
+		{Key: "target_account_id", Value: targetAccountID},
+	}, f); err == nil {
+		fURI = f.URI
+		if err := p.db.DeleteByID(f.ID, f); err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("AccountFollowRemove: error removing follow from db: %s", err))
+		}
+		fChanged = true
+	}
+
+	// follow request status changed so send the UNDO activity to the channel for async processing
+	if frChanged {
+		p.fromClientAPI <- gtsmodel.FromClientAPI{
+			APObjectType:   gtsmodel.ActivityStreamsFollow,
+			APActivityType: gtsmodel.ActivityStreamsUndo,
+			GTSModel: &gtsmodel.Follow{
+				AccountID:       authed.Account.ID,
+				TargetAccountID: targetAccountID,
+				URI:             frURI,
+			},
+			OriginAccount: authed.Account,
+			TargetAccount: targetAcct,
+		}
+	}
+
+	// follow status changed so send the UNDO activity to the channel for async processing
+	if fChanged {
+		p.fromClientAPI <- gtsmodel.FromClientAPI{
+			APObjectType:   gtsmodel.ActivityStreamsFollow,
+			APActivityType: gtsmodel.ActivityStreamsUndo,
+			GTSModel: &gtsmodel.Follow{
+				AccountID:       authed.Account.ID,
+				TargetAccountID: targetAccountID,
+				URI:             fURI,
+			},
+			OriginAccount: authed.Account,
+			TargetAccount: targetAcct,
+		}
+	}
+
+	// return whatever relationship results from all this
+	return p.AccountRelationshipGet(authed, targetAccountID)
+}

internal/processing/admin.go

@@ -0,0 +1,66 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (p *processor) AdminEmojiCreate(authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) {
+	if !authed.User.Admin {
+		return nil, fmt.Errorf("user %s not an admin", authed.User.ID)
+	}
+
+	// open the emoji and extract the bytes from it
+	f, err := form.Image.Open()
+	if err != nil {
+		return nil, fmt.Errorf("error opening emoji: %s", err)
+	}
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("error reading emoji: %s", err)
+	}
+	if size == 0 {
+		return nil, errors.New("could not read provided emoji: size 0 bytes")
+	}
+
+	// allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using
+	emoji, err := p.mediaHandler.ProcessLocalEmoji(buf.Bytes(), form.Shortcode)
+	if err != nil {
+		return nil, fmt.Errorf("error reading emoji: %s", err)
+	}
+
+	mastoEmoji, err := p.tc.EmojiToMasto(emoji)
+	if err != nil {
+		return nil, fmt.Errorf("error converting emoji to mastotype: %s", err)
+	}
+
+	if err := p.db.Put(emoji); err != nil {
+		return nil, fmt.Errorf("database error while processing emoji: %s", err)
+	}
+
+	return &mastoEmoji, nil
+}

internal/processing/app.go

@@ -0,0 +1,77 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"github.com/google/uuid"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (p *processor) AppCreate(authed *oauth.Auth, form *apimodel.ApplicationCreateRequest) (*apimodel.Application, error) {
+	// set default 'read' for scopes if it's not set, this follows the default of the mastodon api https://docs.joinmastodon.org/methods/apps/
+	var scopes string
+	if form.Scopes == "" {
+		scopes = "read"
+	} else {
+		scopes = form.Scopes
+	}
+
+	// generate new IDs for this application and its associated client
+	clientID := uuid.NewString()
+	clientSecret := uuid.NewString()
+	vapidKey := uuid.NewString()
+
+	// generate the application to put in the database
+	app := &gtsmodel.Application{
+		Name:         form.ClientName,
+		Website:      form.Website,
+		RedirectURI:  form.RedirectURIs,
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		Scopes:       scopes,
+		VapidKey:     vapidKey,
+	}
+
+	// chuck it in the db
+	if err := p.db.Put(app); err != nil {
+		return nil, err
+	}
+
+	// now we need to model an oauth client from the application that the oauth library can use
+	oc := &oauth.Client{
+		ID:     clientID,
+		Secret: clientSecret,
+		Domain: form.RedirectURIs,
+		UserID: "", // This client isn't yet associated with a specific user,  it's just an app client right now
+	}
+
+	// chuck it in the db
+	if err := p.db.Put(oc); err != nil {
+		return nil, err
+	}
+
+	mastoApp, err := p.tc.AppToMastoSensitive(app)
+	if err != nil {
+		return nil, err
+	}
+
+	return mastoApp, nil
+}

internal/processing/error.go

@@ -0,0 +1,124 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+)
+
+// ErrorWithCode wraps an internal error with an http code, and a 'safe' version of
+// the error that can be served to clients without revealing internal business logic.
+//
+// A typical use of this error would be to first log the Original error, then return
+// the Safe error and the StatusCode to an API caller.
+type ErrorWithCode interface {
+	// Error returns the original internal error for debugging within the GoToSocial logs.
+	// This should *NEVER* be returned to a client as it may contain sensitive information.
+	Error() string
+	// Safe returns the API-safe version of the error for serialization towards a client.
+	// There's not much point logging this internally because it won't contain much helpful information.
+	Safe() string
+	//  Code returns the status code for serving to a client.
+	Code() int
+}
+
+type errorWithCode struct {
+	original error
+	safe     error
+	code     int
+}
+
+func (e errorWithCode) Error() string {
+	return e.original.Error()
+}
+
+func (e errorWithCode) Safe() string {
+	return e.safe.Error()
+}
+
+func (e errorWithCode) Code() int {
+	return e.code
+}
+
+// NewErrorBadRequest returns an ErrorWithCode 400 with the given original error and optional help text.
+func NewErrorBadRequest(original error, helpText ...string) ErrorWithCode {
+	safe := "bad request"
+	if helpText != nil {
+		safe = safe + ": " + strings.Join(helpText, ": ")
+	}
+	return errorWithCode{
+		original: original,
+		safe:     errors.New(safe),
+		code:     http.StatusBadRequest,
+	}
+}
+
+// NewErrorNotAuthorized returns an ErrorWithCode 401 with the given original error and optional help text.
+func NewErrorNotAuthorized(original error, helpText ...string) ErrorWithCode {
+	safe := "not authorized"
+	if helpText != nil {
+		safe = safe + ": " + strings.Join(helpText, ": ")
+	}
+	return errorWithCode{
+		original: original,
+		safe:     errors.New(safe),
+		code:     http.StatusUnauthorized,
+	}
+}
+
+// NewErrorForbidden returns an ErrorWithCode 403 with the given original error and optional help text.
+func NewErrorForbidden(original error, helpText ...string) ErrorWithCode {
+	safe := "forbidden"
+	if helpText != nil {
+		safe = safe + ": " + strings.Join(helpText, ": ")
+	}
+	return errorWithCode{
+		original: original,
+		safe:     errors.New(safe),
+		code:     http.StatusForbidden,
+	}
+}
+
+// NewErrorNotFound returns an ErrorWithCode 404 with the given original error and optional help text.
+func NewErrorNotFound(original error, helpText ...string) ErrorWithCode {
+	safe := "404 not found"
+	if helpText != nil {
+		safe = safe + ": " + strings.Join(helpText, ": ")
+	}
+	return errorWithCode{
+		original: original,
+		safe:     errors.New(safe),
+		code:     http.StatusNotFound,
+	}
+}
+
+// NewErrorInternalError returns an ErrorWithCode 500 with the given original error and optional help text.
+func NewErrorInternalError(original error, helpText ...string) ErrorWithCode {
+	safe := "internal server error"
+	if helpText != nil {
+		safe = safe + ": " + strings.Join(helpText, ": ")
+	}
+	return errorWithCode{
+		original: original,
+		safe:     errors.New(safe),
+		code:     http.StatusInternalServerError,
+	}
+}

internal/processing/federation.go

@@ -0,0 +1,282 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// authenticateAndDereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
+// username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account
+// for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database,
+// and passing it into the processor through a channel for further asynchronous processing.
+func (p *processor) authenticateAndDereferenceFediRequest(username string, r *http.Request) (*gtsmodel.Account, error) {
+
+	// first authenticate
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(username, r)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't authenticate request for username %s: %s", username, err)
+	}
+
+	// OK now we can do the dereferencing part
+	// we might already have an entry for this account so check that first
+	requestingAccount := &gtsmodel.Account{}
+
+	err = p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
+	if err == nil {
+		// we do have it yay, return it
+		return requestingAccount, nil
+	}
+
+	if _, ok := err.(db.ErrNoEntries); !ok {
+		// something has actually gone wrong so bail
+		return nil, fmt.Errorf("database error getting account with uri %s: %s", requestingAccountURI.String(), err)
+	}
+
+	// we just don't have an entry for this account yet
+	// what we do now should depend on our chosen federation method
+	// for now though, we'll just dereference it
+	// TODO: slow-fed
+	requestingPerson, err := p.federator.DereferenceRemoteAccount(username, requestingAccountURI)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't dereference %s: %s", requestingAccountURI.String(), err)
+	}
+
+	// convert it to our internal account representation
+	requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err)
+	}
+
+	// shove it in the database for later
+	if err := p.db.Put(requestingAccount); err != nil {
+		return nil, fmt.Errorf("database error inserting account with uri %s: %s", requestingAccountURI.String(), err)
+	}
+
+	// put it in our channel to queue it for async processing
+	p.FromFederator() <- gtsmodel.FromFederator{
+		APObjectType:   gtsmodel.ActivityStreamsProfile,
+		APActivityType: gtsmodel.ActivityStreamsCreate,
+		GTSModel:       requestingAccount,
+	}
+
+	return requestingAccount, nil
+}
+
+func (p *processor) GetFediUser(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// authenticate the request
+	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	if err != nil {
+		return nil, NewErrorNotAuthorized(err)
+	}
+
+	blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+	}
+
+	requestedPerson, err := p.tc.AccountToAS(requestedAccount)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	data, err := streams.Serialize(requestedPerson)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	return data, nil
+}
+
+func (p *processor) GetFediFollowers(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// authenticate the request
+	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	if err != nil {
+		return nil, NewErrorNotAuthorized(err)
+	}
+
+	blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+	}
+
+	requestedAccountURI, err := url.Parse(requestedAccount.URI)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", requestedAccount.URI, err))
+	}
+
+	requestedFollowers, err := p.federator.FederatingDB().Followers(context.Background(), requestedAccountURI)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error fetching followers for uri %s: %s", requestedAccountURI.String(), err))
+	}
+
+	data, err := streams.Serialize(requestedFollowers)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	return data, nil
+}
+
+func (p *processor) GetFediFollowing(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// authenticate the request
+	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	if err != nil {
+		return nil, NewErrorNotAuthorized(err)
+	}
+
+	blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+	}
+
+	requestedAccountURI, err := url.Parse(requestedAccount.URI)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", requestedAccount.URI, err))
+	}
+
+	requestedFollowing, err := p.federator.FederatingDB().Following(context.Background(), requestedAccountURI)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error fetching following for uri %s: %s", requestedAccountURI.String(), err))
+	}
+
+	data, err := streams.Serialize(requestedFollowing)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	return data, nil
+}
+
+func (p *processor) GetFediStatus(requestedUsername string, requestedStatusID string, request *http.Request) (interface{}, ErrorWithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// authenticate the request
+	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	if err != nil {
+		return nil, NewErrorNotAuthorized(err)
+	}
+
+	blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+	}
+
+	s := &gtsmodel.Status{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "id", Value: requestedStatusID},
+		{Key: "account_id", Value: requestedAccount.ID},
+	}, s); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting status with id %s and account id %s: %s", requestedStatusID, requestedAccount.ID, err))
+	}
+
+	asStatus, err := p.tc.StatusToAS(s)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	data, err := streams.Serialize(asStatus)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	return data, nil
+}
+
+func (p *processor) GetWebfingerAccount(requestedUsername string, request *http.Request) (*apimodel.WebfingerAccountResponse, ErrorWithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// return the webfinger representation
+	return &apimodel.WebfingerAccountResponse{
+		Subject: fmt.Sprintf("acct:%s@%s", requestedAccount.Username, p.config.Host),
+		Aliases: []string{
+			requestedAccount.URI,
+			requestedAccount.URL,
+		},
+		Links: []apimodel.WebfingerLink{
+			{
+				Rel:  "http://webfinger.net/rel/profile-page",
+				Type: "text/html",
+				Href: requestedAccount.URL,
+			},
+			{
+				Rel:  "self",
+				Type: "application/activity+json",
+				Href: requestedAccount.URI,
+			},
+		},
+	}, nil
+}
+
+func (p *processor) InboxPost(ctx context.Context, w http.ResponseWriter, r *http.Request) (bool, error) {
+	contextWithChannel := context.WithValue(ctx, util.APFromFederatorChanKey, p.fromFederator)
+	posted, err := p.federator.FederatingActor().PostInbox(contextWithChannel, w, r)
+	return posted, err
+}

internal/processing/followrequest.go

@@ -0,0 +1,90 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (p *processor) FollowRequestsGet(auth *oauth.Auth) ([]apimodel.Account, ErrorWithCode) {
+	frs := []gtsmodel.FollowRequest{}
+	if err := p.db.GetFollowRequestsForAccountID(auth.Account.ID, &frs); err != nil {
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			return nil, NewErrorInternalError(err)
+		}
+	}
+
+	accts := []apimodel.Account{}
+	for _, fr := range frs {
+		acct := &gtsmodel.Account{}
+		if err := p.db.GetByID(fr.AccountID, acct); err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		mastoAcct, err := p.tc.AccountToMastoPublic(acct)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		accts = append(accts, *mastoAcct)
+	}
+	return accts, nil
+}
+
+func (p *processor) FollowRequestAccept(auth *oauth.Auth, accountID string) (*apimodel.Relationship, ErrorWithCode) {
+	follow, err := p.db.AcceptFollowRequest(accountID, auth.Account.ID)
+	if err != nil {
+		return nil, NewErrorNotFound(err)
+	}
+
+	originAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(follow.AccountID, originAccount); err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(follow.TargetAccountID, targetAccount); err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsFollow,
+		APActivityType: gtsmodel.ActivityStreamsAccept,
+		GTSModel:       follow,
+		OriginAccount:  originAccount,
+		TargetAccount:  targetAccount,
+	}
+
+	gtsR, err := p.db.GetRelationship(auth.Account.ID, accountID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	r, err := p.tc.RelationshipToMasto(gtsR)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	return r, nil
+}
+
+func (p *processor) FollowRequestDeny(auth *oauth.Auth) ErrorWithCode {
+	return nil
+}

internal/processing/fromclientapi.go

@@ -0,0 +1,313 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) processFromClientAPI(clientMsg gtsmodel.FromClientAPI) error {
+	switch clientMsg.APActivityType {
+	case gtsmodel.ActivityStreamsCreate:
+		// CREATE
+		switch clientMsg.APObjectType {
+		case gtsmodel.ActivityStreamsNote:
+			// CREATE NOTE
+			status, ok := clientMsg.GTSModel.(*gtsmodel.Status)
+			if !ok {
+				return errors.New("note was not parseable as *gtsmodel.Status")
+			}
+
+			if err := p.notifyStatus(status); err != nil {
+				return err
+			}
+
+			if status.VisibilityAdvanced != nil && status.VisibilityAdvanced.Federated {
+				return p.federateStatus(status)
+			}
+			return nil
+		case gtsmodel.ActivityStreamsFollow:
+			// CREATE FOLLOW REQUEST
+			followRequest, ok := clientMsg.GTSModel.(*gtsmodel.FollowRequest)
+			if !ok {
+				return errors.New("followrequest was not parseable as *gtsmodel.FollowRequest")
+			}
+
+			if err := p.notifyFollowRequest(followRequest, clientMsg.TargetAccount); err != nil {
+				return err
+			}
+
+			return p.federateFollow(followRequest, clientMsg.OriginAccount, clientMsg.TargetAccount)
+		case gtsmodel.ActivityStreamsLike:
+			// CREATE LIKE/FAVE
+			fave, ok := clientMsg.GTSModel.(*gtsmodel.StatusFave)
+			if !ok {
+				return errors.New("fave was not parseable as *gtsmodel.StatusFave")
+			}
+
+			if err := p.notifyFave(fave, clientMsg.TargetAccount); err != nil {
+				return err
+			}
+
+			return p.federateFave(fave, clientMsg.OriginAccount, clientMsg.TargetAccount)
+
+		case gtsmodel.ActivityStreamsAnnounce:
+			// CREATE BOOST/ANNOUNCE
+			boostWrapperStatus, ok := clientMsg.GTSModel.(*gtsmodel.Status)
+			if !ok {
+				return errors.New("boost was not parseable as *gtsmodel.Status")
+			}
+
+			if err := p.notifyAnnounce(boostWrapperStatus); err != nil {
+				return err
+			}
+
+			return p.federateAnnounce(boostWrapperStatus, clientMsg.OriginAccount, clientMsg.TargetAccount)
+		}
+	case gtsmodel.ActivityStreamsUpdate:
+		// UPDATE
+		switch clientMsg.APObjectType {
+		case gtsmodel.ActivityStreamsProfile, gtsmodel.ActivityStreamsPerson:
+			// UPDATE ACCOUNT/PROFILE
+			account, ok := clientMsg.GTSModel.(*gtsmodel.Account)
+			if !ok {
+				return errors.New("account was not parseable as *gtsmodel.Account")
+			}
+
+			return p.federateAccountUpdate(account, clientMsg.OriginAccount)
+		}
+	case gtsmodel.ActivityStreamsAccept:
+		// ACCEPT
+		switch clientMsg.APObjectType {
+		case gtsmodel.ActivityStreamsFollow:
+			// ACCEPT FOLLOW
+			follow, ok := clientMsg.GTSModel.(*gtsmodel.Follow)
+			if !ok {
+				return errors.New("accept was not parseable as *gtsmodel.Follow")
+			}
+
+			if err := p.notifyFollow(follow, clientMsg.TargetAccount); err != nil {
+				return err
+			}
+
+			return p.federateAcceptFollowRequest(follow, clientMsg.OriginAccount, clientMsg.TargetAccount)
+		}
+	case gtsmodel.ActivityStreamsUndo:
+		// UNDO
+		switch clientMsg.APObjectType {
+		case gtsmodel.ActivityStreamsFollow:
+			// UNDO FOLLOW
+			follow, ok := clientMsg.GTSModel.(*gtsmodel.Follow)
+			if !ok {
+				return errors.New("undo was not parseable as *gtsmodel.Follow")
+			}
+			return p.federateUnfollow(follow, clientMsg.OriginAccount, clientMsg.TargetAccount)
+		}
+	}
+	return nil
+}
+
+func (p *processor) federateStatus(status *gtsmodel.Status) error {
+	asStatus, err := p.tc.StatusToAS(status)
+	if err != nil {
+		return fmt.Errorf("federateStatus: error converting status to as format: %s", err)
+	}
+
+	outboxIRI, err := url.Parse(status.GTSAuthorAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateStatus: error parsing outboxURI %s: %s", status.GTSAuthorAccount.OutboxURI, err)
+	}
+
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, asStatus)
+	return err
+}
+
+func (p *processor) federateFollow(followRequest *gtsmodel.FollowRequest, originAccount *gtsmodel.Account, targetAccount *gtsmodel.Account) error {
+	// if both accounts are local there's nothing to do here
+	if originAccount.Domain == "" && targetAccount.Domain == "" {
+		return nil
+	}
+
+	follow := p.tc.FollowRequestToFollow(followRequest)
+
+	asFollow, err := p.tc.FollowToAS(follow, originAccount, targetAccount)
+	if err != nil {
+		return fmt.Errorf("federateFollow: error converting follow to as format: %s", err)
+	}
+
+	outboxIRI, err := url.Parse(originAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateFollow: error parsing outboxURI %s: %s", originAccount.OutboxURI, err)
+	}
+
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, asFollow)
+	return err
+}
+
+func (p *processor) federateUnfollow(follow *gtsmodel.Follow, originAccount *gtsmodel.Account, targetAccount *gtsmodel.Account) error {
+	// if both accounts are local there's nothing to do here
+	if originAccount.Domain == "" && targetAccount.Domain == "" {
+		return nil
+	}
+
+	// recreate the follow
+	asFollow, err := p.tc.FollowToAS(follow, originAccount, targetAccount)
+	if err != nil {
+		return fmt.Errorf("federateUnfollow: error converting follow to as format: %s", err)
+	}
+
+	targetAccountURI, err := url.Parse(targetAccount.URI)
+	if err != nil {
+		return fmt.Errorf("error parsing uri %s: %s", targetAccount.URI, err)
+	}
+
+	// create an Undo and set the appropriate actor on it
+	undo := streams.NewActivityStreamsUndo()
+	undo.SetActivityStreamsActor(asFollow.GetActivityStreamsActor())
+
+	// Set the recreated follow as the 'object' property.
+	undoObject := streams.NewActivityStreamsObjectProperty()
+	undoObject.AppendActivityStreamsFollow(asFollow)
+	undo.SetActivityStreamsObject(undoObject)
+
+	// Set the To of the undo as the target of the recreated follow
+	undoTo := streams.NewActivityStreamsToProperty()
+	undoTo.AppendIRI(targetAccountURI)
+	undo.SetActivityStreamsTo(undoTo)
+
+	outboxIRI, err := url.Parse(originAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateUnfollow: error parsing outboxURI %s: %s", originAccount.OutboxURI, err)
+	}
+
+	// send off the Undo
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, undo)
+	return err
+}
+
+func (p *processor) federateAcceptFollowRequest(follow *gtsmodel.Follow, originAccount *gtsmodel.Account, targetAccount *gtsmodel.Account) error {
+	// if both accounts are local there's nothing to do here
+	if originAccount.Domain == "" && targetAccount.Domain == "" {
+		return nil
+	}
+
+	// recreate the AS follow
+	asFollow, err := p.tc.FollowToAS(follow, originAccount, targetAccount)
+	if err != nil {
+		return fmt.Errorf("federateUnfollow: error converting follow to as format: %s", err)
+	}
+
+	acceptingAccountURI, err := url.Parse(targetAccount.URI)
+	if err != nil {
+		return fmt.Errorf("error parsing uri %s: %s", targetAccount.URI, err)
+	}
+
+	requestingAccountURI, err := url.Parse(originAccount.URI)
+	if err != nil {
+		return fmt.Errorf("error parsing uri %s: %s", targetAccount.URI, err)
+	}
+
+	// create an Accept
+	accept := streams.NewActivityStreamsAccept()
+
+	// set the accepting actor on it
+	acceptActorProp := streams.NewActivityStreamsActorProperty()
+	acceptActorProp.AppendIRI(acceptingAccountURI)
+	accept.SetActivityStreamsActor(acceptActorProp)
+
+	// Set the recreated follow as the 'object' property.
+	acceptObject := streams.NewActivityStreamsObjectProperty()
+	acceptObject.AppendActivityStreamsFollow(asFollow)
+	accept.SetActivityStreamsObject(acceptObject)
+
+	// Set the To of the accept as the originator of the follow
+	acceptTo := streams.NewActivityStreamsToProperty()
+	acceptTo.AppendIRI(requestingAccountURI)
+	accept.SetActivityStreamsTo(acceptTo)
+
+	outboxIRI, err := url.Parse(targetAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateAcceptFollowRequest: error parsing outboxURI %s: %s", originAccount.OutboxURI, err)
+	}
+
+	// send off the accept using the accepter's outbox
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, accept)
+	return err
+}
+
+func (p *processor) federateFave(fave *gtsmodel.StatusFave, originAccount *gtsmodel.Account, targetAccount *gtsmodel.Account) error {
+	// if both accounts are local there's nothing to do here
+	if originAccount.Domain == "" && targetAccount.Domain == "" {
+		return nil
+	}
+
+	// create the AS fave
+	asFave, err := p.tc.FaveToAS(fave)
+	if err != nil {
+		return fmt.Errorf("federateFave: error converting fave to as format: %s", err)
+	}
+
+	outboxIRI, err := url.Parse(originAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateFave: error parsing outboxURI %s: %s", originAccount.OutboxURI, err)
+	}
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, asFave)
+	return err
+}
+
+func (p *processor) federateAnnounce(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) error {
+	announce, err := p.tc.BoostToAS(boostWrapperStatus, boostingAccount, boostedAccount)
+	if err != nil {
+		return fmt.Errorf("federateAnnounce: error converting status to announce: %s", err)
+	}
+
+	outboxIRI, err := url.Parse(boostingAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateAnnounce: error parsing outboxURI %s: %s", boostingAccount.OutboxURI, err)
+	}
+
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, announce)
+	return err
+}
+
+func (p *processor) federateAccountUpdate(updatedAccount *gtsmodel.Account, originAccount *gtsmodel.Account) error {
+	person, err := p.tc.AccountToAS(updatedAccount)
+	if err != nil {
+		return fmt.Errorf("federateAccountUpdate: error converting account to person: %s", err)
+	}
+
+	update, err := p.tc.WrapPersonInUpdate(person, originAccount)
+	if err != nil {
+		return fmt.Errorf("federateAccountUpdate: error wrapping person in update: %s", err)
+	}
+
+	outboxIRI, err := url.Parse(originAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateAnnounce: error parsing outboxURI %s: %s", originAccount.OutboxURI, err)
+	}
+
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, update)
+	return err
+}

internal/processing/fromcommon.go

@@ -0,0 +1,164 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) notifyStatus(status *gtsmodel.Status) error {
+	// if there are no mentions in this status then just bail
+	if len(status.Mentions) == 0 {
+		return nil
+	}
+
+	if status.GTSMentions == nil {
+		// there are mentions but they're not fully populated on the status yet so do this
+		menchies := []*gtsmodel.Mention{}
+		for _, m := range status.Mentions {
+			gtsm := &gtsmodel.Mention{}
+			if err := p.db.GetByID(m, gtsm); err != nil {
+				return fmt.Errorf("notifyStatus: error getting mention with id %s from the db: %s", m, err)
+			}
+			menchies = append(menchies, gtsm)
+		}
+		status.GTSMentions = menchies
+	}
+
+	// now we have mentions as full gtsmodel.Mention structs on the status we can continue
+	for _, m := range status.GTSMentions {
+		// make sure this is a local account, otherwise we don't need to create a notification for it
+		if m.GTSAccount == nil {
+			a := &gtsmodel.Account{}
+			if err := p.db.GetByID(m.TargetAccountID, a); err != nil {
+				// we don't have the account or there's been an error
+				return fmt.Errorf("notifyStatus: error getting account with id %s from the db: %s", m.TargetAccountID, err)
+			}
+			m.GTSAccount = a
+		}
+		if m.GTSAccount.Domain != "" {
+			// not a local account so skip it
+			continue
+		}
+
+		// make sure a notif doesn't already exist for this mention
+		err := p.db.GetWhere([]db.Where{
+			{Key: "notification_type", Value: gtsmodel.NotificationMention},
+			{Key: "target_account_id", Value: m.TargetAccountID},
+			{Key: "origin_account_id", Value: status.AccountID},
+			{Key: "status_id", Value: status.ID},
+		}, &gtsmodel.Notification{})
+		if err == nil {
+			// notification exists already so just continue
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// there's a real error in the db
+			return fmt.Errorf("notifyStatus: error checking existence of notification for mention with id %s : %s", m.ID, err)
+		}
+
+		// if we've reached this point we know the mention is for a local account, and the notification doesn't exist, so create it
+		notif := &gtsmodel.Notification{
+			NotificationType: gtsmodel.NotificationMention,
+			TargetAccountID:  m.TargetAccountID,
+			OriginAccountID:  status.AccountID,
+			StatusID:         status.ID,
+		}
+
+		if err := p.db.Put(notif); err != nil {
+			return fmt.Errorf("notifyStatus: error putting notification in database: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func (p *processor) notifyFollowRequest(followRequest *gtsmodel.FollowRequest, receivingAccount *gtsmodel.Account) error {
+	// return if this isn't a local account
+	if receivingAccount.Domain != "" {
+		return nil
+	}
+
+	notif := &gtsmodel.Notification{
+		NotificationType: gtsmodel.NotificationFollowRequest,
+		TargetAccountID:  followRequest.TargetAccountID,
+		OriginAccountID:  followRequest.AccountID,
+	}
+
+	if err := p.db.Put(notif); err != nil {
+		return fmt.Errorf("notifyFollowRequest: error putting notification in database: %s", err)
+	}
+
+	return nil
+}
+
+func (p *processor) notifyFollow(follow *gtsmodel.Follow, receivingAccount *gtsmodel.Account) error {
+	// return if this isn't a local account
+	if receivingAccount.Domain != "" {
+		return nil
+	}
+
+	// first remove the follow request notification
+	if err := p.db.DeleteWhere([]db.Where{
+		{Key: "notification_type", Value: gtsmodel.NotificationFollowRequest},
+		{Key: "target_account_id", Value: follow.TargetAccountID},
+		{Key: "origin_account_id", Value: follow.AccountID},
+	}, &gtsmodel.Notification{}); err != nil {
+		return fmt.Errorf("notifyFollow: error removing old follow request notification from database: %s", err)
+	}
+
+	// now create the new follow notification
+	notif := &gtsmodel.Notification{
+		NotificationType: gtsmodel.NotificationFollow,
+		TargetAccountID:  follow.TargetAccountID,
+		OriginAccountID:  follow.AccountID,
+	}
+	if err := p.db.Put(notif); err != nil {
+		return fmt.Errorf("notifyFollow: error putting notification in database: %s", err)
+	}
+
+	return nil
+}
+
+func (p *processor) notifyFave(fave *gtsmodel.StatusFave, receivingAccount *gtsmodel.Account) error {
+	// return if this isn't a local account
+	if receivingAccount.Domain != "" {
+		return nil
+	}
+
+	notif := &gtsmodel.Notification{
+		NotificationType: gtsmodel.NotificationFave,
+		TargetAccountID:  fave.TargetAccountID,
+		OriginAccountID:  fave.AccountID,
+		StatusID:         fave.StatusID,
+	}
+
+	if err := p.db.Put(notif); err != nil {
+		return fmt.Errorf("notifyFave: error putting notification in database: %s", err)
+	}
+
+	return nil
+}
+
+func (p *processor) notifyAnnounce(status *gtsmodel.Status) error {
+	return nil
+}

internal/processing/fromfederator.go

@@ -0,0 +1,433 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) error {
+	l := p.log.WithFields(logrus.Fields{
+		"func":         "processFromFederator",
+		"federatorMsg": fmt.Sprintf("%+v", federatorMsg),
+	})
+
+	l.Debug("entering function PROCESS FROM FEDERATOR")
+
+	switch federatorMsg.APActivityType {
+	case gtsmodel.ActivityStreamsCreate:
+		// CREATE
+		switch federatorMsg.APObjectType {
+		case gtsmodel.ActivityStreamsNote:
+			// CREATE A STATUS
+			incomingStatus, ok := federatorMsg.GTSModel.(*gtsmodel.Status)
+			if !ok {
+				return errors.New("note was not parseable as *gtsmodel.Status")
+			}
+
+			l.Debug("will now derefence incoming status")
+			if err := p.dereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
+				return fmt.Errorf("error dereferencing status from federator: %s", err)
+			}
+			if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
+				return fmt.Errorf("error updating dereferenced status in the db: %s", err)
+			}
+
+			if err := p.notifyStatus(incomingStatus); err != nil {
+				return err
+			}
+		case gtsmodel.ActivityStreamsProfile:
+			// CREATE AN ACCOUNT
+			incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account)
+			if !ok {
+				return errors.New("profile was not parseable as *gtsmodel.Account")
+			}
+
+			l.Debug("will now derefence incoming account")
+			if err := p.dereferenceAccountFields(incomingAccount, "", false); err != nil {
+				return fmt.Errorf("error dereferencing account from federator: %s", err)
+			}
+			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
+				return fmt.Errorf("error updating dereferenced account in the db: %s", err)
+			}
+		case gtsmodel.ActivityStreamsLike:
+			// CREATE A FAVE
+			incomingFave, ok := federatorMsg.GTSModel.(*gtsmodel.StatusFave)
+			if !ok {
+				return errors.New("like was not parseable as *gtsmodel.StatusFave")
+			}
+
+			if err := p.notifyFave(incomingFave, federatorMsg.ReceivingAccount); err != nil {
+				return err
+			}
+		case gtsmodel.ActivityStreamsFollow:
+			// CREATE A FOLLOW REQUEST
+			incomingFollowRequest, ok := federatorMsg.GTSModel.(*gtsmodel.FollowRequest)
+			if !ok {
+				return errors.New("incomingFollowRequest was not parseable as *gtsmodel.FollowRequest")
+			}
+
+			if err := p.notifyFollowRequest(incomingFollowRequest, federatorMsg.ReceivingAccount); err != nil {
+				return err
+			}
+		case gtsmodel.ActivityStreamsAnnounce:
+			// CREATE AN ANNOUNCE
+			incomingAnnounce, ok := federatorMsg.GTSModel.(*gtsmodel.Status)
+			if !ok {
+				return errors.New("announce was not parseable as *gtsmodel.Status")
+			}
+
+			if err := p.dereferenceAnnounce(incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
+				return fmt.Errorf("error dereferencing announce from federator: %s", err)
+			}
+
+			if err := p.db.Put(incomingAnnounce); err != nil {
+				if _, ok := err.(db.ErrAlreadyExists); !ok {
+					return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
+				}
+			}
+
+			if err := p.notifyAnnounce(incomingAnnounce); err != nil {
+				return err
+			}
+		}
+	case gtsmodel.ActivityStreamsUpdate:
+		// UPDATE
+		switch federatorMsg.APObjectType {
+		case gtsmodel.ActivityStreamsProfile:
+			// UPDATE AN ACCOUNT
+			incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account)
+			if !ok {
+				return errors.New("profile was not parseable as *gtsmodel.Account")
+			}
+
+			l.Debug("will now derefence incoming account")
+			if err := p.dereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
+				return fmt.Errorf("error dereferencing account from federator: %s", err)
+			}
+			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
+				return fmt.Errorf("error updating dereferenced account in the db: %s", err)
+			}
+		}
+	case gtsmodel.ActivityStreamsDelete:
+		// DELETE
+		switch federatorMsg.APObjectType {
+		case gtsmodel.ActivityStreamsNote:
+			// DELETE A STATUS
+			// TODO: handle side effects of status deletion here:
+			// 1. delete all media associated with status
+			// 2. delete boosts of status
+			// 3. etc etc etc
+		case gtsmodel.ActivityStreamsProfile:
+			// DELETE A PROFILE/ACCOUNT
+			// TODO: handle side effects of account deletion here: delete all objects, statuses, media etc associated with account
+		}
+	case gtsmodel.ActivityStreamsAccept:
+		// ACCEPT
+		switch federatorMsg.APObjectType {
+		case gtsmodel.ActivityStreamsFollow:
+			// ACCEPT A FOLLOW
+			follow, ok := federatorMsg.GTSModel.(*gtsmodel.Follow)
+			if !ok {
+				return errors.New("follow was not parseable as *gtsmodel.Follow")
+			}
+
+			if err := p.notifyFollow(follow, federatorMsg.ReceivingAccount); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
+// federated status, back in the federating db's Create function.
+//
+// When a status comes in from the federation API, there are certain fields that
+// haven't been dereferenced yet, because we needed to provide a snappy synchronous
+// response to the caller. By the time it reaches this function though, it's being
+// processed asynchronously, so we have all the time in the world to fetch the various
+// bits and bobs that are attached to the status, and properly flesh it out, before we
+// send the status to any timelines and notify people.
+//
+// Things to dereference and fetch here:
+//
+// 1. Media attachments.
+// 2. Hashtags.
+// 3. Emojis.
+// 4. Mentions.
+// 5. Posting account.
+// 6. Replied-to-status.
+//
+// SIDE EFFECTS:
+// This function will deference all of the above, insert them in the database as necessary,
+// and attach them to the status. The status itself will not be added to the database yet,
+// that's up the caller to do.
+func (p *processor) dereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
+	l := p.log.WithFields(logrus.Fields{
+		"func":   "dereferenceStatusFields",
+		"status": fmt.Sprintf("%+v", status),
+	})
+	l.Debug("entering function")
+
+	t, err := p.federator.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error creating transport: %s", err)
+	}
+
+	// the status should have an ID by now, but just in case it doesn't let's generate one here
+	// because we'll need it further down
+	if status.ID == "" {
+		status.ID = uuid.NewString()
+	}
+
+	// 1. Media attachments.
+	//
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the blurhash (a.Blurhash)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to pass along to the media processor.
+	attachmentIDs := []string{}
+	for _, a := range status.GTSMediaAttachments {
+		l.Debugf("dereferencing attachment: %+v", a)
+
+		// it might have been processed elsewhere so check first if it's already in the database or not
+		maybeAttachment := &gtsmodel.MediaAttachment{}
+		err := p.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
+		if err == nil {
+			// we already have it in the db, dereferenced, no need to do it again
+			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
+			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// we have a real error
+			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		// it just doesn't exist yet so carry on
+		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
+		deferencedAttachment, err := p.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
+		if err != nil {
+			p.log.Errorf("error dereferencing status attachment: %s", err)
+			continue
+		}
+		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
+		deferencedAttachment.StatusID = status.ID
+		deferencedAttachment.Description = a.Description
+		if err := p.db.Put(deferencedAttachment); err != nil {
+			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
+	}
+	status.Attachments = attachmentIDs
+
+	// 2. Hashtags
+
+	// 3. Emojis
+
+	// 4. Mentions
+	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
+	//
+	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+	mentions := []string{}
+	for _, m := range status.GTSMentions {
+		uri, err := url.Parse(m.MentionedAccountURI)
+		if err != nil {
+			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
+			continue
+		}
+
+		m.StatusID = status.ID
+		m.OriginAccountID = status.GTSAuthorAccount.ID
+		m.OriginAccountURI = status.GTSAuthorAccount.URI
+
+		targetAccount := &gtsmodel.Account{}
+		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
+			// proper error
+			if _, ok := err.(db.ErrNoEntries); !ok {
+				return fmt.Errorf("db error checking for account with uri %s", uri.String())
+			}
+
+			// we just don't have it yet, so we should go get it....
+			accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, uri)
+			if err != nil {
+				// we can't dereference it so just skip it
+				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
+				continue
+			}
+
+			targetAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
+			if err != nil {
+				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
+				continue
+			}
+
+			if err := p.db.Put(targetAccount); err != nil {
+				return fmt.Errorf("db error inserting account with uri %s", uri.String())
+			}
+		}
+
+		// by this point, we know the targetAccount exists in our database with an ID :)
+		m.TargetAccountID = targetAccount.ID
+		if err := p.db.Put(m); err != nil {
+			return fmt.Errorf("error creating mention: %s", err)
+		}
+		mentions = append(mentions, m.ID)
+	}
+	status.Mentions = mentions
+
+	return nil
+}
+
+func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
+	l := p.log.WithFields(logrus.Fields{
+		"func":               "dereferenceAccountFields",
+		"requestingUsername": requestingUsername,
+	})
+
+	t, err := p.federator.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error getting transport for user: %s", err)
+	}
+
+	// fetch the header and avatar
+	if err := p.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
+		// if this doesn't work, just skip it -- we can do it later
+		l.Debugf("error fetching header/avi for account: %s", err)
+	}
+
+	if err := p.db.UpdateByID(account.ID, account); err != nil {
+		return fmt.Errorf("error updating account in database: %s", err)
+	}
+
+	return nil
+}
+
+func (p *processor) dereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
+	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+		// we can't do anything unfortunately
+		return errors.New("dereferenceAnnounce: no URI to dereference")
+	}
+
+	// check if we already have the boosted status in the database
+	boostedStatus := &gtsmodel.Status{}
+	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
+	if err == nil {
+		// nice, we already have it so we don't actually need to dereference it from remote
+		announce.Content = boostedStatus.Content
+		announce.ContentWarning = boostedStatus.ContentWarning
+		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+		announce.Sensitive = boostedStatus.Sensitive
+		announce.Language = boostedStatus.Language
+		announce.Text = boostedStatus.Text
+		announce.BoostOfID = boostedStatus.ID
+		announce.Visibility = boostedStatus.Visibility
+		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+		announce.GTSBoostedStatus = boostedStatus
+		return nil
+	}
+
+	// we don't have it so we need to dereference it
+	remoteStatusID, err := url.Parse(announce.GTSBoostedStatus.URI)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error parsing url %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	statusable, err := p.federator.DereferenceRemoteStatus(requestingUsername, remoteStatusID)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// make sure we have the author account in the db
+	attributedToProp := statusable.GetActivityStreamsAttributedTo()
+	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
+		accountURI := iter.GetIRI()
+		if accountURI == nil {
+			continue
+		}
+
+		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
+			// we already have it, fine
+			continue
+		}
+
+		// we don't have the boosted status author account yet so dereference it
+		accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, accountURI)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
+		}
+		account, err := p.tc.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
+		}
+
+		// insert the dereferenced account so it gets an ID etc
+		if err := p.db.Put(account); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
+		}
+
+		if err := p.dereferenceAccountFields(account, requestingUsername, false); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
+		}
+	}
+
+	// now convert the statusable into something we can understand
+	boostedStatus, err = p.tc.ASStatusToStatus(statusable)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// put it in the db already so it gets an ID generated for it
+	if err := p.db.Put(boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// now dereference additional fields straight away (we're already async here so we have time)
+	if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// update with the newly dereferenced fields
+	if err := p.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
+	}
+
+	// we have everything we need!
+	announce.Content = boostedStatus.Content
+	announce.ContentWarning = boostedStatus.ContentWarning
+	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+	announce.Sensitive = boostedStatus.Sensitive
+	announce.Language = boostedStatus.Language
+	announce.Text = boostedStatus.Text
+	announce.BoostOfID = boostedStatus.ID
+	announce.Visibility = boostedStatus.Visibility
+	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.GTSBoostedStatus = boostedStatus
+	return nil
+}

internal/processing/instance.go

@@ -0,0 +1,41 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) InstanceGet(domain string) (*apimodel.Instance, ErrorWithCode) {
+	i := &gtsmodel.Instance{}
+	if err := p.db.GetWhere([]db.Where{{Key: "domain", Value: domain}}, i); err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("db error fetching instance %s: %s", p.config.Host, err))
+	}
+
+	ai, err := p.tc.InstanceToMasto(i)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error converting instance to api representation: %s", err))
+	}
+
+	return ai, nil
+}

internal/processing/media.go

@@ -0,0 +1,285 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (p *processor) MediaCreate(authed *oauth.Auth, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) {
+	// First check this user/account is permitted to create media
+	// There's no point continuing otherwise.
+	//
+	// TODO: move this check to the oauth.Authed function and do it for all accounts
+	if authed.User.Disabled || !authed.User.Approved || !authed.Account.SuspendedAt.IsZero() {
+		return nil, errors.New("not authorized to post new media")
+	}
+
+	// open the attachment and extract the bytes from it
+	f, err := form.File.Open()
+	if err != nil {
+		return nil, fmt.Errorf("error opening attachment: %s", err)
+	}
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("error reading attachment: %s", err)
+
+	}
+	if size == 0 {
+		return nil, errors.New("could not read provided attachment: size 0 bytes")
+	}
+
+	// allow the mediaHandler to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using
+	attachment, err := p.mediaHandler.ProcessAttachment(buf.Bytes(), authed.Account.ID, "")
+	if err != nil {
+		return nil, fmt.Errorf("error reading attachment: %s", err)
+	}
+
+	// now we need to add extra fields that the attachment processor doesn't know (from the form)
+	// TODO: handle this inside mediaHandler.ProcessAttachment (just pass more params to it)
+
+	// first description
+	attachment.Description = form.Description
+
+	// now parse the focus parameter
+	focusx, focusy, err := parseFocus(form.Focus)
+	if err != nil {
+		return nil, err
+	}
+	attachment.FileMeta.Focus.X = focusx
+	attachment.FileMeta.Focus.Y = focusy
+
+	// prepare the frontend representation now -- if there are any errors here at least we can bail without
+	// having already put something in the database and then having to clean it up again (eugh)
+	mastoAttachment, err := p.tc.AttachmentToMasto(attachment)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing media attachment to frontend type: %s", err)
+	}
+
+	// now we can confidently put the attachment in the database
+	if err := p.db.Put(attachment); err != nil {
+		return nil, fmt.Errorf("error storing media attachment in db: %s", err)
+	}
+
+	return &mastoAttachment, nil
+}
+
+func (p *processor) MediaGet(authed *oauth.Auth, mediaAttachmentID string) (*apimodel.Attachment, ErrorWithCode) {
+	attachment := &gtsmodel.MediaAttachment{}
+	if err := p.db.GetByID(mediaAttachmentID, attachment); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			// attachment doesn't exist
+			return nil, NewErrorNotFound(errors.New("attachment doesn't exist in the db"))
+		}
+		return nil, NewErrorNotFound(fmt.Errorf("db error getting attachment: %s", err))
+	}
+
+	if attachment.AccountID != authed.Account.ID {
+		return nil, NewErrorNotFound(errors.New("attachment not owned by requesting account"))
+	}
+
+	a, err := p.tc.AttachmentToMasto(attachment)
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error converting attachment: %s", err))
+	}
+
+	return &a, nil
+}
+
+func (p *processor) MediaUpdate(authed *oauth.Auth, mediaAttachmentID string, form *apimodel.AttachmentUpdateRequest) (*apimodel.Attachment, ErrorWithCode) {
+	attachment := &gtsmodel.MediaAttachment{}
+	if err := p.db.GetByID(mediaAttachmentID, attachment); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			// attachment doesn't exist
+			return nil, NewErrorNotFound(errors.New("attachment doesn't exist in the db"))
+		}
+		return nil, NewErrorNotFound(fmt.Errorf("db error getting attachment: %s", err))
+	}
+
+	if attachment.AccountID != authed.Account.ID {
+		return nil, NewErrorNotFound(errors.New("attachment not owned by requesting account"))
+	}
+
+	if form.Description != nil {
+		attachment.Description = *form.Description
+		if err := p.db.UpdateByID(mediaAttachmentID, attachment); err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("database error updating description: %s", err))
+		}
+	}
+
+	if form.Focus != nil {
+		focusx, focusy, err := parseFocus(*form.Focus)
+		if err != nil {
+			return nil, NewErrorBadRequest(err)
+		}
+		attachment.FileMeta.Focus.X = focusx
+		attachment.FileMeta.Focus.Y = focusy
+		if err := p.db.UpdateByID(mediaAttachmentID, attachment); err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("database error updating focus: %s", err))
+		}
+	}
+
+	a, err := p.tc.AttachmentToMasto(attachment)
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error converting attachment: %s", err))
+	}
+
+	return &a, nil
+}
+
+func (p *processor) FileGet(authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
+	// parse the form fields
+	mediaSize, err := media.ParseMediaSize(form.MediaSize)
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("media size %s not valid", form.MediaSize))
+	}
+
+	mediaType, err := media.ParseMediaType(form.MediaType)
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("media type %s not valid", form.MediaType))
+	}
+
+	spl := strings.Split(form.FileName, ".")
+	if len(spl) != 2 || spl[0] == "" || spl[1] == "" {
+		return nil, NewErrorNotFound(fmt.Errorf("file name %s not parseable", form.FileName))
+	}
+	wantedMediaID := spl[0]
+
+	// get the account that owns the media and make sure it's not suspended
+	acct := &gtsmodel.Account{}
+	if err := p.db.GetByID(form.AccountID, acct); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", form.AccountID, err))
+	}
+	if !acct.SuspendedAt.IsZero() {
+		return nil, NewErrorNotFound(fmt.Errorf("account with id %s is suspended", form.AccountID))
+	}
+
+	// make sure the requesting account and the media account don't block each other
+	if authed.Account != nil {
+		blocked, err := p.db.Blocked(authed.Account.ID, form.AccountID)
+		if err != nil {
+			return nil, NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", form.AccountID, authed.Account.ID, err))
+		}
+		if blocked {
+			return nil, NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", form.AccountID, authed.Account.ID))
+		}
+	}
+
+	// the way we store emojis is a little different from the way we store other attachments,
+	// so we need to take different steps depending on the media type being requested
+	content := &apimodel.Content{}
+	var storagePath string
+	switch mediaType {
+	case media.Emoji:
+		e := &gtsmodel.Emoji{}
+		if err := p.db.GetByID(wantedMediaID, e); err != nil {
+			return nil, NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedMediaID, err))
+		}
+		if e.Disabled {
+			return nil, NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedMediaID))
+		}
+		switch mediaSize {
+		case media.Original:
+			content.ContentType = e.ImageContentType
+			storagePath = e.ImagePath
+		case media.Static:
+			content.ContentType = e.ImageStaticContentType
+			storagePath = e.ImageStaticPath
+		default:
+			return nil, NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", mediaSize))
+		}
+	case media.Attachment, media.Header, media.Avatar:
+		a := &gtsmodel.MediaAttachment{}
+		if err := p.db.GetByID(wantedMediaID, a); err != nil {
+			return nil, NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
+		}
+		if a.AccountID != form.AccountID {
+			return nil, NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, form.AccountID))
+		}
+		switch mediaSize {
+		case media.Original:
+			content.ContentType = a.File.ContentType
+			storagePath = a.File.Path
+		case media.Small:
+			content.ContentType = a.Thumbnail.ContentType
+			storagePath = a.Thumbnail.Path
+		default:
+			return nil, NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
+		}
+	}
+
+	bytes, err := p.storage.RetrieveFileFrom(storagePath)
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))
+	}
+
+	content.ContentLength = int64(len(bytes))
+	content.Content = bytes
+	return content, nil
+}
+
+func parseFocus(focus string) (focusx, focusy float32, err error) {
+	if focus == "" {
+		return
+	}
+	spl := strings.Split(focus, ",")
+	if len(spl) != 2 {
+		err = fmt.Errorf("improperly formatted focus %s", focus)
+		return
+	}
+	xStr := spl[0]
+	yStr := spl[1]
+	if xStr == "" || yStr == "" {
+		err = fmt.Errorf("improperly formatted focus %s", focus)
+		return
+	}
+	fx, err := strconv.ParseFloat(xStr, 32)
+	if err != nil {
+		err = fmt.Errorf("improperly formatted focus %s: %s", focus, err)
+		return
+	}
+	if fx > 1 || fx < -1 {
+		err = fmt.Errorf("improperly formatted focus %s", focus)
+		return
+	}
+	focusx = float32(fx)
+	fy, err := strconv.ParseFloat(yStr, 32)
+	if err != nil {
+		err = fmt.Errorf("improperly formatted focus %s: %s", focus, err)
+		return
+	}
+	if fy > 1 || fy < -1 {
+		err = fmt.Errorf("improperly formatted focus %s", focus)
+		return
+	}
+	focusy = float32(fy)
+	return
+}

internal/processing/notification.go

@@ -0,0 +1,45 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (p *processor) NotificationsGet(authed *oauth.Auth, limit int, maxID string) ([]*apimodel.Notification, ErrorWithCode) {
+	l := p.log.WithField("func", "NotificationsGet")
+
+	notifs, err := p.db.GetNotificationsForAccount(authed.Account.ID, limit, maxID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	mastoNotifs := []*apimodel.Notification{}
+	for _, n := range notifs {
+		mastoNotif, err := p.tc.NotificationToMasto(n)
+		if err != nil {
+			l.Debugf("got an error converting a notification to masto, will skip it: %s", err)
+			continue
+		}
+		mastoNotifs = append(mastoNotifs, mastoNotif)
+	}
+
+	return mastoNotifs, nil
+}

internal/processing/processor.go

@@ -0,0 +1,255 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/blob"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+// Processor should be passed to api modules (see internal/apimodule/...). It is used for
+// passing messages back and forth from the client API and the federating interface, via channels.
+// It also contains logic for filtering which messages should end up where.
+// It is designed to be used asynchronously: the client API and the federating API should just be able to
+// fire messages into the processor and not wait for a reply before proceeding with other work. This allows
+// for clean distribution of messages without slowing down the client API and harming the user experience.
+type Processor interface {
+	// ToClientAPI returns a channel for putting in messages that need to go to the gts client API.
+	// ToClientAPI() chan gtsmodel.ToClientAPI
+	// FromClientAPI returns a channel for putting messages in that come from the client api going to the processor
+	FromClientAPI() chan gtsmodel.FromClientAPI
+	// ToFederator returns a channel for putting in messages that need to go to the federator (activitypub).
+	// ToFederator() chan gtsmodel.ToFederator
+	// FromFederator returns a channel for putting messages in that come from the federator (activitypub) going into the processor
+	FromFederator() chan gtsmodel.FromFederator
+	// Start starts the Processor, reading from its channels and passing messages back and forth.
+	Start() error
+	// Stop stops the processor cleanly, finishing handling any remaining messages before closing down.
+	Stop() error
+
+	/*
+		CLIENT API-FACING PROCESSING FUNCTIONS
+		These functions are intended to be called when the API client needs an immediate (ie., synchronous) reply
+		to an HTTP request. As such, they will only do the bare-minimum of work necessary to give a properly
+		formed reply. For more intensive (and time-consuming) calls, where you don't require an immediate
+		response, pass work to the processor using a channel instead.
+	*/
+
+	// AccountCreate processes the given form for creating a new account, returning an oauth token for that account if successful.
+	AccountCreate(authed *oauth.Auth, form *apimodel.AccountCreateRequest) (*apimodel.Token, error)
+	// AccountGet processes the given request for account information.
+	AccountGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Account, error)
+	// AccountUpdate processes the update of an account with the given form
+	AccountUpdate(authed *oauth.Auth, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error)
+	// AccountStatusesGet fetches a number of statuses (in time descending order) from the given account, filtered by visibility for
+	// the account given in authed.
+	AccountStatusesGet(authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, ErrorWithCode)
+	// AccountFollowersGet fetches a list of the target account's followers.
+	AccountFollowersGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, ErrorWithCode)
+	// AccountFollowingGet fetches a list of the accounts that target account is following.
+	AccountFollowingGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, ErrorWithCode)
+	// AccountRelationshipGet returns a relationship model describing the relationship of the targetAccount to the Authed account.
+	AccountRelationshipGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, ErrorWithCode)
+	// AccountFollowCreate handles a follow request to an account, either remote or local.
+	AccountFollowCreate(authed *oauth.Auth, form *apimodel.AccountFollowRequest) (*apimodel.Relationship, ErrorWithCode)
+	// AccountFollowRemove handles the removal of a follow/follow request to an account, either remote or local.
+	AccountFollowRemove(authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, ErrorWithCode)
+
+	// AdminEmojiCreate handles the creation of a new instance emoji by an admin, using the given form.
+	AdminEmojiCreate(authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error)
+
+	// AppCreate processes the creation of a new API application
+	AppCreate(authed *oauth.Auth, form *apimodel.ApplicationCreateRequest) (*apimodel.Application, error)
+
+	// FileGet handles the fetching of a media attachment file via the fileserver.
+	FileGet(authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
+
+	// FollowRequestsGet handles the getting of the authed account's incoming follow requests
+	FollowRequestsGet(auth *oauth.Auth) ([]apimodel.Account, ErrorWithCode)
+	// FollowRequestAccept handles the acceptance of a follow request from the given account ID
+	FollowRequestAccept(auth *oauth.Auth, accountID string) (*apimodel.Relationship, ErrorWithCode)
+
+	// InstanceGet retrieves instance information for serving at api/v1/instance
+	InstanceGet(domain string) (*apimodel.Instance, ErrorWithCode)
+
+	// MediaCreate handles the creation of a media attachment, using the given form.
+	MediaCreate(authed *oauth.Auth, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error)
+	// MediaGet handles the GET of a media attachment with the given ID
+	MediaGet(authed *oauth.Auth, attachmentID string) (*apimodel.Attachment, ErrorWithCode)
+	// MediaUpdate handles the PUT of a media attachment with the given ID and form
+	MediaUpdate(authed *oauth.Auth, attachmentID string, form *apimodel.AttachmentUpdateRequest) (*apimodel.Attachment, ErrorWithCode)
+
+	// NotificationsGet
+	NotificationsGet(authed *oauth.Auth, limit int, maxID string) ([]*apimodel.Notification, ErrorWithCode)
+
+	// SearchGet performs a search with the given params, resolving/dereferencing remotely as desired
+	SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode)
+
+	// StatusCreate processes the given form to create a new status, returning the api model representation of that status if it's OK.
+	StatusCreate(authed *oauth.Auth, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, error)
+	// StatusDelete processes the delete of a given status, returning the deleted status if the delete goes through.
+	StatusDelete(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error)
+	// StatusFave processes the faving of a given status, returning the updated status if the fave goes through.
+	StatusFave(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error)
+	// StatusBoost processes the boost/reblog of a given status, returning the newly-created boost if all is well.
+	StatusBoost(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, ErrorWithCode)
+	// StatusFavedBy returns a slice of accounts that have liked the given status, filtered according to privacy settings.
+	StatusFavedBy(authed *oauth.Auth, targetStatusID string) ([]*apimodel.Account, error)
+	// StatusGet gets the given status, taking account of privacy settings and blocks etc.
+	StatusGet(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error)
+	// StatusUnfave processes the unfaving of a given status, returning the updated status if the fave goes through.
+	StatusUnfave(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error)
+
+	// HomeTimelineGet returns statuses from the home timeline, with the given filters/parameters.
+	HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]apimodel.Status, ErrorWithCode)
+
+	/*
+		FEDERATION API-FACING PROCESSING FUNCTIONS
+		These functions are intended to be called when the federating client needs an immediate (ie., synchronous) reply
+		to an HTTP request. As such, they will only do the bare-minimum of work necessary to give a properly
+		formed reply. For more intensive (and time-consuming) calls, where you don't require an immediate
+		response, pass work to the processor using a channel instead.
+	*/
+
+	// GetFediUser handles the getting of a fedi/activitypub representation of a user/account, performing appropriate authentication
+	// before returning a JSON serializable interface to the caller.
+	GetFediUser(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode)
+
+	// GetFediFollowers handles the getting of a fedi/activitypub representation of a user/account's followers, performing appropriate
+	// authentication before returning a JSON serializable interface to the caller.
+	GetFediFollowers(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode)
+
+	// GetFediFollowing handles the getting of a fedi/activitypub representation of a user/account's following, performing appropriate
+	// authentication before returning a JSON serializable interface to the caller.
+	GetFediFollowing(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode)
+
+	// GetFediStatus handles the getting of a fedi/activitypub representation of a particular status, performing appropriate
+	// authentication before returning a JSON serializable interface to the caller.
+	GetFediStatus(requestedUsername string, requestedStatusID string, request *http.Request) (interface{}, ErrorWithCode)
+
+	// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
+	GetWebfingerAccount(requestedUsername string, request *http.Request) (*apimodel.WebfingerAccountResponse, ErrorWithCode)
+
+	// InboxPost handles POST requests to a user's inbox for new activitypub messages.
+	//
+	// InboxPost returns true if the request was handled as an ActivityPub POST to an actor's inbox.
+	// If false, the request was not an ActivityPub request and may still be handled by the caller in another way, such as serving a web page.
+	//
+	// If the error is nil, then the ResponseWriter's headers and response has already been written. If a non-nil error is returned, then no response has been written.
+	//
+	// If the Actor was constructed with the Federated Protocol enabled, side effects will occur.
+	//
+	// If the Federated Protocol is not enabled, writes the http.StatusMethodNotAllowed status code in the response. No side effects occur.
+	InboxPost(ctx context.Context, w http.ResponseWriter, r *http.Request) (bool, error)
+}
+
+// processor just implements the Processor interface
+type processor struct {
+	// federator     pub.FederatingActor
+	// toClientAPI   chan gtsmodel.ToClientAPI
+	fromClientAPI chan gtsmodel.FromClientAPI
+	// toFederator   chan gtsmodel.ToFederator
+	fromFederator chan gtsmodel.FromFederator
+	federator     federation.Federator
+	stop          chan interface{}
+	log           *logrus.Logger
+	config        *config.Config
+	tc            typeutils.TypeConverter
+	oauthServer   oauth.Server
+	mediaHandler  media.Handler
+	storage       blob.Storage
+	db            db.DB
+}
+
+// NewProcessor returns a new Processor that uses the given federator and logger
+func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator federation.Federator, oauthServer oauth.Server, mediaHandler media.Handler, storage blob.Storage, db db.DB, log *logrus.Logger) Processor {
+	return &processor{
+		// toClientAPI:   make(chan gtsmodel.ToClientAPI, 100),
+		fromClientAPI: make(chan gtsmodel.FromClientAPI, 100),
+		// toFederator:   make(chan gtsmodel.ToFederator, 100),
+		fromFederator: make(chan gtsmodel.FromFederator, 100),
+		federator:     federator,
+		stop:          make(chan interface{}),
+		log:           log,
+		config:        config,
+		tc:            tc,
+		oauthServer:   oauthServer,
+		mediaHandler:  mediaHandler,
+		storage:       storage,
+		db:            db,
+	}
+}
+
+// func (p *processor) ToClientAPI() chan gtsmodel.ToClientAPI {
+// 	return p.toClientAPI
+// }
+
+func (p *processor) FromClientAPI() chan gtsmodel.FromClientAPI {
+	return p.fromClientAPI
+}
+
+// func (p *processor) ToFederator() chan gtsmodel.ToFederator {
+// 	return p.toFederator
+// }
+
+func (p *processor) FromFederator() chan gtsmodel.FromFederator {
+	return p.fromFederator
+}
+
+// Start starts the Processor, reading from its channels and passing messages back and forth.
+func (p *processor) Start() error {
+	go func() {
+	DistLoop:
+		for {
+			select {
+			case clientMsg := <-p.fromClientAPI:
+				p.log.Infof("received message FROM client API: %+v", clientMsg)
+				if err := p.processFromClientAPI(clientMsg); err != nil {
+					p.log.Error(err)
+				}
+			case federatorMsg := <-p.fromFederator:
+				p.log.Infof("received message FROM federator: %+v", federatorMsg)
+				if err := p.processFromFederator(federatorMsg); err != nil {
+					p.log.Error(err)
+				}
+			case <-p.stop:
+				break DistLoop
+			}
+		}
+	}()
+	return nil
+}
+
+// Stop stops the processor cleanly, finishing handling any remaining messages before closing down.
+// TODO: empty message buffer properly before stopping otherwise we'll lose federating messages.
+func (p *processor) Stop() error {
+	close(p.stop)
+	return nil
+}

internal/processing/search.go

@@ -0,0 +1,295 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode) {
+	l := p.log.WithFields(logrus.Fields{
+		"func":  "SearchGet",
+		"query": searchQuery.Query,
+	})
+
+	results := &apimodel.SearchResult{
+		Accounts: []apimodel.Account{},
+		Statuses: []apimodel.Status{},
+		Hashtags: []apimodel.Tag{},
+	}
+	foundAccounts := []*gtsmodel.Account{}
+	foundStatuses := []*gtsmodel.Status{}
+	// foundHashtags := []*gtsmodel.Tag{}
+
+	// convert the query to lowercase and trim leading/trailing spaces
+	query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
+
+	var foundOne bool
+	// check if the query is something like @whatever_username@example.org -- this means it's a remote account
+	if !foundOne && util.IsMention(searchQuery.Query) {
+		l.Debug("search term is a mention, looking it up...")
+		foundAccount, err := p.searchAccountByMention(authed, searchQuery.Query, searchQuery.Resolve)
+		if err == nil && foundAccount != nil {
+			foundAccounts = append(foundAccounts, foundAccount)
+			foundOne = true
+			l.Debug("got an account by searching by mention")
+		}
+	}
+
+	// check if the query is a URI and just do a lookup for that, straight up
+	if uri, err := url.Parse(query); err == nil && !foundOne {
+		// 1. check if it's a status
+		if foundStatus, err := p.searchStatusByURI(authed, uri, searchQuery.Resolve); err == nil && foundStatus != nil {
+			foundStatuses = append(foundStatuses, foundStatus)
+			foundOne = true
+			l.Debug("got a status by searching by URI")
+		}
+
+		// 2. check if it's an account
+		if foundAccount, err := p.searchAccountByURI(authed, uri, searchQuery.Resolve); err == nil && foundAccount != nil {
+			foundAccounts = append(foundAccounts, foundAccount)
+			foundOne = true
+			l.Debug("got an account by searching by URI")
+		}
+	}
+
+	if !foundOne {
+		// we haven't found anything yet so search for text now
+		l.Debug("nothing found by mention or by URI, will fall back to searching by text now")
+	}
+
+	/*
+		FROM HERE ON we have our search results, it's just a matter of filtering them according to what this user is allowed to see,
+		and then converting them into our frontend format.
+	*/
+	for _, foundAccount := range foundAccounts {
+		// make sure there's no block in either direction between the account and the requester
+		if blocked, err := p.db.Blocked(authed.Account.ID, foundAccount.ID); err == nil && !blocked {
+			// all good, convert it and add it to the results
+			if acctMasto, err := p.tc.AccountToMastoPublic(foundAccount); err == nil && acctMasto != nil {
+				results.Accounts = append(results.Accounts, *acctMasto)
+			}
+		}
+	}
+
+	for _, foundStatus := range foundStatuses {
+		statusOwner := &gtsmodel.Account{}
+		if err := p.db.GetByID(foundStatus.AccountID, statusOwner); err != nil {
+			continue
+		}
+
+		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(foundStatus)
+		if err != nil {
+			continue
+		}
+		if visible, err := p.db.StatusVisible(foundStatus, statusOwner, authed.Account, relevantAccounts); !visible || err != nil {
+			continue
+		}
+
+		statusMasto, err := p.tc.StatusToMasto(foundStatus, statusOwner, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, nil)
+		if err != nil {
+			continue
+		}
+
+		results.Statuses = append(results.Statuses, *statusMasto)
+	}
+
+	return results, nil
+}
+
+func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) {
+
+	maybeStatus := &gtsmodel.Status{}
+	if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
+		// we have it and it's a status
+		return maybeStatus, nil
+	} else if err := p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
+		// we have it and it's a status
+		return maybeStatus, nil
+	}
+
+	// we don't have it locally so dereference it if we're allowed to
+	if resolve {
+		statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri)
+		if err == nil {
+			// it IS a status!
+
+			// extract the status owner's IRI from the statusable
+			var statusOwnerURI *url.URL
+			statusAttributedTo := statusable.GetActivityStreamsAttributedTo()
+			for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() {
+				if i.IsIRI() {
+					statusOwnerURI = i.GetIRI()
+					break
+				}
+			}
+			if statusOwnerURI == nil {
+				return nil, errors.New("couldn't extract ownerAccountURI from statusable")
+			}
+
+			// make sure the status owner exists in the db by searching for it
+			_, err := p.searchAccountByURI(authed, statusOwnerURI, resolve)
+			if err != nil {
+				return nil, err
+			}
+
+			// we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly
+
+			// first turn it into a gtsmodel.Status
+			status, err := p.tc.ASStatusToStatus(statusable)
+			if err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			// put it in the DB so it gets a UUID
+			if err := p.db.Put(status); err != nil {
+				return nil, fmt.Errorf("error putting status in the db: %s", err)
+			}
+
+			// properly dereference everything in the status (media attachments etc)
+			if err := p.dereferenceStatusFields(status, authed.Account.Username); err != nil {
+				return nil, fmt.Errorf("error dereferencing status fields: %s", err)
+			}
+
+			// update with the nicely dereferenced status
+			if err := p.db.UpdateByID(status.ID, status); err != nil {
+				return nil, fmt.Errorf("error updating status in the db: %s", err)
+			}
+
+			return status, nil
+		}
+	}
+	return nil, nil
+}
+
+func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Account, error) {
+	maybeAccount := &gtsmodel.Account{}
+	if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
+		// we have it and it's an account
+		return maybeAccount, nil
+	} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
+		// we have it and it's an account
+		return maybeAccount, nil
+	}
+	if resolve {
+		// we don't have it locally so try and dereference it
+		accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri)
+		if err != nil {
+			return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
+		}
+
+		// it IS an account!
+		account, err := p.tc.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
+		}
+
+		if err := p.db.Put(account); err != nil {
+			return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err)
+		}
+
+		if err := p.dereferenceAccountFields(account, authed.Account.Username, false); err != nil {
+			return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err)
+		}
+
+		return account, nil
+	}
+	return nil, nil
+}
+
+func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, resolve bool) (*gtsmodel.Account, error) {
+	// query is for a remote account
+	username, domain, err := util.ExtractMentionParts(mention)
+	if err != nil {
+		return nil, fmt.Errorf("searchAccountByMention: error extracting mention parts: %s", err)
+	}
+
+	// if it's a local account we can skip a whole bunch of stuff
+	maybeAcct := &gtsmodel.Account{}
+	if domain == p.config.Host {
+		if err = p.db.GetLocalAccountByUsername(username, maybeAcct); err != nil {
+			return nil, fmt.Errorf("searchAccountByMention: error getting local account by username: %s", err)
+		}
+		return maybeAcct, nil
+	}
+
+	// it's not a local account so first we'll check if it's in the database already...
+	where := []db.Where{
+		{Key: "username", Value: username, CaseInsensitive: true},
+		{Key: "domain", Value: domain, CaseInsensitive: true},
+	}
+	err = p.db.GetWhere(where, maybeAcct)
+	if err == nil {
+		// we've got it stored locally already!
+		return maybeAcct, nil
+	}
+
+	if _, ok := err.(db.ErrNoEntries); !ok {
+		// if it's  not errNoEntries there's been a real database error so bail at this point
+		return nil, fmt.Errorf("searchAccountByMention: database error: %s", err)
+	}
+
+	// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
+	if resolve {
+		// we're allowed to resolve it so let's try
+
+		// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
+		acctURI, err := p.federator.FingerRemoteAccount(authed.Account.Username, username, domain)
+		if err != nil {
+			// something went wrong doing the webfinger lookup so we can't process the request
+			return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err)
+		}
+
+		// dereference the account based on the URI we retrieved from the webfinger lookup
+		accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI)
+		if err != nil {
+			// something went wrong doing the dereferencing so we can't process the request
+			return nil, fmt.Errorf("searchAccountByMention: error dereferencing remote account with uri %s: %s", acctURI.String(), err)
+		}
+
+		// convert the dereferenced account to the gts model of that account
+		foundAccount, err := p.tc.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			// something went wrong doing the conversion to a gtsmodel.Account so we can't process the request
+			return nil, fmt.Errorf("searchAccountByMention: error converting account with uri %s: %s", acctURI.String(), err)
+		}
+
+		// put this new account in our database
+		if err := p.db.Put(foundAccount); err != nil {
+			return nil, fmt.Errorf("searchAccountByMention: error inserting account with uri %s: %s", acctURI.String(), err)
+		}
+
+		// properly dereference all the fields on the account immediately
+		if err := p.dereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
+			return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err)
+		}
+	}
+
+	return nil, nil
+}

internal/processing/status.go

@@ -0,0 +1,481 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (p *processor) StatusCreate(auth *oauth.Auth, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, error) {
+	uris := util.GenerateURIsForAccount(auth.Account.Username, p.config.Protocol, p.config.Host)
+	thisStatusID := uuid.NewString()
+	thisStatusURI := fmt.Sprintf("%s/%s", uris.StatusesURI, thisStatusID)
+	thisStatusURL := fmt.Sprintf("%s/%s", uris.StatusesURL, thisStatusID)
+	newStatus := &gtsmodel.Status{
+		ID:                       thisStatusID,
+		URI:                      thisStatusURI,
+		URL:                      thisStatusURL,
+		Content:                  util.HTMLFormat(form.Status),
+		CreatedAt:                time.Now(),
+		UpdatedAt:                time.Now(),
+		Local:                    true,
+		AccountID:                auth.Account.ID,
+		ContentWarning:           form.SpoilerText,
+		ActivityStreamsType:      gtsmodel.ActivityStreamsNote,
+		Sensitive:                form.Sensitive,
+		Language:                 form.Language,
+		CreatedWithApplicationID: auth.Application.ID,
+		Text:                     form.Status,
+	}
+
+	// check if replyToID is ok
+	if err := p.processReplyToID(form, auth.Account.ID, newStatus); err != nil {
+		return nil, err
+	}
+
+	// check if mediaIDs are ok
+	if err := p.processMediaIDs(form, auth.Account.ID, newStatus); err != nil {
+		return nil, err
+	}
+
+	// check if visibility settings are ok
+	if err := p.processVisibility(form, auth.Account.Privacy, newStatus); err != nil {
+		return nil, err
+	}
+
+	// handle language settings
+	if err := p.processLanguage(form, auth.Account.Language, newStatus); err != nil {
+		return nil, err
+	}
+
+	// handle mentions
+	if err := p.processMentions(form, auth.Account.ID, newStatus); err != nil {
+		return nil, err
+	}
+
+	if err := p.processTags(form, auth.Account.ID, newStatus); err != nil {
+		return nil, err
+	}
+
+	if err := p.processEmojis(form, auth.Account.ID, newStatus); err != nil {
+		return nil, err
+	}
+
+	// put the new status in the database, generating an ID for it in the process
+	if err := p.db.Put(newStatus); err != nil {
+		return nil, err
+	}
+
+	// change the status ID of the media attachments to the new status
+	for _, a := range newStatus.GTSMediaAttachments {
+		a.StatusID = newStatus.ID
+		a.UpdatedAt = time.Now()
+		if err := p.db.UpdateByID(a.ID, a); err != nil {
+			return nil, err
+		}
+	}
+
+	// put the new status in the appropriate channel for async processing
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   newStatus.ActivityStreamsType,
+		APActivityType: gtsmodel.ActivityStreamsCreate,
+		GTSModel:       newStatus,
+	}
+
+	// return the frontend representation of the new status to the submitter
+	return p.tc.StatusToMasto(newStatus, auth.Account, auth.Account, nil, newStatus.GTSReplyToAccount, nil)
+}
+
+func (p *processor) StatusDelete(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error) {
+	l := p.log.WithField("func", "StatusDelete")
+	l.Tracef("going to search for target status %s", targetStatusID)
+	targetStatus := &gtsmodel.Status{}
+	if err := p.db.GetByID(targetStatusID, targetStatus); err != nil {
+		return nil, fmt.Errorf("error fetching status %s: %s", targetStatusID, err)
+	}
+
+	if targetStatus.AccountID != authed.Account.ID {
+		return nil, errors.New("status doesn't belong to requesting account")
+	}
+
+	l.Trace("going to get relevant accounts")
+	relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(targetStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching related accounts for status %s: %s", targetStatusID, err)
+	}
+
+	var boostOfStatus *gtsmodel.Status
+	if targetStatus.BoostOfID != "" {
+		boostOfStatus = &gtsmodel.Status{}
+		if err := p.db.GetByID(targetStatus.BoostOfID, boostOfStatus); err != nil {
+			return nil, fmt.Errorf("error fetching boosted status %s: %s", targetStatus.BoostOfID, err)
+		}
+	}
+
+	mastoStatus, err := p.tc.StatusToMasto(targetStatus, authed.Account, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostOfStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error converting status %s to frontend representation: %s", targetStatus.ID, err)
+	}
+
+	if err := p.db.DeleteByID(targetStatus.ID, targetStatus); err != nil {
+		return nil, fmt.Errorf("error deleting status from the database: %s", err)
+	}
+
+	return mastoStatus, nil
+}
+
+func (p *processor) StatusFave(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error) {
+	l := p.log.WithField("func", "StatusFave")
+	l.Tracef("going to search for target status %s", targetStatusID)
+	targetStatus := &gtsmodel.Status{}
+	if err := p.db.GetByID(targetStatusID, targetStatus); err != nil {
+		return nil, fmt.Errorf("error fetching status %s: %s", targetStatusID, err)
+	}
+
+	l.Tracef("going to search for target account %s", targetStatus.AccountID)
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetStatus.AccountID, targetAccount); err != nil {
+		return nil, fmt.Errorf("error fetching target account %s: %s", targetStatus.AccountID, err)
+	}
+
+	l.Trace("going to get relevant accounts")
+	relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(targetStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching related accounts for status %s: %s", targetStatusID, err)
+	}
+
+	var boostOfStatus *gtsmodel.Status
+	if targetStatus.BoostOfID != "" {
+		boostOfStatus = &gtsmodel.Status{}
+		if err := p.db.GetByID(targetStatus.BoostOfID, boostOfStatus); err != nil {
+			return nil, fmt.Errorf("error fetching boosted status %s: %s", targetStatus.BoostOfID, err)
+		}
+	}
+
+	l.Trace("going to see if status is visible")
+	visible, err := p.db.StatusVisible(targetStatus, targetAccount, authed.Account, relevantAccounts) // requestingAccount might well be nil here, but StatusVisible knows how to take care of that
+	if err != nil {
+		return nil, fmt.Errorf("error seeing if status %s is visible: %s", targetStatus.ID, err)
+	}
+
+	if !visible {
+		return nil, errors.New("status is not visible")
+	}
+
+	// is the status faveable?
+	if targetStatus.VisibilityAdvanced != nil {
+		if !targetStatus.VisibilityAdvanced.Likeable {
+			return nil, errors.New("status is not faveable")
+		}
+	}
+
+	// first check if the status is already faved, if so we don't need to do anything
+	newFave := true
+	gtsFave := &gtsmodel.Status{}
+	if err := p.db.GetWhere([]db.Where{{Key: "status_id", Value: targetStatus.ID}, {Key: "account_id", Value: authed.Account.ID}}, gtsFave); err == nil {
+		// we already have a fave for this status
+		newFave = false
+	}
+
+	if newFave {
+		thisFaveID := uuid.NewString()
+
+		// we need to create a new fave in the database
+		gtsFave := &gtsmodel.StatusFave{
+			ID:               thisFaveID,
+			AccountID:        authed.Account.ID,
+			TargetAccountID:  targetAccount.ID,
+			StatusID:         targetStatus.ID,
+			URI:              util.GenerateURIForLike(authed.Account.Username, p.config.Protocol, p.config.Host, thisFaveID),
+			GTSStatus:        targetStatus,
+			GTSTargetAccount: targetAccount,
+			GTSFavingAccount: authed.Account,
+		}
+
+		if err := p.db.Put(gtsFave); err != nil {
+			return nil, err
+		}
+
+		// send the new fave through the processor channel for federation etc
+		p.fromClientAPI <- gtsmodel.FromClientAPI{
+			APObjectType:   gtsmodel.ActivityStreamsLike,
+			APActivityType: gtsmodel.ActivityStreamsCreate,
+			GTSModel:       gtsFave,
+			OriginAccount:  authed.Account,
+			TargetAccount:  targetAccount,
+		}
+	}
+
+	// return the mastodon representation of the target status
+	mastoStatus, err := p.tc.StatusToMasto(targetStatus, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostOfStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error converting status %s to frontend representation: %s", targetStatus.ID, err)
+	}
+
+	return mastoStatus, nil
+}
+
+func (p *processor) StatusBoost(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, ErrorWithCode) {
+	l := p.log.WithField("func", "StatusBoost")
+
+	l.Tracef("going to search for target status %s", targetStatusID)
+	targetStatus := &gtsmodel.Status{}
+	if err := p.db.GetByID(targetStatusID, targetStatus); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error fetching status %s: %s", targetStatusID, err))
+	}
+
+	l.Tracef("going to search for target account %s", targetStatus.AccountID)
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetStatus.AccountID, targetAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error fetching target account %s: %s", targetStatus.AccountID, err))
+	}
+
+	l.Trace("going to get relevant accounts")
+	relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(targetStatus)
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error fetching related accounts for status %s: %s", targetStatusID, err))
+	}
+
+	l.Trace("going to see if status is visible")
+	visible, err := p.db.StatusVisible(targetStatus, targetAccount, authed.Account, relevantAccounts) // requestingAccount might well be nil here, but StatusVisible knows how to take care of that
+	if err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("error seeing if status %s is visible: %s", targetStatus.ID, err))
+	}
+
+	if !visible {
+		return nil, NewErrorNotFound(errors.New("status is not visible"))
+	}
+
+	if targetStatus.VisibilityAdvanced != nil {
+		if !targetStatus.VisibilityAdvanced.Boostable {
+			return nil, NewErrorForbidden(errors.New("status is not boostable"))
+		}
+	}
+
+	// it's visible! it's boostable! so let's boost the FUCK out of it
+	boostWrapperStatus, err := p.tc.StatusToBoost(targetStatus, authed.Account)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	boostWrapperStatus.CreatedWithApplicationID = authed.Application.ID
+	boostWrapperStatus.GTSBoostedAccount = targetAccount
+
+	// put the boost in the database
+	if err := p.db.Put(boostWrapperStatus); err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	// send it to the processor for async processing
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsAnnounce,
+		APActivityType: gtsmodel.ActivityStreamsCreate,
+		GTSModel:       boostWrapperStatus,
+		OriginAccount:  authed.Account,
+		TargetAccount:  targetAccount,
+	}
+
+	// return the frontend representation of the new status to the submitter
+	mastoStatus, err := p.tc.StatusToMasto(boostWrapperStatus, authed.Account, authed.Account, targetAccount, nil, targetStatus)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error converting status %s to frontend representation: %s", targetStatus.ID, err))
+	}
+
+	return mastoStatus, nil
+}
+
+func (p *processor) StatusFavedBy(authed *oauth.Auth, targetStatusID string) ([]*apimodel.Account, error) {
+	l := p.log.WithField("func", "StatusFavedBy")
+
+	l.Tracef("going to search for target status %s", targetStatusID)
+	targetStatus := &gtsmodel.Status{}
+	if err := p.db.GetByID(targetStatusID, targetStatus); err != nil {
+		return nil, fmt.Errorf("error fetching status %s: %s", targetStatusID, err)
+	}
+
+	l.Tracef("going to search for target account %s", targetStatus.AccountID)
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetStatus.AccountID, targetAccount); err != nil {
+		return nil, fmt.Errorf("error fetching target account %s: %s", targetStatus.AccountID, err)
+	}
+
+	l.Trace("going to get relevant accounts")
+	relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(targetStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching related accounts for status %s: %s", targetStatusID, err)
+	}
+
+	l.Trace("going to see if status is visible")
+	visible, err := p.db.StatusVisible(targetStatus, targetAccount, authed.Account, relevantAccounts) // requestingAccount might well be nil here, but StatusVisible knows how to take care of that
+	if err != nil {
+		return nil, fmt.Errorf("error seeing if status %s is visible: %s", targetStatus.ID, err)
+	}
+
+	if !visible {
+		return nil, errors.New("status is not visible")
+	}
+
+	// get ALL accounts that faved a status -- doesn't take account of blocks and mutes and stuff
+	favingAccounts, err := p.db.WhoFavedStatus(targetStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error seeing who faved status: %s", err)
+	}
+
+	// filter the list so the user doesn't see accounts they blocked or which blocked them
+	filteredAccounts := []*gtsmodel.Account{}
+	for _, acc := range favingAccounts {
+		blocked, err := p.db.Blocked(authed.Account.ID, acc.ID)
+		if err != nil {
+			return nil, fmt.Errorf("error checking blocks: %s", err)
+		}
+		if !blocked {
+			filteredAccounts = append(filteredAccounts, acc)
+		}
+	}
+
+	// TODO: filter other things here? suspended? muted? silenced?
+
+	// now we can return the masto representation of those accounts
+	mastoAccounts := []*apimodel.Account{}
+	for _, acc := range filteredAccounts {
+		mastoAccount, err := p.tc.AccountToMastoPublic(acc)
+		if err != nil {
+			return nil, fmt.Errorf("error converting account to api model: %s", err)
+		}
+		mastoAccounts = append(mastoAccounts, mastoAccount)
+	}
+
+	return mastoAccounts, nil
+}
+
+func (p *processor) StatusGet(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error) {
+	l := p.log.WithField("func", "StatusGet")
+
+	l.Tracef("going to search for target status %s", targetStatusID)
+	targetStatus := &gtsmodel.Status{}
+	if err := p.db.GetByID(targetStatusID, targetStatus); err != nil {
+		return nil, fmt.Errorf("error fetching status %s: %s", targetStatusID, err)
+	}
+
+	l.Tracef("going to search for target account %s", targetStatus.AccountID)
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetStatus.AccountID, targetAccount); err != nil {
+		return nil, fmt.Errorf("error fetching target account %s: %s", targetStatus.AccountID, err)
+	}
+
+	l.Trace("going to get relevant accounts")
+	relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(targetStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching related accounts for status %s: %s", targetStatusID, err)
+	}
+
+	l.Trace("going to see if status is visible")
+	visible, err := p.db.StatusVisible(targetStatus, targetAccount, authed.Account, relevantAccounts) // requestingAccount might well be nil here, but StatusVisible knows how to take care of that
+	if err != nil {
+		return nil, fmt.Errorf("error seeing if status %s is visible: %s", targetStatus.ID, err)
+	}
+
+	if !visible {
+		return nil, errors.New("status is not visible")
+	}
+
+	var boostOfStatus *gtsmodel.Status
+	if targetStatus.BoostOfID != "" {
+		boostOfStatus = &gtsmodel.Status{}
+		if err := p.db.GetByID(targetStatus.BoostOfID, boostOfStatus); err != nil {
+			return nil, fmt.Errorf("error fetching boosted status %s: %s", targetStatus.BoostOfID, err)
+		}
+	}
+
+	mastoStatus, err := p.tc.StatusToMasto(targetStatus, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostOfStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error converting status %s to frontend representation: %s", targetStatus.ID, err)
+	}
+
+	return mastoStatus, nil
+
+}
+
+func (p *processor) StatusUnfave(authed *oauth.Auth, targetStatusID string) (*apimodel.Status, error) {
+	l := p.log.WithField("func", "StatusUnfave")
+	l.Tracef("going to search for target status %s", targetStatusID)
+	targetStatus := &gtsmodel.Status{}
+	if err := p.db.GetByID(targetStatusID, targetStatus); err != nil {
+		return nil, fmt.Errorf("error fetching status %s: %s", targetStatusID, err)
+	}
+
+	l.Tracef("going to search for target account %s", targetStatus.AccountID)
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetStatus.AccountID, targetAccount); err != nil {
+		return nil, fmt.Errorf("error fetching target account %s: %s", targetStatus.AccountID, err)
+	}
+
+	l.Trace("going to get relevant accounts")
+	relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(targetStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching related accounts for status %s: %s", targetStatusID, err)
+	}
+
+	l.Trace("going to see if status is visible")
+	visible, err := p.db.StatusVisible(targetStatus, targetAccount, authed.Account, relevantAccounts) // requestingAccount might well be nil here, but StatusVisible knows how to take care of that
+	if err != nil {
+		return nil, fmt.Errorf("error seeing if status %s is visible: %s", targetStatus.ID, err)
+	}
+
+	if !visible {
+		return nil, errors.New("status is not visible")
+	}
+
+	// is the status faveable?
+	if targetStatus.VisibilityAdvanced != nil {
+		if !targetStatus.VisibilityAdvanced.Likeable {
+			return nil, errors.New("status is not faveable")
+		}
+	}
+
+	// it's visible! it's faveable! so let's unfave the FUCK out of it
+	_, err = p.db.UnfaveStatus(targetStatus, authed.Account.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error unfaveing status: %s", err)
+	}
+
+	var boostOfStatus *gtsmodel.Status
+	if targetStatus.BoostOfID != "" {
+		boostOfStatus = &gtsmodel.Status{}
+		if err := p.db.GetByID(targetStatus.BoostOfID, boostOfStatus); err != nil {
+			return nil, fmt.Errorf("error fetching boosted status %s: %s", targetStatus.BoostOfID, err)
+		}
+	}
+
+	mastoStatus, err := p.tc.StatusToMasto(targetStatus, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostOfStatus)
+	if err != nil {
+		return nil, fmt.Errorf("error converting status %s to frontend representation: %s", targetStatus.ID, err)
+	}
+
+	return mastoStatus, nil
+}

internal/processing/timeline.go

@@ -0,0 +1,99 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]apimodel.Status, ErrorWithCode) {
+	l := p.log.WithField("func", "HomeTimelineGet")
+
+	statuses, err := p.db.GetHomeTimelineForAccount(authed.Account.ID, maxID, sinceID, minID, limit, local)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	apiStatuses := []apimodel.Status{}
+	for _, s := range statuses {
+		targetAccount := &gtsmodel.Account{}
+		if err := p.db.GetByID(s.AccountID, targetAccount); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				l.Debugf("skipping status %s because account %s can't be found in the db", s.ID, s.AccountID)
+				continue
+			}
+			return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting status author: %s", err))
+		}
+
+		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(s)
+		if err != nil {
+			l.Debugf("skipping status %s because we couldn't pull relevant accounts from the db", s.ID)
+			continue
+		}
+
+		visible, err := p.db.StatusVisible(s, targetAccount, authed.Account, relevantAccounts)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error checking status visibility: %s", err))
+		}
+		if !visible {
+			continue
+		}
+
+		var boostedStatus *gtsmodel.Status
+		if s.BoostOfID != "" {
+			bs := &gtsmodel.Status{}
+			if err := p.db.GetByID(s.BoostOfID, bs); err != nil {
+				if _, ok := err.(db.ErrNoEntries); ok {
+					l.Debugf("skipping status %s because status %s can't be found in the db", s.ID, s.BoostOfID)
+					continue
+				}
+				return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting boosted status: %s", err))
+			}
+			boostedRelevantAccounts, err := p.db.PullRelevantAccountsFromStatus(bs)
+			if err != nil {
+				l.Debugf("skipping status %s because we couldn't pull relevant accounts from the db", s.ID)
+				continue
+			}
+
+			boostedVisible, err := p.db.StatusVisible(bs, relevantAccounts.BoostedAccount, authed.Account, boostedRelevantAccounts)
+			if err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error checking boosted status visibility: %s", err))
+			}
+
+			if boostedVisible {
+				boostedStatus = bs
+			}
+		}
+
+		apiStatus, err := p.tc.StatusToMasto(s, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostedStatus)
+		if err != nil {
+			l.Debugf("skipping status %s because it couldn't be converted to its mastodon representation: %s", s.ID, err)
+			continue
+		}
+
+		apiStatuses = append(apiStatuses, *apiStatus)
+	}
+
+	return apiStatuses, nil
+}

internal/processing/util.go

@@ -0,0 +1,357 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
+	// by default all flags are set to true
+	gtsAdvancedVis := &gtsmodel.VisibilityAdvanced{
+		Federated: true,
+		Boostable: true,
+		Replyable: true,
+		Likeable:  true,
+	}
+
+	var gtsBasicVis gtsmodel.Visibility
+	// Advanced takes priority if it's set.
+	// If it's not set, take whatever masto visibility is set.
+	// If *that's* not set either, then just take the account default.
+	// If that's also not set, take the default for the whole instance.
+	if form.VisibilityAdvanced != nil {
+		gtsBasicVis = gtsmodel.Visibility(*form.VisibilityAdvanced)
+	} else if form.Visibility != "" {
+		gtsBasicVis = p.tc.MastoVisToVis(form.Visibility)
+	} else if accountDefaultVis != "" {
+		gtsBasicVis = accountDefaultVis
+	} else {
+		gtsBasicVis = gtsmodel.VisibilityDefault
+	}
+
+	switch gtsBasicVis {
+	case gtsmodel.VisibilityPublic:
+		// for public, there's no need to change any of the advanced flags from true regardless of what the user filled out
+		break
+	case gtsmodel.VisibilityUnlocked:
+		// for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them
+		if form.Federated != nil {
+			gtsAdvancedVis.Federated = *form.Federated
+		}
+
+		if form.Boostable != nil {
+			gtsAdvancedVis.Boostable = *form.Boostable
+		}
+
+		if form.Replyable != nil {
+			gtsAdvancedVis.Replyable = *form.Replyable
+		}
+
+		if form.Likeable != nil {
+			gtsAdvancedVis.Likeable = *form.Likeable
+		}
+
+	case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly:
+		// for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them
+		gtsAdvancedVis.Boostable = false
+
+		if form.Federated != nil {
+			gtsAdvancedVis.Federated = *form.Federated
+		}
+
+		if form.Replyable != nil {
+			gtsAdvancedVis.Replyable = *form.Replyable
+		}
+
+		if form.Likeable != nil {
+			gtsAdvancedVis.Likeable = *form.Likeable
+		}
+
+	case gtsmodel.VisibilityDirect:
+		// direct is pretty easy: there's only one possible setting so return it
+		gtsAdvancedVis.Federated = true
+		gtsAdvancedVis.Boostable = false
+		gtsAdvancedVis.Federated = true
+		gtsAdvancedVis.Likeable = true
+	}
+
+	status.Visibility = gtsBasicVis
+	status.VisibilityAdvanced = gtsAdvancedVis
+	return nil
+}
+
+func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
+	if form.InReplyToID == "" {
+		return nil
+	}
+
+	// If this status is a reply to another status, we need to do a bit of work to establish whether or not this status can be posted:
+	//
+	// 1. Does the replied status exist in the database?
+	// 2. Is the replied status marked as replyable?
+	// 3. Does a block exist between either the current account or the account that posted the status it's replying to?
+	//
+	// If this is all OK, then we fetch the repliedStatus and the repliedAccount for later processing.
+	repliedStatus := &gtsmodel.Status{}
+	repliedAccount := &gtsmodel.Account{}
+	// check replied status exists + is replyable
+	if err := p.db.GetByID(form.InReplyToID, repliedStatus); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return fmt.Errorf("status with id %s not replyable because it doesn't exist", form.InReplyToID)
+		}
+		return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
+	}
+
+	if repliedStatus.VisibilityAdvanced != nil {
+		if !repliedStatus.VisibilityAdvanced.Replyable {
+			return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID)
+		}
+	}
+
+	// check replied account is known to us
+	if err := p.db.GetByID(repliedStatus.AccountID, repliedAccount); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return fmt.Errorf("status with id %s not replyable because account id %s is not known", form.InReplyToID, repliedStatus.AccountID)
+		}
+		return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
+	}
+	// check if a block exists
+	if blocked, err := p.db.Blocked(thisAccountID, repliedAccount.ID); err != nil {
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
+		}
+	} else if blocked {
+		return fmt.Errorf("status with id %s not replyable", form.InReplyToID)
+	}
+	status.InReplyToID = repliedStatus.ID
+	status.InReplyToAccountID = repliedAccount.ID
+
+	return nil
+}
+
+func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
+	if form.MediaIDs == nil {
+		return nil
+	}
+
+	gtsMediaAttachments := []*gtsmodel.MediaAttachment{}
+	attachments := []string{}
+	for _, mediaID := range form.MediaIDs {
+		// check these attachments exist
+		a := &gtsmodel.MediaAttachment{}
+		if err := p.db.GetByID(mediaID, a); err != nil {
+			return fmt.Errorf("invalid media type or media not found for media id %s", mediaID)
+		}
+		// check they belong to the requesting account id
+		if a.AccountID != thisAccountID {
+			return fmt.Errorf("media with id %s does not belong to account %s", mediaID, thisAccountID)
+		}
+		// check they're not already used in a status
+		if a.StatusID != "" || a.ScheduledStatusID != "" {
+			return fmt.Errorf("media with id %s is already attached to a status", mediaID)
+		}
+		gtsMediaAttachments = append(gtsMediaAttachments, a)
+		attachments = append(attachments, a.ID)
+	}
+	status.GTSMediaAttachments = gtsMediaAttachments
+	status.Attachments = attachments
+	return nil
+}
+
+func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
+	if form.Language != "" {
+		status.Language = form.Language
+	} else {
+		status.Language = accountDefaultLanguage
+	}
+	if status.Language == "" {
+		return errors.New("no language given either in status create form or account default")
+	}
+	return nil
+}
+
+func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+	menchies := []string{}
+	gtsMenchies, err := p.db.MentionStringsToMentions(util.DeriveMentionsFromStatus(form.Status), accountID, status.ID)
+	if err != nil {
+		return fmt.Errorf("error generating mentions from status: %s", err)
+	}
+	for _, menchie := range gtsMenchies {
+		if err := p.db.Put(menchie); err != nil {
+			return fmt.Errorf("error putting mentions in db: %s", err)
+		}
+		menchies = append(menchies, menchie.ID)
+	}
+	// add full populated gts menchies to the status for passing them around conveniently
+	status.GTSMentions = gtsMenchies
+	// add just the ids of the mentioned accounts to the status for putting in the db
+	status.Mentions = menchies
+	return nil
+}
+
+func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+	tags := []string{}
+	gtsTags, err := p.db.TagStringsToTags(util.DeriveHashtagsFromStatus(form.Status), accountID, status.ID)
+	if err != nil {
+		return fmt.Errorf("error generating hashtags from status: %s", err)
+	}
+	for _, tag := range gtsTags {
+		if err := p.db.Upsert(tag, "name"); err != nil {
+			return fmt.Errorf("error putting tags in db: %s", err)
+		}
+		tags = append(tags, tag.ID)
+	}
+	// add full populated gts tags to the status for passing them around conveniently
+	status.GTSTags = gtsTags
+	// add just the ids of the used tags to the status for putting in the db
+	status.Tags = tags
+	return nil
+}
+
+func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+	emojis := []string{}
+	gtsEmojis, err := p.db.EmojiStringsToEmojis(util.DeriveEmojisFromStatus(form.Status), accountID, status.ID)
+	if err != nil {
+		return fmt.Errorf("error generating emojis from status: %s", err)
+	}
+	for _, e := range gtsEmojis {
+		emojis = append(emojis, e.ID)
+	}
+	// add full populated gts emojis to the status for passing them around conveniently
+	status.GTSEmojis = gtsEmojis
+	// add just the ids of the used emojis to the status for putting in the db
+	status.Emojis = emojis
+	return nil
+}
+
+/*
+	HELPER FUNCTIONS
+*/
+
+// TODO: try to combine the below two functions because this is a lot of code repetition.
+
+// updateAccountAvatar does the dirty work of checking the avatar part of an account update form,
+// parsing and checking the image, and doing the necessary updates in the database for this to become
+// the account's new avatar image.
+func (p *processor) updateAccountAvatar(avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
+	var err error
+	if int(avatar.Size) > p.config.MediaConfig.MaxImageSize {
+		err = fmt.Errorf("avatar with size %d exceeded max image size of %d bytes", avatar.Size, p.config.MediaConfig.MaxImageSize)
+		return nil, err
+	}
+	f, err := avatar.Open()
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided avatar: %s", err)
+	}
+
+	// extract the bytes
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided avatar: %s", err)
+	}
+	if size == 0 {
+		return nil, errors.New("could not read provided avatar: size 0 bytes")
+	}
+
+	// do the setting
+	avatarInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(buf.Bytes(), accountID, media.Avatar, "")
+	if err != nil {
+		return nil, fmt.Errorf("error processing avatar: %s", err)
+	}
+
+	return avatarInfo, f.Close()
+}
+
+// updateAccountHeader does the dirty work of checking the header part of an account update form,
+// parsing and checking the image, and doing the necessary updates in the database for this to become
+// the account's new header image.
+func (p *processor) updateAccountHeader(header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
+	var err error
+	if int(header.Size) > p.config.MediaConfig.MaxImageSize {
+		err = fmt.Errorf("header with size %d exceeded max image size of %d bytes", header.Size, p.config.MediaConfig.MaxImageSize)
+		return nil, err
+	}
+	f, err := header.Open()
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided header: %s", err)
+	}
+
+	// extract the bytes
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided header: %s", err)
+	}
+	if size == 0 {
+		return nil, errors.New("could not read provided header: size 0 bytes")
+	}
+
+	// do the setting
+	headerInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(buf.Bytes(), accountID, media.Header, "")
+	if err != nil {
+		return nil, fmt.Errorf("error processing header: %s", err)
+	}
+
+	return headerInfo, f.Close()
+}
+
+// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
+// on behalf of requestingUsername.
+//
+// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
+//
+// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
+// to reflect the creation of these new attachments.
+func (p *processor) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
+		a, err := p.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.AvatarRemoteURL,
+			Avatar:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing avatar for user: %s", err)
+		}
+		targetAccount.AvatarMediaAttachmentID = a.ID
+	}
+
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
+		a, err := p.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.HeaderRemoteURL,
+			Header:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing header for user: %s", err)
+		}
+		targetAccount.HeaderMediaAttachmentID = a.ID
+	}
+	return nil
+}