[feature] Custom emoji updates (serve emoji via s2s api, tune db models) (#805)

* migrate emojis

* add get emoji to s2s (federation) API

* add new emoji db + cache functions

* add shortcodeDomain lookup for emojis

* check existing emojis w/cache, not w/constraints

* go fmt

* add putEmoji func

* use new db emoji funcs instead of where

* remove emojistringstotags func

* add unique constraint back in

* fix up broken migration

* update index

internal/processing/admin/emoji.go

@@ -20,7 +20,6 @@ package admin
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 
@@ -37,9 +36,13 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
 		return nil, gtserror.NewErrorUnauthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
 	}
 
-	data := func(innerCtx context.Context) (io.Reader, int, error) {
-		f, err := form.Image.Open()
-		return f, int(form.Image.Size), err
+	maybeExisting, err := p.db.GetEmojiByShortcodeDomain(ctx, form.Shortcode, "")
+	if maybeExisting != nil {
+		return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode))
+	}
+
+	if err != nil && err != db.ErrNoEntries {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error checking existence of emoji with shortcode %s: %s", form.Shortcode, err))
 	}
 
 	emojiID, err := id.NewRandomULID()
@@ -49,6 +52,11 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
 
 	emojiURI := uris.GenerateURIForEmoji(emojiID)
 
+	data := func(innerCtx context.Context) (io.Reader, int, error) {
+		f, err := form.Image.Open()
+		return f, int(form.Image.Size), err
+	}
+
 	processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil)
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji")
@@ -56,10 +64,6 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
 
 	emoji, err := processingEmoji.LoadEmoji(ctx)
 	if err != nil {
-		var alreadyExistsError *db.ErrAlreadyExists
-		if errors.As(err, &alreadyExistsError) {
-			return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode))
-		}
 		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error loading emoji: %s", err), "error loading emoji")
 	}
 

internal/processing/federation.go

@@ -51,6 +51,10 @@ func (p *processor) GetFediOutbox(ctx context.Context, requestedUsername string,
 	return p.federationProcessor.GetOutbox(ctx, requestedUsername, page, maxID, minID, requestURL)
 }
 
+func (p *processor) GetFediEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
+	return p.federationProcessor.GetEmoji(ctx, requestedEmojiID, requestURL)
+}
+
 func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string) (*apimodel.WellKnownResponse, gtserror.WithCode) {
 	return p.federationProcessor.GetWebfingerAccount(ctx, requestedUsername)
 }

internal/processing/federation/federation.go

@@ -56,6 +56,9 @@ type Processor interface {
 	// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
 	GetWebfingerAccount(ctx context.Context, requestedUsername string) (*apimodel.WellKnownResponse, gtserror.WithCode)
 
+	// GetFediEmoji handles the GET for a federated emoji originating from this instance.
+	GetEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
+
 	// GetNodeInfoRel returns a well known response giving the path to node info.
 	GetNodeInfoRel(ctx context.Context, request *http.Request) (*apimodel.WellKnownResponse, gtserror.WithCode)
 

internal/processing/federation/getemoji.go

@@ -0,0 +1,59 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package federation
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+)
+
+func (p *processor) GetEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
+	if _, errWithCode := p.federator.AuthenticateFederatedRequest(ctx, ""); errWithCode != nil {
+		return nil, errWithCode
+	}
+
+	requestedEmoji, err := p.db.GetEmojiByID(ctx, requestedEmojiID)
+	if err != nil {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting emoji with id %s: %s", requestedEmojiID, err))
+	}
+
+	if requestedEmoji.Domain != "" {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji with id %s doesn't belong to this instance (domain %s)", requestedEmojiID, requestedEmoji.Domain))
+	}
+
+	if *requestedEmoji.Disabled {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji with id %s has been disabled", requestedEmojiID))
+	}
+
+	apEmoji, err := p.tc.EmojiToAS(ctx, requestedEmoji)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting gtsmodel emoji with id %s to ap emoji: %s", requestedEmojiID, err))
+	}
+
+	data, err := streams.Serialize(apEmoji)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	return data, nil
+}

internal/processing/media/getfile.go

@@ -231,8 +231,8 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e
 	emojiContent := &apimodel.Content{}
 	var storagePath string
 
-	e := &gtsmodel.Emoji{}
-	if err := p.db.GetByID(ctx, wantedEmojiID, e); err != nil {
+	e, err := p.db.GetEmojiByID(ctx, wantedEmojiID)
+	if err != nil {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err))
 	}
 

internal/processing/processor.go

@@ -228,6 +228,8 @@ type Processor interface {
 	GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
 	// GetFediOutbox returns the public outbox of the requested user, with the given parameters.
 	GetFediOutbox(ctx context.Context, requestedUsername string, page bool, maxID string, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
+	// GetFediEmoji returns the AP representation of an emoji on this instance.
+	GetFediEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
 	// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
 	GetWebfingerAccount(ctx context.Context, requestedUsername string) (*apimodel.WellKnownResponse, gtserror.WithCode)
 	// GetNodeInfoRel returns a well known response giving the path to node info.

internal/processing/status/util.go

@@ -249,18 +249,27 @@ func (p *processor) ProcessTags(ctx context.Context, form *apimodel.AdvancedStat
 }
 
 func (p *processor) ProcessEmojis(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
-	gtsEmojis, err := p.db.EmojiStringsToEmojis(ctx, util.DeriveEmojisFromText(form.Status))
-	if err != nil {
-		return fmt.Errorf("error generating emojis from status: %s", err)
+	// for each emoji shortcode in the text, check if it's an enabled
+	// emoji on this instance, and if so, add it to the status
+	emojiShortcodes := util.DeriveEmojisFromText(form.Status)
+	status.Emojis = make([]*gtsmodel.Emoji, 0, len(emojiShortcodes))
+	status.EmojiIDs = make([]string, 0, len(emojiShortcodes))
+
+	for _, shortcode := range emojiShortcodes {
+		emoji, err := p.db.GetEmojiByShortcodeDomain(ctx, shortcode, "")
+		if err != nil {
+			if err != db.ErrNoEntries {
+				log.Errorf("error getting local emoji with shortcode %s: %s", shortcode, err)
+			}
+			continue
+		}
+
+		if *emoji.VisibleInPicker && !*emoji.Disabled {
+			status.Emojis = append(status.Emojis, emoji)
+			status.EmojiIDs = append(status.EmojiIDs, emoji.ID)
+		}
 	}
-	emojis := make([]string, 0, len(gtsEmojis))
-	for _, e := range gtsEmojis {
-		emojis = append(emojis, e.ID)
-	}
-	// add full populated gts emojis to the status for passing them around conveniently
-	status.Emojis = gtsEmojis
-	// add just the ids of the used emojis to the status for putting in the db
-	status.EmojiIDs = emojis
+
 	return nil
 }