[feature] Start implementing refetch of lost media files via /api/v1/admin/media_refetch (#1221)

* [chore] Move ShortcodeDomain to its own little util func

* [feature] Add RefetchEmojis function to media manager

* [feature] Expose admin media refresh via admin API

* update following review feedback

- change/fix log levels
- make sure not to try to refetch local emojis
- small style refactoring + comments

* log on emoji refetch start

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: kim <grufwub@gmail.com>

internal/processing/admin.go

@@ -77,3 +77,7 @@ func (p *processor) AdminDomainBlockDelete(ctx context.Context, authed *oauth.Au
 func (p *processor) AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode {
 	return p.adminProcessor.MediaPrune(ctx, mediaRemoteCacheDays)
 }
+
+func (p *processor) AdminMediaRefetch(ctx context.Context, authed *oauth.Auth, domain string) gtserror.WithCode {
+	return p.adminProcessor.MediaRefetch(ctx, authed.Account, domain)
+}

internal/processing/admin/admin.go

@@ -30,6 +30,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -48,23 +49,26 @@ type Processor interface {
 	EmojiUpdate(ctx context.Context, id string, form *apimodel.EmojiUpdateRequest) (*apimodel.AdminEmoji, gtserror.WithCode)
 	EmojiCategoriesGet(ctx context.Context) ([]*apimodel.EmojiCategory, gtserror.WithCode)
 	MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode
+	MediaRefetch(ctx context.Context, requestingAccount *gtsmodel.Account, domain string) gtserror.WithCode
 }
 
 type processor struct {
-	tc           typeutils.TypeConverter
-	mediaManager media.Manager
-	storage      *storage.Driver
-	clientWorker *concurrency.WorkerPool[messages.FromClientAPI]
-	db           db.DB
+	tc                  typeutils.TypeConverter
+	mediaManager        media.Manager
+	transportController transport.Controller
+	storage             *storage.Driver
+	clientWorker        *concurrency.WorkerPool[messages.FromClientAPI]
+	db                  db.DB
 }
 
 // New returns a new admin processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *storage.Driver, clientWorker *concurrency.WorkerPool[messages.FromClientAPI]) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *storage.Driver, clientWorker *concurrency.WorkerPool[messages.FromClientAPI]) Processor {
 	return &processor{
-		tc:           tc,
-		mediaManager: mediaManager,
-		storage:      storage,
-		clientWorker: clientWorker,
-		db:           db,
+		tc:                  tc,
+		mediaManager:        mediaManager,
+		transportController: transportController,
+		storage:             storage,
+		clientWorker:        clientWorker,
+		db:                  db,
 	}
 }

internal/processing/admin/getemojis.go

@@ -88,14 +88,10 @@ func (p *processor) EmojisGet(ctx context.Context, account *gtsmodel.Account, us
 		Items:            items,
 		Path:             "api/v1/admin/custom_emojis",
 		NextMaxIDKey:     "max_shortcode_domain",
-		NextMaxIDValue:   shortcodeDomain(emojis[count-1]),
+		NextMaxIDValue:   util.ShortcodeDomain(emojis[count-1]),
 		PrevMinIDKey:     "min_shortcode_domain",
-		PrevMinIDValue:   shortcodeDomain(emojis[0]),
+		PrevMinIDValue:   util.ShortcodeDomain(emojis[0]),
 		Limit:            limit,
 		ExtraQueryParams: []string{filterBuilder.String()},
 	})
 }
-
-func shortcodeDomain(emoji *gtsmodel.Emoji) string {
-	return emoji.Shortcode + "@" + emoji.Domain
-}

internal/processing/admin/mediarefetch.go

@@ -0,0 +1,48 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+)
+
+func (p *processor) MediaRefetch(ctx context.Context, requestingAccount *gtsmodel.Account, domain string) gtserror.WithCode {
+	transport, err := p.transportController.NewTransportForUsername(ctx, requestingAccount.Username)
+	if err != nil {
+		err = fmt.Errorf("error getting transport for user %s during media refetch request: %w", requestingAccount.Username, err)
+		return gtserror.NewErrorInternalError(err)
+	}
+
+	go func() {
+		log.Info("starting emoji refetch")
+		refetched, err := p.mediaManager.RefetchEmojis(context.Background(), domain, transport.DereferenceMedia)
+		if err != nil {
+			log.Errorf("error refetching emojis: %s", err)
+		} else {
+			log.Infof("refetched %d emojis from remote", refetched)
+		}
+	}()
+
+	return nil
+}

internal/processing/processor.go

@@ -136,6 +136,8 @@ type Processor interface {
 	AdminDomainBlockDelete(ctx context.Context, authed *oauth.Auth, id string) (*apimodel.DomainBlock, gtserror.WithCode)
 	// AdminMediaRemotePrune triggers a prune of remote media according to the given number of mediaRemoteCacheDays
 	AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode
+	// AdminMediaRefetch triggers a refetch of remote media for the given domain (or all if domain is empty).
+	AdminMediaRefetch(ctx context.Context, authed *oauth.Auth, domain string) gtserror.WithCode
 
 	// AppCreate processes the creation of a new API application
 	AppCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.ApplicationCreateRequest) (*apimodel.Application, gtserror.WithCode)
@@ -318,7 +320,7 @@ func NewProcessor(
 	statusProcessor := status.New(db, tc, clientWorker, parseMentionFunc)
 	streamingProcessor := streaming.New(db, oauthServer)
 	accountProcessor := account.New(db, tc, mediaManager, oauthServer, clientWorker, federator, parseMentionFunc)
-	adminProcessor := admin.New(db, tc, mediaManager, storage, clientWorker)
+	adminProcessor := admin.New(db, tc, mediaManager, federator.TransportController(), storage, clientWorker)
 	mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
 	userProcessor := user.New(db, emailSender)
 	federationProcessor := federationProcessor.New(db, tc, federator)