[feature/performance] support uncaching remote emoji + scheduled cleanup functions (#1987)

internal/processing/admin/media.go

@@ -58,7 +58,7 @@ func (p *Processor) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gt
 	go func() {
 		ctx := context.Background()
 		p.cleaner.Media().All(ctx, mediaRemoteCacheDays)
-		p.cleaner.Emoji().All(ctx)
+		p.cleaner.Emoji().All(ctx, mediaRemoteCacheDays)
 	}()
 
 	return nil

internal/processing/fromclientapi.go

@@ -982,7 +982,7 @@ func (p *Processor) federateReport(ctx context.Context, report *gtsmodel.Report)
 	}
 
 	if len(report.StatusIDs) > 0 && len(report.Statuses) == 0 {
-		statuses, err := p.state.DB.GetStatuses(ctx, report.StatusIDs)
+		statuses, err := p.state.DB.GetStatusesByIDs(ctx, report.StatusIDs)
 		if err != nil {
 			return fmt.Errorf("federateReport: error getting report statuses from database: %w", err)
 		}

internal/processing/media/getfile.go

@@ -118,7 +118,7 @@ func (p *Processor) getAttachmentContent(ctx context.Context, requestingAccount
 	// retrieve attachment from the database and do basic checks on it
 	a, err := p.state.DB.GetAttachmentByID(ctx, wantedMediaID)
 	if err != nil {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %w", wantedMediaID, err))
 	}
 
 	if a.AccountID != owningAccountID {
@@ -131,7 +131,7 @@ func (p *Processor) getAttachmentContent(ctx context.Context, requestingAccount
 		// 2. we need to fetch it again using a transport and the media manager
 		remoteMediaIRI, err := url.Parse(a.RemoteURL)
 		if err != nil {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err))
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %w", a.RemoteURL, err))
 		}
 
 		// use an empty string as requestingUsername to use the instance account, unless the request for this
@@ -151,24 +151,24 @@ func (p *Processor) getAttachmentContent(ctx context.Context, requestingAccount
 		//   recache operation -> holding open a media worker.
 		// ]
 
-		dataFn := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
-			t, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
+		dataFn := func(ctx context.Context) (io.ReadCloser, int64, error) {
+			t, err := p.transportController.NewTransportForUsername(ctx, requestingUsername)
 			if err != nil {
 				return nil, 0, err
 			}
-			return t.DereferenceMedia(gtscontext.SetFastFail(innerCtx), remoteMediaIRI)
+			return t.DereferenceMedia(gtscontext.SetFastFail(ctx), remoteMediaIRI)
 		}
 
 		// Start recaching this media with the prepared data function.
 		processingMedia, err := p.mediaManager.PreProcessMediaRecache(ctx, dataFn, wantedMediaID)
 		if err != nil {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %w", err))
 		}
 
 		// Load attachment and block until complete
 		a, err = processingMedia.LoadAttachment(ctx)
 		if err != nil {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %w", err))
 		}
 	}
 
@@ -205,17 +205,53 @@ func (p *Processor) getEmojiContent(ctx context.Context, fileName string, owning
 	// for using the static URL rather than full size url
 	// is that static emojis are always encoded as png,
 	// so this is more reliable than using full size url
-	imageStaticURL := uris.GenerateURIForAttachment(owningAccountID, string(media.TypeEmoji), string(media.SizeStatic), fileName, "png")
+	imageStaticURL := uris.GenerateURIForAttachment(
+		owningAccountID,
+		string(media.TypeEmoji),
+		string(media.SizeStatic),
+		fileName,
+		"png",
+	)
 
 	e, err := p.state.DB.GetEmojiByStaticURL(ctx, imageStaticURL)
 	if err != nil {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", fileName, err))
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %w", fileName, err))
 	}
 
 	if *e.Disabled {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", fileName))
 	}
 
+	if !*e.Cached {
+		// if we don't have it cached, then we can assume two things:
+		// 1. this is remote emoji, since local emoji should never be uncached
+		// 2. we need to fetch it again using a transport and the media manager
+		remoteURL, err := url.Parse(e.ImageRemoteURL)
+		if err != nil {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote emoji iri %s: %w", e.ImageRemoteURL, err))
+		}
+
+		dataFn := func(ctx context.Context) (io.ReadCloser, int64, error) {
+			t, err := p.transportController.NewTransportForUsername(ctx, "")
+			if err != nil {
+				return nil, 0, err
+			}
+			return t.DereferenceMedia(gtscontext.SetFastFail(ctx), remoteURL)
+		}
+
+		// Start recaching this emoji with the prepared data function.
+		processingEmoji, err := p.mediaManager.PreProcessEmojiRecache(ctx, dataFn, e.ID)
+		if err != nil {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching emoji: %w", err))
+		}
+
+		// Load attachment and block until complete
+		e, err = processingEmoji.LoadEmoji(ctx)
+		if err != nil {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached emoji: %w", err))
+		}
+	}
+
 	switch emojiSize {
 	case media.SizeOriginal:
 		emojiContent.ContentType = e.ImageContentType

internal/processing/report/create.go

@@ -51,7 +51,7 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form
 	}
 
 	// fetch statuses by IDs given in the report form (noop if no statuses given)
-	statuses, err := p.state.DB.GetStatuses(ctx, form.StatusIDs)
+	statuses, err := p.state.DB.GetStatusesByIDs(ctx, form.StatusIDs)
 	if err != nil {
 		err = fmt.Errorf("db error fetching report target statuses: %w", err)
 		return nil, gtserror.NewErrorInternalError(err)