SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

[chore] media and emoji refactoring (#3000)

Browse Source 
* start updating media manager interface ready for storing attachments / emoji right away

* store emoji and media as uncached immediately, then (re-)cache on Processing{}.Load()

* remove now unused media workers

* fix tests and issues

* fix another test!

* fix emoji activitypub uri setting behaviour, fix remainder of test compilation issues

* fix more tests

* fix (most of) remaining tests, add debouncing to repeatedly failing media / emojis

* whoops, rebase issue

* remove kim's whacky experiments

* do some reshuffling, ensure emoji uri gets set

* ensure marked as not cached on cleanup

* tweaks to media / emoji processing to handle context canceled better

* ensure newly fetched emojis actually get set in returned slice

* use different varnames to be a bit more obvious

* move emoji refresh rate limiting to dereferencer

* add exported dereferencer functions for remote media, use these for recaching in processor

* add check for nil attachment in updateAttachment()

* remove unused emoji and media fields + columns

* see previous commit

* fix old migrations expecting image_updated_at to exists (from copies of old models)

* remove freshness checking code (seems to be broken...)

* fix error arg causing nil ptr exception

* finish documentating functions with comments, slight tweaks to media / emoji deref error logic

* remove some extra unneeded boolean checking

* finish writing documentation (code comments) for exported media manager methods

* undo changes to migration snapshot gtsmodels, updated failing migration to have its own snapshot

* move doesColumnExist() to util.go in migrations package
This commit is contained in:
kim
2024-06-26 15:01:16 +00:00
committed by GitHub
parent fa710057c8
commit 21bb324156
48 changed files with 2578 additions and 1926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
internal/federation/dereferencing/util.go
View File

@@ -18,120 +18,36 @@
package dereferencing
import (
"context"
"io"
"net/url"
"slices"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// loadAttachment handles the case of a new media attachment
// that requires loading. it stores and caches from given data.
func (d *Dereferencer) loadAttachment(
ctx context.Context,
tsport transport.Transport,
accountID string, // media account owner
remoteURL string,
info *media.AdditionalMediaInfo,
// getEmojiByShortcodeDomain searches input slice
// for emoji with given shortcode and domain.
func getEmojiByShortcodeDomain(
emojis []*gtsmodel.Emoji,
shortcode string,
domain string,
) (
*gtsmodel.MediaAttachment,
error,
*gtsmodel.Emoji,
bool,
) {
// Parse str as valid URL object.
url, err := url.Parse(remoteURL)
if err != nil {
return nil, gtserror.Newf("invalid remote media url %q: %v", remoteURL, err)
for _, emoji := range emojis {
if emoji.Shortcode == shortcode &&
emoji.Domain == domain {
return emoji, true
}
}
// Start pre-processing remote media at remote URL.
processing := d.mediaManager.PreProcessMedia(
func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, url)
},
accountID,
info,
)
// Force attachment loading *right now*.
return processing.LoadAttachment(ctx)
return nil, false
}
// updateAttachment handles the case of an existing media attachment
// that *may* have changes or need recaching. it checks for changed
// fields, updating in the database if so, and recaches uncached media.
func (d *Dereferencer) updateAttachment(
ctx context.Context,
tsport transport.Transport,
existing *gtsmodel.MediaAttachment, // existing attachment
media *gtsmodel.MediaAttachment, // (optional) changed media
) (
*gtsmodel.MediaAttachment, // always set
error,
) {
if media != nil {
// Possible changed media columns.
changed := make([]string, 0, 3)
// Check if attachment description has changed.
if existing.Description != media.Description {
changed = append(changed, "description")
existing.Description = media.Description
}
// Check if attachment blurhash has changed (i.e. content change).
if existing.Blurhash != media.Blurhash && media.Blurhash != "" {
changed = append(changed, "blurhash", "cached")
existing.Blurhash = media.Blurhash
existing.Cached = util.Ptr(false)
}
if len(changed) > 0 {
// Update the existing attachment model in the database.
err := d.state.DB.UpdateAttachment(ctx, existing, changed...)
if err != nil {
return media, gtserror.Newf("error updating media: %w", err)
}
}
}
// Check if cached.
if *existing.Cached {
return existing, nil
}
// Parse str as valid URL object.
url, err := url.Parse(existing.RemoteURL)
if err != nil {
return nil, gtserror.Newf("invalid remote media url %q: %v", media.RemoteURL, err)
}
// Start pre-processing remote media recaching from remote.
processing, err := d.mediaManager.PreProcessMediaRecache(
ctx,
func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, url)
},
existing.ID,
)
if err != nil {
return nil, gtserror.Newf("error processing recache: %w", err)
}
// Force load attachment recache *right now*.
recached, err := processing.LoadAttachment(ctx)
// Always return the error we
// receive, but ensure we return
// most up-to-date media file.
if recached != nil {
return recached, err
}
return existing, err
// emojiChanged returns whether an emoji has changed in a way
// that indicates that it should be refetched and refreshed.
func emojiChanged(existing, latest *gtsmodel.Emoji) bool {
return existing.URI != latest.URI ||
existing.ImageRemoteURL != latest.ImageRemoteURL ||
existing.ImageStaticRemoteURL != latest.ImageStaticRemoteURL
}
// pollChanged returns whether a poll has changed in way that