[chore] media pipeline improvements (#3110)

* don't set emoji / media image paths on failed download, migrate FileType from string to integer

* fix incorrect uses of util.PtrOr, fix returned frontend media

* fix migration not setting arguments correctly in where clause

* fix not providing default with not null column

* whoops

* ensure a default gets set for media attachment file type

* remove the exclusive flag from writing files in disk storage

* rename PtrOr -> PtrOrZero, and rename PtrValueOr -> PtrOrValue to match

* slight wording changes

* use singular / plural word forms (no parentheses), is better for screen readers

* update testmodels with unknown media type to have unset file details, update attachment focus handling converting to frontend, update tests

* store first instance in ffmpeg wasm pool, fill remaining with closed instances
This commit is contained in:
kim
2024-07-17 15:26:33 +00:00
committed by GitHub
parent 0aadc2db2a
commit 72ba5666a6
29 changed files with 669 additions and 399 deletions

View File

@@ -21,8 +21,6 @@ import (
"context"
"errors"
"fmt"
"math"
"strconv"
"strings"
"time"
@@ -321,9 +319,9 @@ func (c *Converter) accountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A
}
var (
locked = util.PtrValueOr(a.Locked, true)
discoverable = util.PtrValueOr(a.Discoverable, false)
bot = util.PtrValueOr(a.Bot, false)
locked = util.PtrOrValue(a.Locked, true)
discoverable = util.PtrOrValue(a.Discoverable, false)
bot = util.PtrOrValue(a.Bot, false)
)
// Remaining properties are simple and
@@ -565,84 +563,59 @@ func (c *Converter) AppToAPIAppPublic(ctx context.Context, a *gtsmodel.Applicati
}
// AttachmentToAPIAttachment converts a gts model media attacahment into its api representation for serialization on the API.
func (c *Converter) AttachmentToAPIAttachment(ctx context.Context, a *gtsmodel.MediaAttachment) (apimodel.Attachment, error) {
apiAttachment := apimodel.Attachment{
ID: a.ID,
Type: strings.ToLower(string(a.Type)),
}
func (c *Converter) AttachmentToAPIAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) (apimodel.Attachment, error) {
var api apimodel.Attachment
api.Type = media.Type.String()
api.ID = media.ID
// Don't try to serialize meta for
// unknown attachments, there's no point.
if a.Type != gtsmodel.FileTypeUnknown {
apiAttachment.Meta = &apimodel.MediaMeta{
Original: apimodel.MediaDimensions{
Width: a.FileMeta.Original.Width,
Height: a.FileMeta.Original.Height,
},
Small: apimodel.MediaDimensions{
Width: a.FileMeta.Small.Width,
Height: a.FileMeta.Small.Height,
Size: strconv.Itoa(a.FileMeta.Small.Width) + "x" + strconv.Itoa(a.FileMeta.Small.Height),
Aspect: float32(a.FileMeta.Small.Aspect),
},
// Only add file details if
// we have stored locally.
if media.File.Path != "" {
api.Meta = new(apimodel.MediaMeta)
api.Meta.Original = apimodel.MediaDimensions{
Width: media.FileMeta.Original.Width,
Height: media.FileMeta.Original.Height,
Aspect: media.FileMeta.Original.Aspect,
Size: toAPISize(media.FileMeta.Original.Width, media.FileMeta.Original.Height),
FrameRate: toAPIFrameRate(media.FileMeta.Original.Framerate),
Duration: util.PtrOrZero(media.FileMeta.Original.Duration),
Bitrate: int(util.PtrOrZero(media.FileMeta.Original.Bitrate)),
}
// Copy over local file URL.
api.URL = util.Ptr(media.URL)
api.TextURL = util.Ptr(media.URL)
// Set file focus details.
// (this doesn't make much sense if media
// has no image, but the API doesn't yet
// distinguish between zero values vs. none).
api.Meta.Focus = new(apimodel.MediaFocus)
api.Meta.Focus.X = media.FileMeta.Focus.X
api.Meta.Focus.Y = media.FileMeta.Focus.Y
// Only add thumbnail details if
// we have thumbnail stored locally.
if media.Thumbnail.Path != "" {
api.Meta.Small = apimodel.MediaDimensions{
Width: media.FileMeta.Small.Width,
Height: media.FileMeta.Small.Height,
Aspect: media.FileMeta.Small.Aspect,
Size: toAPISize(media.FileMeta.Small.Width, media.FileMeta.Small.Height),
}
// Copy over local thumbnail file URL.
api.PreviewURL = util.Ptr(media.Thumbnail.URL)
}
}
if i := a.Blurhash; i != "" {
apiAttachment.Blurhash = &i
}
// Set remaining API attachment fields.
api.Blurhash = util.PtrIf(media.Blurhash)
api.RemoteURL = util.PtrIf(media.RemoteURL)
api.PreviewRemoteURL = util.PtrIf(media.Thumbnail.RemoteURL)
api.Description = util.PtrIf(media.Description)
if i := a.URL; i != "" {
apiAttachment.URL = &i
apiAttachment.TextURL = &i
}
if i := a.Thumbnail.URL; i != "" {
apiAttachment.PreviewURL = &i
}
if i := a.RemoteURL; i != "" {
apiAttachment.RemoteURL = &i
}
if i := a.Thumbnail.RemoteURL; i != "" {
apiAttachment.PreviewRemoteURL = &i
}
if i := a.Description; i != "" {
apiAttachment.Description = &i
}
// Type-specific fields.
switch a.Type {
case gtsmodel.FileTypeImage:
apiAttachment.Meta.Original.Size = strconv.Itoa(a.FileMeta.Original.Width) + "x" + strconv.Itoa(a.FileMeta.Original.Height)
apiAttachment.Meta.Original.Aspect = float32(a.FileMeta.Original.Aspect)
apiAttachment.Meta.Focus = &apimodel.MediaFocus{
X: a.FileMeta.Focus.X,
Y: a.FileMeta.Focus.Y,
}
case gtsmodel.FileTypeVideo, gtsmodel.FileTypeAudio:
if i := a.FileMeta.Original.Duration; i != nil {
apiAttachment.Meta.Original.Duration = *i
}
if i := a.FileMeta.Original.Framerate; i != nil {
// The masto api expects this as a string in
// the format `integer/1`, so 30fps is `30/1`.
round := math.Round(float64(*i))
fr := strconv.Itoa(int(round))
apiAttachment.Meta.Original.FrameRate = fr + "/1"
}
if i := a.FileMeta.Original.Bitrate; i != nil {
apiAttachment.Meta.Original.Bitrate = int(*i)
}
}
return apiAttachment, nil
return api, nil
}
// MentionToAPIMention converts a gts model mention into its api (frontend) representation for serialization on the API.
@@ -681,6 +654,7 @@ func (c *Converter) MentionToAPIMention(ctx context.Context, m *gtsmodel.Mention
// EmojiToAPIEmoji converts a gts model emoji into its api (frontend) representation for serialization on the API.
func (c *Converter) EmojiToAPIEmoji(ctx context.Context, e *gtsmodel.Emoji) (apimodel.Emoji, error) {
var category string
if e.CategoryID != "" {
if e.Category == nil {
var err error
@@ -778,14 +752,15 @@ func (c *Converter) StatusToAPIStatus(
return nil, err
}
// Normalize status for the API by pruning
// out unknown attachment types and replacing
// them with a helpful message.
// Normalize status for API by pruning
// attachments that were not locally
// stored, replacing them with a helpful
// message + links to remote.
var aside string
aside, apiStatus.MediaAttachments = placeholdUnknownAttachments(apiStatus.MediaAttachments)
aside, apiStatus.MediaAttachments = placeholderAttachments(apiStatus.MediaAttachments)
apiStatus.Content += aside
if apiStatus.Reblog != nil {
aside, apiStatus.Reblog.MediaAttachments = placeholdUnknownAttachments(apiStatus.Reblog.MediaAttachments)
aside, apiStatus.Reblog.MediaAttachments = placeholderAttachments(apiStatus.Reblog.MediaAttachments)
apiStatus.Reblog.Content += aside
}
@@ -962,15 +937,15 @@ func filterableTextFields(s *gtsmodel.Status) []string {
func filterAppliesInContext(filter *gtsmodel.Filter, filterContext statusfilter.FilterContext) bool {
switch filterContext {
case statusfilter.FilterContextHome:
return util.PtrValueOr(filter.ContextHome, false)
return util.PtrOrValue(filter.ContextHome, false)
case statusfilter.FilterContextNotifications:
return util.PtrValueOr(filter.ContextNotifications, false)
return util.PtrOrValue(filter.ContextNotifications, false)
case statusfilter.FilterContextPublic:
return util.PtrValueOr(filter.ContextPublic, false)
return util.PtrOrValue(filter.ContextPublic, false)
case statusfilter.FilterContextThread:
return util.PtrValueOr(filter.ContextThread, false)
return util.PtrOrValue(filter.ContextThread, false)
case statusfilter.FilterContextAccount:
return util.PtrValueOr(filter.ContextAccount, false)
return util.PtrOrValue(filter.ContextAccount, false)
}
return false
}
@@ -2083,7 +2058,7 @@ func (c *Converter) FilterKeywordToAPIFilterV1(ctx context.Context, filterKeywor
ID: filterKeyword.ID,
Phrase: filterKeyword.Keyword,
Context: filterToAPIFilterContexts(filter),
WholeWord: util.PtrValueOr(filterKeyword.WholeWord, false),
WholeWord: util.PtrOrValue(filterKeyword.WholeWord, false),
ExpiresAt: filterExpiresAtToAPIFilterExpiresAt(filter.ExpiresAt),
Irreversible: filter.Action == gtsmodel.FilterActionHide,
}, nil
@@ -2121,19 +2096,19 @@ func filterExpiresAtToAPIFilterExpiresAt(expiresAt time.Time) *string {
func filterToAPIFilterContexts(filter *gtsmodel.Filter) []apimodel.FilterContext {
apiContexts := make([]apimodel.FilterContext, 0, apimodel.FilterContextNumValues)
if util.PtrValueOr(filter.ContextHome, false) {
if util.PtrOrValue(filter.ContextHome, false) {
apiContexts = append(apiContexts, apimodel.FilterContextHome)
}
if util.PtrValueOr(filter.ContextNotifications, false) {
if util.PtrOrValue(filter.ContextNotifications, false) {
apiContexts = append(apiContexts, apimodel.FilterContextNotifications)
}
if util.PtrValueOr(filter.ContextPublic, false) {
if util.PtrOrValue(filter.ContextPublic, false) {
apiContexts = append(apiContexts, apimodel.FilterContextPublic)
}
if util.PtrValueOr(filter.ContextThread, false) {
if util.PtrOrValue(filter.ContextThread, false) {
apiContexts = append(apiContexts, apimodel.FilterContextThread)
}
if util.PtrValueOr(filter.ContextAccount, false) {
if util.PtrOrValue(filter.ContextAccount, false) {
apiContexts = append(apiContexts, apimodel.FilterContextAccount)
}
return apiContexts
@@ -2154,7 +2129,7 @@ func (c *Converter) FilterKeywordToAPIFilterKeyword(ctx context.Context, filterK
return &apimodel.FilterKeyword{
ID: filterKeyword.ID,
Keyword: filterKeyword.Keyword,
WholeWord: util.PtrValueOr(filterKeyword.WholeWord, false),
WholeWord: util.PtrOrValue(filterKeyword.WholeWord, false),
}
}

View File

@@ -851,7 +851,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments
"muted": false,
"bookmarked": false,
"pinned": false,
"content": "\u003cp\u003ehi \u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003eadmin\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e here's some media for ya\u003c/p\u003e\u003chr\u003e\u003cp\u003e\u003ci lang=\"en\"\u003e Note from localhost:8080: 2 attachments in this status could not be downloaded. Treat the following external links with care:\u003c/i\u003e\u003c/p\u003e\u003cul\u003e\u003cli\u003e\u003ca href=\"http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE7ZGJYTSYMXF927GF9353KR.svg\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e01HE7ZGJYTSYMXF927GF9353KR.svg\u003c/a\u003e [SVG line art of a sloth, public domain]\u003c/li\u003e\u003cli\u003e\u003ca href=\"http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE892Y8ZS68TQCNPX7J888P3.mp3\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e01HE892Y8ZS68TQCNPX7J888P3.mp3\u003c/a\u003e [Jolly salsa song, public domain.]\u003c/li\u003e\u003c/ul\u003e",
"content": "\u003cp\u003ehi \u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003eadmin\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e here's some media for ya\u003c/p\u003e\u003chr\u003e\u003chr\u003e\u003cp\u003e\u003ci lang=\"en\"\u003e Note from localhost:8080: 2 attachments in this status were not downloaded. Treat the following external links with care:\u003c/i\u003e\u003c/p\u003e\u003cul\u003e\u003cli\u003e\u003ca href=\"http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE7ZGJYTSYMXF927GF9353KR.svg\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e01HE7ZGJYTSYMXF927GF9353KR.svg\u003c/a\u003e [SVG line art of a sloth, public domain]\u003c/li\u003e\u003cli\u003e\u003ca href=\"http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE892Y8ZS68TQCNPX7J888P3.mp3\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e01HE892Y8ZS68TQCNPX7J888P3.mp3\u003c/a\u003e [Jolly salsa song, public domain.]\u003c/li\u003e\u003c/ul\u003e",
"reblog": null,
"account": {
"id": "01FHMQX3GAABWSM0S2VZEC2SWC",
@@ -1070,30 +1070,30 @@ func (suite *InternalToFrontendTestSuite) TestStatusToWebStatus() {
{
"id": "01HE7ZFX9GKA5ZZVD4FACABSS9",
"type": "unknown",
"url": "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE7ZFX9GKA5ZZVD4FACABSS9.svg",
"text_url": "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE7ZFX9GKA5ZZVD4FACABSS9.svg",
"preview_url": "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE7ZFX9GKA5ZZVD4FACABSS9.jpg",
"url": null,
"text_url": null,
"preview_url": null,
"remote_url": "http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE7ZGJYTSYMXF927GF9353KR.svg",
"preview_remote_url": null,
"meta": null,
"description": "SVG line art of a sloth, public domain",
"blurhash": "L26*j+~qE1RP?wxut7ofRlM{R*of",
"Sensitive": true,
"MIMEType": "image/svg"
"MIMEType": ""
},
{
"id": "01HE88YG74PVAB81PX2XA9F3FG",
"type": "unknown",
"url": "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE88YG74PVAB81PX2XA9F3FG.mp3",
"text_url": "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE88YG74PVAB81PX2XA9F3FG.mp3",
"preview_url": "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE88YG74PVAB81PX2XA9F3FG.jpg",
"url": null,
"text_url": null,
"preview_url": null,
"remote_url": "http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE892Y8ZS68TQCNPX7J888P3.mp3",
"preview_remote_url": null,
"meta": null,
"description": "Jolly salsa song, public domain.",
"blurhash": null,
"Sensitive": true,
"MIMEType": "audio/mpeg"
"MIMEType": ""
}
],
"LanguageTag": "en",
@@ -1357,13 +1357,19 @@ func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() {
"height": 404,
"frame_rate": "30/1",
"duration": 15.033334,
"bitrate": 1206522
"bitrate": 1206522,
"size": "720x404",
"aspect": 1.7821782
},
"small": {
"width": 720,
"height": 404,
"size": "720x404",
"aspect": 1.7821782
},
"focus": {
"x": 0,
"y": 0
}
},
"description": "A cow adorably licking another cow!",

View File

@@ -20,6 +20,7 @@ package typeutils
import (
"context"
"fmt"
"math"
"net/url"
"path"
"slices"
@@ -35,6 +36,26 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/text"
)
// toAPISize converts a set of media dimensions
// to mastodon API compatible size string.
func toAPISize(width, height int) string {
return strconv.Itoa(width) +
"x" +
strconv.Itoa(height)
}
// toAPIFrameRate converts a media framerate ptr
// to mastodon API compatible framerate string.
func toAPIFrameRate(framerate *float32) string {
if framerate == nil {
return ""
}
// The masto api expects this as a string in
// the format `integer/1`, so 30fps is `30/1`.
round := math.Round(float64(*framerate))
return strconv.Itoa(int(round)) + "/1"
}
type statusInteractions struct {
Favourited bool
Muted bool
@@ -92,7 +113,7 @@ func misskeyReportInlineURLs(content string) []*url.URL {
return urls
}
// placeholdUnknownAttachments separates any attachments with type `unknown`
// placeholderAttachments separates any attachments with missing local URL
// out of the given slice, and returns a piece of text containing links to
// those attachments, as well as the slice of remaining "known" attachments.
// If there are no unknown-type attachments in the provided slice, an empty
@@ -104,62 +125,61 @@ func misskeyReportInlineURLs(content string) []*url.URL {
// Example:
//
// <hr>
// <p><i lang="en"> Note from your.instance.com: 2 attachments in this status could not be downloaded. Treat the following external links with care:</i></p>
// <p><i lang="en"> Note from your.instance.com: 2 attachment(s) in this status were not downloaded. Treat the following external link(s) with care:</i></p>
// <ul>
// <li><a href="http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE7ZGJYTSYMXF927GF9353KR.svg" rel="nofollow noreferrer noopener" target="_blank">01HE7ZGJYTSYMXF927GF9353KR.svg</a> [SVG line art of a sloth, public domain]</li>
// <li><a href="http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE892Y8ZS68TQCNPX7J888P3.mp3" rel="nofollow noreferrer noopener" target="_blank">01HE892Y8ZS68TQCNPX7J888P3.mp3</a> [Jolly salsa song, public domain.]</li>
// </ul>
func placeholdUnknownAttachments(arr []*apimodel.Attachment) (string, []*apimodel.Attachment) {
// Extract unknown-type attachments into a separate
// slice, deleting them from arr in the process.
var unknowns []*apimodel.Attachment
arr = slices.DeleteFunc(arr, func(elem *apimodel.Attachment) bool {
unknown := elem.Type == "unknown"
if unknown {
// Set aside unknown-type attachment.
unknowns = append(unknowns, elem)
}
func placeholderAttachments(arr []*apimodel.Attachment) (string, []*apimodel.Attachment) {
return unknown
// Extract non-locally stored attachments into a
// separate slice, deleting them from input slice.
var nonLocal []*apimodel.Attachment
arr = slices.DeleteFunc(arr, func(elem *apimodel.Attachment) bool {
if elem.URL == nil {
nonLocal = append(nonLocal, elem)
return true
}
return false
})
unknownsLen := len(unknowns)
if unknownsLen == 0 {
// No unknown attachments,
// nothing to do.
if len(nonLocal) == 0 {
// No non-locally
// stored media.
return "", arr
}
// Plural / singular.
var (
attachments string
links string
)
if unknownsLen == 1 {
attachments = "1 attachment"
links = "link"
} else {
attachments = strconv.Itoa(unknownsLen) + " attachments"
links = "links"
}
var note strings.Builder
note.WriteString(`<hr>`)
note.WriteString(`<p><i lang="en">`)
note.WriteString(` Note from ` + config.GetHost() + `: ` + attachments + ` in this status could not be downloaded. Treat the following external ` + links + ` with care:`)
note.WriteString(`</i></p>`)
note.WriteString(`<ul>`)
for _, a := range unknowns {
var (
remoteURL = *a.RemoteURL
base = path.Base(remoteURL)
entry = fmt.Sprintf(`<a href="%s">%s</a>`, remoteURL, base)
)
note.WriteString(`<hr><p><i lang="en"> Note from `)
note.WriteString(config.GetHost())
note.WriteString(`: `)
note.WriteString(strconv.Itoa(len(nonLocal)))
if len(nonLocal) > 1 {
// Use plural word form.
note.WriteString(` attachments in this status were not downloaded. ` +
`Treat the following external links with care:`)
} else {
// Use singular word form.
note.WriteString(` attachment in this status was not downloaded. ` +
`Treat the following external link with care:`)
}
note.WriteString(`</i></p><ul>`)
for _, a := range nonLocal {
note.WriteString(`<li>`)
note.WriteString(`<a href="`)
note.WriteString(*a.RemoteURL)
note.WriteString(`">`)
note.WriteString(path.Base(*a.RemoteURL))
note.WriteString(`</a>`)
if d := a.Description; d != nil && *d != "" {
entry += ` [` + *d + `]`
note.WriteString(` [`)
note.WriteString(*d)
note.WriteString(`]`)
}
note.WriteString(`<li>` + entry + `</li>`)
note.WriteString(`</li>`)
}
note.WriteString(`</ul>`)