[bugfix] take into account rotation when generating thumbnail (#3147)

* take into account rotation when generating thumbnail, simplify ffprobe output to show only fields we need

* only show rotation side data

* remove unnecessary comment

* fix code comments

* remove debug logging
This commit is contained in:
kim 2024-07-28 19:10:41 +00:00 committed by GitHub
parent 58f8082795
commit 368c97f0f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 102 additions and 20 deletions

View File

@ -52,6 +52,8 @@ func ffmpegClearMetadata(ctx context.Context, filepath string) error {
// Clear metadata with ffmpeg. // Clear metadata with ffmpeg.
if err := ffmpeg(ctx, dirpath, if err := ffmpeg(ctx, dirpath,
// Only log errors.
"-loglevel", "error", "-loglevel", "error",
// Input file path. // Input file path.
@ -101,6 +103,8 @@ func ffmpegGenerateThumb(ctx context.Context, filepath string, width, height int
// Generate thumb with ffmpeg. // Generate thumb with ffmpeg.
if err := ffmpeg(ctx, dirpath, if err := ffmpeg(ctx, dirpath,
// Only log errors.
"-loglevel", "error", "-loglevel", "error",
// Input file. // Input file.
@ -158,6 +162,8 @@ func ffmpegGenerateStatic(ctx context.Context, filepath string) (string, error)
// Generate static with ffmpeg. // Generate static with ffmpeg.
if err := ffmpeg(ctx, dirpath, if err := ffmpeg(ctx, dirpath,
// Only log errors.
"-loglevel", "error", "-loglevel", "error",
// Input file. // Input file.
@ -216,12 +222,29 @@ func ffprobe(ctx context.Context, filepath string) (*result, error) {
Stdout: &stdout, Stdout: &stdout,
Args: []string{ Args: []string{
"-i", filepath, // Don't show any excess logging
// information, all goes in JSON.
"-loglevel", "quiet", "-loglevel", "quiet",
// Print in compact JSON format.
"-print_format", "json=compact=1", "-print_format", "json=compact=1",
"-show_streams",
"-show_format", // Show error in our
// chosen format type.
"-show_error", "-show_error",
// Show specifically container format, total duration and bitrate.
"-show_entries", "format=format_name,duration,bit_rate" + ":" +
// Show specifically stream codec names, types, frame rate, duration and dimens.
"stream=codec_name,codec_type,r_frame_rate,duration_ts,width,height" + ":" +
// Show any rotation
// side data stored.
"side_data=rotation",
// Input file.
"-i", filepath,
}, },
Config: func(modcfg wazero.ModuleConfig) wazero.ModuleConfig { Config: func(modcfg wazero.ModuleConfig) wazero.ModuleConfig {
@ -257,8 +280,9 @@ type result struct {
format string format string
audio []audioStream audio []audioStream
video []videoStream video []videoStream
bitrate uint64
duration float64 duration float64
bitrate uint64
rotation int
} }
type stream struct { type stream struct {
@ -456,15 +480,61 @@ func (res *ffprobeResult) Process() (*result, error) {
} }
} }
// Check extra packet / frame information
// for provided orientation (not always set).
for _, pf := range res.PacketsAndFrames {
for _, d := range pf.SideDataList {
// Ensure frame side
// data IS rotation data.
if d.Rotation == 0 {
continue
}
// Ensure rotation not
// already been specified.
if r.rotation != 0 {
return nil, errors.New("multiple sets of rotation data")
}
// Drop any decimal
// rotation value.
rot := int(d.Rotation)
// Round rotation to multiple of 90.
// More granularity is not needed.
if q := rot % 90; q > 45 {
rot += (90 - q)
} else {
rot -= q
}
// Drop any value above 360
// or below -360, these are
// just repeat full turns.
r.rotation = (rot % 360)
}
}
return &r, nil return &r, nil
} }
// ffprobeResult contains parsed JSON data from // ffprobeResult contains parsed JSON data from
// result of calling `ffprobe` on a media file. // result of calling `ffprobe` on a media file.
type ffprobeResult struct { type ffprobeResult struct {
Streams []ffprobeStream `json:"streams"` PacketsAndFrames []ffprobePacketOrFrame `json:"packets_and_frames"`
Format *ffprobeFormat `json:"format"` Streams []ffprobeStream `json:"streams"`
Error *ffprobeError `json:"error"` Format *ffprobeFormat `json:"format"`
Error *ffprobeError `json:"error"`
}
type ffprobePacketOrFrame struct {
Type string `json:"type"`
SideDataList []ffprobeSideData `json:"side_data_list"`
}
type ffprobeSideData struct {
Rotation float64 `json:"rotation"`
} }
type ffprobeStream struct { type ffprobeStream struct {
@ -474,14 +544,12 @@ type ffprobeStream struct {
DurationTS uint `json:"duration_ts"` DurationTS uint `json:"duration_ts"`
Width int `json:"width"` Width int `json:"width"`
Height int `json:"height"` Height int `json:"height"`
// + unused fields.
} }
type ffprobeFormat struct { type ffprobeFormat struct {
FormatName string `json:"format_name"` FormatName string `json:"format_name"`
Duration string `json:"duration"` Duration string `json:"duration"`
BitRate string `json:"bit_rate"` BitRate string `json:"bit_rate"`
// + unused fields
} }
type ffprobeError struct { type ffprobeError struct {

View File

@ -483,7 +483,7 @@ func (suite *ManagerTestSuite) TestLongerMp4Process() {
suite.EqualValues(float32(10), *attachment.FileMeta.Original.Framerate) suite.EqualValues(float32(10), *attachment.FileMeta.Original.Framerate)
suite.EqualValues(0xce3a, *attachment.FileMeta.Original.Bitrate) suite.EqualValues(0xce3a, *attachment.FileMeta.Original.Bitrate)
suite.EqualValues(gtsmodel.Small{ suite.EqualValues(gtsmodel.Small{
Width: 512, Height: 281, Size: 143872, Aspect: 1.822064, Width: 512, Height: 281, Size: 143872, Aspect: 1.8181819,
}, attachment.FileMeta.Small) }, attachment.FileMeta.Small)
suite.Equal("video/mp4", attachment.File.ContentType) suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/webp", attachment.Thumbnail.ContentType) suite.Equal("image/webp", attachment.Thumbnail.ContentType)
@ -543,7 +543,7 @@ func (suite *ManagerTestSuite) TestBirdnestMp4Process() {
suite.EqualValues(float32(30), *attachment.FileMeta.Original.Framerate) suite.EqualValues(float32(30), *attachment.FileMeta.Original.Framerate)
suite.EqualValues(0x11844c, *attachment.FileMeta.Original.Bitrate) suite.EqualValues(0x11844c, *attachment.FileMeta.Original.Bitrate)
suite.EqualValues(gtsmodel.Small{ suite.EqualValues(gtsmodel.Small{
Width: 287, Height: 512, Size: 146944, Aspect: 0.5605469, Width: 287, Height: 512, Size: 146944, Aspect: 0.5611111,
}, attachment.FileMeta.Small) }, attachment.FileMeta.Small)
suite.Equal("video/mp4", attachment.File.ContentType) suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/webp", attachment.Thumbnail.ContentType) suite.Equal("image/webp", attachment.Thumbnail.ContentType)

View File

@ -176,10 +176,11 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
// This will always be used regardless of type, // This will always be used regardless of type,
// as even audio files may contain embedded album art. // as even audio files may contain embedded album art.
width, height, framerate := result.ImageMeta() width, height, framerate := result.ImageMeta()
aspect := util.Div(float32(width), float32(height))
p.media.FileMeta.Original.Width = width p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height p.media.FileMeta.Original.Height = height
p.media.FileMeta.Original.Size = (width * height) p.media.FileMeta.Original.Size = (width * height)
p.media.FileMeta.Original.Aspect = util.Div(float32(width), float32(height)) p.media.FileMeta.Original.Aspect = aspect
p.media.FileMeta.Original.Framerate = util.PtrIf(framerate) p.media.FileMeta.Original.Framerate = util.PtrIf(framerate)
p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration)) p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate) p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
@ -218,11 +219,11 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
if width > 0 && height > 0 { if width > 0 && height > 0 {
// Determine thumbnail dimensions to use. // Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height) thumbWidth, thumbHeight := thumbSize(width, height, aspect, result.rotation)
p.media.FileMeta.Small.Width = thumbWidth p.media.FileMeta.Small.Width = thumbWidth
p.media.FileMeta.Small.Height = thumbHeight p.media.FileMeta.Small.Height = thumbHeight
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight) p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight) p.media.FileMeta.Small.Aspect = aspect
// Generate a thumbnail image from input image path. // Generate a thumbnail image from input image path.
thumbpath, err = ffmpegGenerateThumb(ctx, temppath, thumbpath, err = ffmpegGenerateThumb(ctx, temppath,

View File

@ -37,12 +37,23 @@ import (
// thumbSize returns the dimensions to use for an input // thumbSize returns the dimensions to use for an input
// image of given width / height, for its outgoing thumbnail. // image of given width / height, for its outgoing thumbnail.
// This maintains the original image aspect ratio. // This attempts to maintains the original image aspect ratio.
func thumbSize(width, height int) (int, int) { func thumbSize(width, height int, aspect float32, rotation int) (int, int) {
const ( const (
maxThumbWidth = 512 maxThumbWidth = 512
maxThumbHeight = 512 maxThumbHeight = 512
) )
// If image is rotated by
// any odd multiples of 90,
// flip width / height to
// get the correct scale.
switch rotation {
case -90, 90, -270, 270:
width, height = height, width
aspect = 1 / aspect
}
switch { switch {
// Simplest case, within bounds! // Simplest case, within bounds!
case width < maxThumbWidth && case width < maxThumbWidth &&
@ -51,13 +62,15 @@ func thumbSize(width, height int) (int, int) {
// Width is larger side. // Width is larger side.
case width > height: case width > height:
p := float32(width) / float32(maxThumbWidth) // i.e. height = newWidth * (height / width)
return maxThumbWidth, int(float32(height) / p) height = int(float32(maxThumbWidth) / aspect)
return maxThumbWidth, height
// Height is larger side. // Height is larger side.
case height > width: case height > width:
p := float32(height) / float32(maxThumbHeight) // i.e. width = newHeight * (width / height)
return int(float32(width) / p), maxThumbHeight width = int(float32(maxThumbHeight) * aspect)
return width, maxThumbHeight
// Square. // Square.
default: default: