[feature] support processing of (many) more media types (#3090)

* initial work replacing our media decoding / encoding pipeline with ffprobe + ffmpeg

* specify the video codec to use when generating static image from emoji

* update go-storage library (fixes incompatibility after updating go-iotools)

* maintain image aspect ratio when generating a thumbnail for it

* update readme to show go-ffmpreg

* fix a bunch of media tests, move filesize checking to callers of media manager for more flexibility

* remove extra debug from error message

* fix up incorrect function signatures

* update PutFile to just use regular file copy, as changes are file is on separate partition

* fix remaining tests, remove some unneeded tests now we're working with ffmpeg/ffprobe

* update more tests, add more code comments

* add utilities to generate processed emoji / media outputs

* fix remaining tests

* add test for opus media file, add license header to utility cmds

* limit the number of concurrently available ffmpeg / ffprobe instances

* reduce number of instances

* further reduce number of instances

* fix envparsing test with configuration variables

* update docs and configuration with new media-{local,remote}-max-size variables
This commit is contained in:
kim
2024-07-12 09:39:47 +00:00
committed by GitHub
parent 5bc567196b
commit cde2fb6244
376 changed files with 8026 additions and 54091 deletions

View File

@ -1,8 +0,0 @@
language: go
go:
- "1.8.x"
- "1.10.x"
- "1.13.x"
- "1.14.x"
- "1.16.x"

View File

@ -1,7 +0,0 @@
Copyright (c) 2015 Conrad Irwin <conrad@bugsnag.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -1,81 +0,0 @@
go-errors/errors
================
[![Build Status](https://travis-ci.org/go-errors/errors.svg?branch=master)](https://travis-ci.org/go-errors/errors)
Package errors adds stacktrace support to errors in go.
This is particularly useful when you want to understand the state of execution
when an error was returned unexpectedly.
It provides the type \*Error which implements the standard golang error
interface, so you can use this library interchangably with code that is
expecting a normal error return.
Usage
-----
Full documentation is available on
[godoc](https://godoc.org/github.com/go-errors/errors), but here's a simple
example:
```go
package crashy
import "github.com/go-errors/errors"
var Crashed = errors.Errorf("oh dear")
func Crash() error {
return errors.New(Crashed)
}
```
This can be called as follows:
```go
package main
import (
"crashy"
"fmt"
"github.com/go-errors/errors"
)
func main() {
err := crashy.Crash()
if err != nil {
if errors.Is(err, crashy.Crashed) {
fmt.Println(err.(*errors.Error).ErrorStack())
} else {
panic(err)
}
}
}
```
Meta-fu
-------
This package was original written to allow reporting to
[Bugsnag](https://bugsnag.com/) from
[bugsnag-go](https://github.com/bugsnag/bugsnag-go), but after I found similar
packages by Facebook and Dropbox, it was moved to one canonical location so
everyone can benefit.
This package is licensed under the MIT license, see LICENSE.MIT for details.
## Changelog
* v1.1.0 updated to use go1.13's standard-library errors.Is method instead of == in errors.Is
* v1.2.0 added `errors.As` from the standard library.
* v1.3.0 *BREAKING* updated error methods to return `error` instead of `*Error`.
> Code that needs access to the underlying `*Error` can use the new errors.AsError(e)
> ```
> // before
> errors.New(err).ErrorStack()
> // after
>. errors.AsError(errors.Wrap(err)).ErrorStack()
> ```
* v1.4.0 *BREAKING* v1.4.0 reverted all changes from v1.3.0 and is identical to v1.2.0
* v1.4.1 no code change, but now without an unnecessary cover.out file.

View File

@ -1,209 +0,0 @@
// Package errors provides errors that have stack-traces.
//
// This is particularly useful when you want to understand the
// state of execution when an error was returned unexpectedly.
//
// It provides the type *Error which implements the standard
// golang error interface, so you can use this library interchangably
// with code that is expecting a normal error return.
//
// For example:
//
// package crashy
//
// import "github.com/go-errors/errors"
//
// var Crashed = errors.Errorf("oh dear")
//
// func Crash() error {
// return errors.New(Crashed)
// }
//
// This can be called as follows:
//
// package main
//
// import (
// "crashy"
// "fmt"
// "github.com/go-errors/errors"
// )
//
// func main() {
// err := crashy.Crash()
// if err != nil {
// if errors.Is(err, crashy.Crashed) {
// fmt.Println(err.(*errors.Error).ErrorStack())
// } else {
// panic(err)
// }
// }
// }
//
// This package was original written to allow reporting to Bugsnag,
// but after I found similar packages by Facebook and Dropbox, it
// was moved to one canonical location so everyone can benefit.
package errors
import (
"bytes"
"fmt"
"reflect"
"runtime"
)
// The maximum number of stackframes on any error.
var MaxStackDepth = 50
// Error is an error with an attached stacktrace. It can be used
// wherever the builtin error interface is expected.
type Error struct {
Err error
stack []uintptr
frames []StackFrame
prefix string
}
// New makes an Error from the given value. If that value is already an
// error then it will be used directly, if not, it will be passed to
// fmt.Errorf("%v"). The stacktrace will point to the line of code that
// called New.
func New(e interface{}) *Error {
var err error
switch e := e.(type) {
case error:
err = e
default:
err = fmt.Errorf("%v", e)
}
stack := make([]uintptr, MaxStackDepth)
length := runtime.Callers(2, stack[:])
return &Error{
Err: err,
stack: stack[:length],
}
}
// Wrap makes an Error from the given value. If that value is already an
// error then it will be used directly, if not, it will be passed to
// fmt.Errorf("%v"). The skip parameter indicates how far up the stack
// to start the stacktrace. 0 is from the current call, 1 from its caller, etc.
func Wrap(e interface{}, skip int) *Error {
if e == nil {
return nil
}
var err error
switch e := e.(type) {
case *Error:
return e
case error:
err = e
default:
err = fmt.Errorf("%v", e)
}
stack := make([]uintptr, MaxStackDepth)
length := runtime.Callers(2+skip, stack[:])
return &Error{
Err: err,
stack: stack[:length],
}
}
// WrapPrefix makes an Error from the given value. If that value is already an
// error then it will be used directly, if not, it will be passed to
// fmt.Errorf("%v"). The prefix parameter is used to add a prefix to the
// error message when calling Error(). The skip parameter indicates how far
// up the stack to start the stacktrace. 0 is from the current call,
// 1 from its caller, etc.
func WrapPrefix(e interface{}, prefix string, skip int) *Error {
if e == nil {
return nil
}
err := Wrap(e, 1+skip)
if err.prefix != "" {
prefix = fmt.Sprintf("%s: %s", prefix, err.prefix)
}
return &Error{
Err: err.Err,
stack: err.stack,
prefix: prefix,
}
}
// Errorf creates a new error with the given message. You can use it
// as a drop-in replacement for fmt.Errorf() to provide descriptive
// errors in return values.
func Errorf(format string, a ...interface{}) *Error {
return Wrap(fmt.Errorf(format, a...), 1)
}
// Error returns the underlying error's message.
func (err *Error) Error() string {
msg := err.Err.Error()
if err.prefix != "" {
msg = fmt.Sprintf("%s: %s", err.prefix, msg)
}
return msg
}
// Stack returns the callstack formatted the same way that go does
// in runtime/debug.Stack()
func (err *Error) Stack() []byte {
buf := bytes.Buffer{}
for _, frame := range err.StackFrames() {
buf.WriteString(frame.String())
}
return buf.Bytes()
}
// Callers satisfies the bugsnag ErrorWithCallerS() interface
// so that the stack can be read out.
func (err *Error) Callers() []uintptr {
return err.stack
}
// ErrorStack returns a string that contains both the
// error message and the callstack.
func (err *Error) ErrorStack() string {
return err.TypeName() + " " + err.Error() + "\n" + string(err.Stack())
}
// StackFrames returns an array of frames containing information about the
// stack.
func (err *Error) StackFrames() []StackFrame {
if err.frames == nil {
err.frames = make([]StackFrame, len(err.stack))
for i, pc := range err.stack {
err.frames[i] = NewStackFrame(pc)
}
}
return err.frames
}
// TypeName returns the type this error. e.g. *errors.stringError.
func (err *Error) TypeName() string {
if _, ok := err.Err.(uncaughtPanic); ok {
return "panic"
}
return reflect.TypeOf(err.Err).String()
}
// Return the wrapped error (implements api for As function).
func (err *Error) Unwrap() error {
return err.Err
}

View File

@ -1,31 +0,0 @@
// +build go1.13
package errors
import (
baseErrors "errors"
)
// find error in any wrapped error
func As(err error, target interface{}) bool {
return baseErrors.As(err, target)
}
// Is detects whether the error is equal to a given error. Errors
// are considered equal by this function if they are matched by errors.Is
// or if their contained errors are matched through errors.Is
func Is(e error, original error) bool {
if baseErrors.Is(e, original) {
return true
}
if e, ok := e.(*Error); ok {
return Is(e.Err, original)
}
if original, ok := original.(*Error); ok {
return Is(e, original.Err)
}
return false
}

View File

@ -1,57 +0,0 @@
// +build !go1.13
package errors
import (
"reflect"
)
type unwrapper interface {
Unwrap() error
}
// As assigns error or any wrapped error to the value target points
// to. If there is no value of the target type of target As returns
// false.
func As(err error, target interface{}) bool {
targetType := reflect.TypeOf(target)
for {
errType := reflect.TypeOf(err)
if errType == nil {
return false
}
if reflect.PtrTo(errType) == targetType {
reflect.ValueOf(target).Elem().Set(reflect.ValueOf(err))
return true
}
wrapped, ok := err.(unwrapper)
if ok {
err = wrapped.Unwrap()
} else {
return false
}
}
}
// Is detects whether the error is equal to a given error. Errors
// are considered equal by this function if they are the same object,
// or if they both contain the same error inside an errors.Error.
func Is(e error, original error) bool {
if e == original {
return true
}
if e, ok := e.(*Error); ok {
return Is(e.Err, original)
}
if original, ok := original.(*Error); ok {
return Is(e, original.Err)
}
return false
}

View File

@ -1,127 +0,0 @@
package errors
import (
"strconv"
"strings"
)
type uncaughtPanic struct{ message string }
func (p uncaughtPanic) Error() string {
return p.message
}
// ParsePanic allows you to get an error object from the output of a go program
// that panicked. This is particularly useful with https://github.com/mitchellh/panicwrap.
func ParsePanic(text string) (*Error, error) {
lines := strings.Split(text, "\n")
state := "start"
var message string
var stack []StackFrame
for i := 0; i < len(lines); i++ {
line := lines[i]
if state == "start" {
if strings.HasPrefix(line, "panic: ") {
message = strings.TrimPrefix(line, "panic: ")
state = "seek"
} else {
return nil, Errorf("bugsnag.panicParser: Invalid line (no prefix): %s", line)
}
} else if state == "seek" {
if strings.HasPrefix(line, "goroutine ") && strings.HasSuffix(line, "[running]:") {
state = "parsing"
}
} else if state == "parsing" {
if line == "" {
state = "done"
break
}
createdBy := false
if strings.HasPrefix(line, "created by ") {
line = strings.TrimPrefix(line, "created by ")
createdBy = true
}
i++
if i >= len(lines) {
return nil, Errorf("bugsnag.panicParser: Invalid line (unpaired): %s", line)
}
frame, err := parsePanicFrame(line, lines[i], createdBy)
if err != nil {
return nil, err
}
stack = append(stack, *frame)
if createdBy {
state = "done"
break
}
}
}
if state == "done" || state == "parsing" {
return &Error{Err: uncaughtPanic{message}, frames: stack}, nil
}
return nil, Errorf("could not parse panic: %v", text)
}
// The lines we're passing look like this:
//
// main.(*foo).destruct(0xc208067e98)
// /0/go/src/github.com/bugsnag/bugsnag-go/pan/main.go:22 +0x151
func parsePanicFrame(name string, line string, createdBy bool) (*StackFrame, error) {
idx := strings.LastIndex(name, "(")
if idx == -1 && !createdBy {
return nil, Errorf("bugsnag.panicParser: Invalid line (no call): %s", name)
}
if idx != -1 {
name = name[:idx]
}
pkg := ""
if lastslash := strings.LastIndex(name, "/"); lastslash >= 0 {
pkg += name[:lastslash] + "/"
name = name[lastslash+1:]
}
if period := strings.Index(name, "."); period >= 0 {
pkg += name[:period]
name = name[period+1:]
}
name = strings.Replace(name, "·", ".", -1)
if !strings.HasPrefix(line, "\t") {
return nil, Errorf("bugsnag.panicParser: Invalid line (no tab): %s", line)
}
idx = strings.LastIndex(line, ":")
if idx == -1 {
return nil, Errorf("bugsnag.panicParser: Invalid line (no line number): %s", line)
}
file := line[1:idx]
number := line[idx+1:]
if idx = strings.Index(number, " +"); idx > -1 {
number = number[:idx]
}
lno, err := strconv.ParseInt(number, 10, 32)
if err != nil {
return nil, Errorf("bugsnag.panicParser: Invalid line (bad line number): %s", line)
}
return &StackFrame{
File: file,
LineNumber: int(lno),
Package: pkg,
Name: name,
}, nil
}

View File

@ -1,114 +0,0 @@
package errors
import (
"bufio"
"bytes"
"fmt"
"os"
"runtime"
"strings"
)
// A StackFrame contains all necessary information about to generate a line
// in a callstack.
type StackFrame struct {
// The path to the file containing this ProgramCounter
File string
// The LineNumber in that file
LineNumber int
// The Name of the function that contains this ProgramCounter
Name string
// The Package that contains this function
Package string
// The underlying ProgramCounter
ProgramCounter uintptr
}
// NewStackFrame popoulates a stack frame object from the program counter.
func NewStackFrame(pc uintptr) (frame StackFrame) {
frame = StackFrame{ProgramCounter: pc}
if frame.Func() == nil {
return
}
frame.Package, frame.Name = packageAndName(frame.Func())
// pc -1 because the program counters we use are usually return addresses,
// and we want to show the line that corresponds to the function call
frame.File, frame.LineNumber = frame.Func().FileLine(pc - 1)
return
}
// Func returns the function that contained this frame.
func (frame *StackFrame) Func() *runtime.Func {
if frame.ProgramCounter == 0 {
return nil
}
return runtime.FuncForPC(frame.ProgramCounter)
}
// String returns the stackframe formatted in the same way as go does
// in runtime/debug.Stack()
func (frame *StackFrame) String() string {
str := fmt.Sprintf("%s:%d (0x%x)\n", frame.File, frame.LineNumber, frame.ProgramCounter)
source, err := frame.SourceLine()
if err != nil {
return str
}
return str + fmt.Sprintf("\t%s: %s\n", frame.Name, source)
}
// SourceLine gets the line of code (from File and Line) of the original source if possible.
func (frame *StackFrame) SourceLine() (string, error) {
if frame.LineNumber <= 0 {
return "???", nil
}
file, err := os.Open(frame.File)
if err != nil {
return "", New(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
currentLine := 1
for scanner.Scan() {
if currentLine == frame.LineNumber {
return string(bytes.Trim(scanner.Bytes(), " \t")), nil
}
currentLine++
}
if err := scanner.Err(); err != nil {
return "", New(err)
}
return "???", nil
}
func packageAndName(fn *runtime.Func) (string, string) {
name := fn.Name()
pkg := ""
// The name includes the path name to the package, which is unnecessary
// since the file name is already included. Plus, it has center dots.
// That is, we see
// runtime/debug.*T·ptrmethod
// and want
// *T.ptrmethod
// Since the package path might contains dots (e.g. code.google.com/...),
// we first remove the path prefix if there is one.
if lastslash := strings.LastIndex(name, "/"); lastslash >= 0 {
pkg += name[:lastslash] + "/"
name = name[lastslash+1:]
}
if period := strings.Index(name, "."); period >= 0 {
pkg += name[:period]
name = name[period+1:]
}
name = strings.Replace(name, "·", ".", -1)
return pkg, name
}