SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

[performance] media processing improvements (#1288)

Browse Source 
* media processor consolidation and reformatting, reduce amount of required syscalls

Signed-off-by: kim <grufwub@gmail.com>

* update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix duration not being set, fix mp4 test expecting error

Signed-off-by: kim <grufwub@gmail.com>

* fix test expecting media files with different extension

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* fix expected storage paths in tests, update expected test thumbnails

Signed-off-by: kim <grufwub@gmail.com>

* remove dead code

Signed-off-by: kim <grufwub@gmail.com>

* fix cached presigned s3 url fetching

Signed-off-by: kim <grufwub@gmail.com>

* fix tests

Signed-off-by: kim <grufwub@gmail.com>

* fix test models

Signed-off-by: kim <grufwub@gmail.com>

* update media processing to use sync.Once{} for concurrency protection

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* fix passing in KVStore GetStream() as stream to PutStream()

Signed-off-by: kim <grufwub@gmail.com>

* fix unlocks of storage keys

Signed-off-by: kim <grufwub@gmail.com>

* whoops, return the error...

Signed-off-by: kim <grufwub@gmail.com>

* pour one out for tobi's code <3

Signed-off-by: kim <grufwub@gmail.com>

* add back the byte slurping code

Signed-off-by: kim <grufwub@gmail.com>

* check for both ErrUnexpectedEOF and EOF

Signed-off-by: kim <grufwub@gmail.com>

* add back links to file format header information

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
2023-01-11 11:13:13 +00:00
committed by GitHub
parent 3512325e46
commit 5318054808
64 changed files with 1279 additions and 1405 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
internal/api/fileserver/servefile.go
View File

@@ -117,14 +117,19 @@ func (m *Module) ServeFile(c *gin.Context) {
return
}
// try to slurp the first few bytes to make sure we have something
b := bytes.NewBuffer(make([]byte, 0, 64))
if _, err := io.CopyN(b, content.Content, 64); err != nil {
// create a "slurp" buffer ;)
b := make([]byte, 64)
// Try read the first 64 bytes into memory, to try return a more useful "not found" error.
if _, err := io.ReadFull(content.Content, b); err != nil &&
(err != io.ErrUnexpectedEOF && err != io.EOF) {
err = fmt.Errorf("ServeFile: error reading from content: %w", err)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet)
return
}
// we're good, return the slurped bytes + the rest of the content
c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil)
c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(
bytes.NewReader(b), content.Content,
), nil)
}

18
internal/api/fileserver/servefile_test.go
View File

@@ -99,7 +99,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@@ -119,7 +119,7 @@ func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@@ -139,7 +139,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@@ -159,7 +159,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@@ -182,7 +182,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@@ -205,7 +205,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@@ -228,7 +228,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusNotFound, code)
@@ -249,7 +249,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusNotFound, code)
@@ -261,7 +261,7 @@ func (suite *ServeFileTestSuite) TestServeFileNotFound() {
"01GMMY4G9B0QEG0PQK5Q5JGJWZ",
media.TypeAttachment,
media.SizeOriginal,
"01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg",
"01GMMY68Y7E5DJ3CA3Y9SS8524.jpg",
)
suite.Equal(http.StatusNotFound, code)