[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

internal/media/processingmedia.go

@@ -66,6 +66,9 @@ type ProcessingMedia struct {
 
 	// track whether this media has already been put in the databse
 	insertedInDB bool
+
+	// true if this is a recache, false if it's brand new media
+	recache bool
 }
 
 // AttachmentID returns the ID of the underlying media attachment without blocking processing.
@@ -93,8 +96,16 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt
 
 	// store the result in the database before returning it
 	if !p.insertedInDB {
-		if err := p.database.Put(ctx, p.attachment); err != nil {
-			return nil, err
+		if p.recache {
+			// if it's a recache we should only need to update
+			if err := p.database.UpdateByPrimaryKey(ctx, p.attachment); err != nil {
+				return nil, err
+			}
+		} else {
+			// otherwise we need to really PUT it
+			if err := p.database.Put(ctx, p.attachment); err != nil {
+				return nil, err
+			}
 		}
 		p.insertedInDB = true
 	}
@@ -305,6 +316,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
 		return fmt.Errorf("store: error storing stream: %s", err)
 	}
+	p.attachment.Cached = true
 
 	// if the original reader is a readcloser, close it since we're done with it now
 	if rc, ok := reader.(io.ReadCloser); ok {
@@ -360,6 +372,7 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
 		Thumbnail:         thumbnail,
 		Avatar:            false,
 		Header:            false,
+		Cached:            false,
 	}
 
 	// check if we have additional info to add to the attachment,
@@ -418,3 +431,24 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
 
 	return processingMedia, nil
 }
+
+func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) {
+	// get the existing attachment
+	attachment, err := m.db.GetAttachmentByID(ctx, attachmentID)
+	if err != nil {
+		return nil, err
+	}
+
+	processingMedia := &ProcessingMedia{
+		attachment:    attachment,
+		data:          data,
+		postData:      postData,
+		thumbState:    int32(received),
+		fullSizeState: int32(received),
+		database:      m.db,
+		storage:       m.storage,
+		recache:       true, // indicate it's a recache
+	}
+
+	return processingMedia, nil
+}