[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

vendor/github.com/jackc/pgx/v4/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 4.15.0 (February 7, 2022)
+
+* Upgrade to pgconn v1.11.0
+* Upgrade to pgtype v1.10.0
+* Upgrade puddle to v1.2.1
+* Make BatchResults.Close safe to be called multiple times
+
 # 4.14.1 (November 28, 2021)
 
 * Upgrade pgtype to v1.9.1 (fixes unintentional change to timestamp binary decoding)

vendor/github.com/jackc/pgx/v4/batch.go

@@ -3,6 +3,7 @@ package pgx
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/jackc/pgconn"
 )
@@ -46,17 +47,18 @@ type BatchResults interface {
 
 	// Close closes the batch operation. This must be called before the underlying connection can be used again. Any error
 	// that occurred during a batch operation may have made it impossible to resyncronize the connection with the server.
-	// In this case the underlying connection will have been closed.
+	// In this case the underlying connection will have been closed. Close is safe to call multiple times.
 	Close() error
 }
 
 type batchResults struct {
-	ctx  context.Context
-	conn *Conn
-	mrr  *pgconn.MultiResultReader
-	err  error
-	b    *Batch
-	ix   int
+	ctx    context.Context
+	conn   *Conn
+	mrr    *pgconn.MultiResultReader
+	err    error
+	b      *Batch
+	ix     int
+	closed bool
 }
 
 // Exec reads the results from the next query in the batch as if the query has been sent with Exec.
@@ -64,6 +66,9 @@ func (br *batchResults) Exec() (pgconn.CommandTag, error) {
 	if br.err != nil {
 		return nil, br.err
 	}
+	if br.closed {
+		return nil, fmt.Errorf("batch already closed")
+	}
 
 	query, arguments, _ := br.nextQueryAndArgs()
 
@@ -114,6 +119,11 @@ func (br *batchResults) Query() (Rows, error) {
 		return &connRows{err: br.err, closed: true}, br.err
 	}
 
+	if br.closed {
+		alreadyClosedErr := fmt.Errorf("batch already closed")
+		return &connRows{err: alreadyClosedErr, closed: true}, alreadyClosedErr
+	}
+
 	rows := br.conn.getRows(br.ctx, query, arguments)
 
 	if !br.mrr.NextResult() {
@@ -140,6 +150,10 @@ func (br *batchResults) Query() (Rows, error) {
 
 // QueryFunc reads the results from the next query in the batch as if the query has been sent with Conn.QueryFunc.
 func (br *batchResults) QueryFunc(scans []interface{}, f func(QueryFuncRow) error) (pgconn.CommandTag, error) {
+	if br.closed {
+		return nil, fmt.Errorf("batch already closed")
+	}
+
 	rows, err := br.Query()
 	if err != nil {
 		return nil, err
@@ -179,6 +193,11 @@ func (br *batchResults) Close() error {
 		return br.err
 	}
 
+	if br.closed {
+		return nil
+	}
+	br.closed = true
+
 	// log any queries that haven't yet been logged by Exec or Query
 	for {
 		query, args, ok := br.nextQueryAndArgs()

vendor/github.com/jackc/pgx/v4/rows.go

@@ -41,10 +41,13 @@ type Rows interface {
 
 	// Scan reads the values from the current row into dest values positionally.
 	// dest can include pointers to core types, values implementing the Scanner
-	// interface, and nil. nil will skip the value entirely.
+	// interface, and nil. nil will skip the value entirely. It is an error to
+	// call Scan without first calling Next() and checking that it returned true.
 	Scan(dest ...interface{}) error
 
-	// Values returns the decoded row values.
+	// Values returns the decoded row values. As with Scan(), it is an error to
+	// call Values without first calling Next() and checking that it returned
+	// true.
 	Values() ([]interface{}, error)
 
 	// RawValues returns the unparsed bytes of the row values. The returned [][]byte is only valid until the next Next