[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

vendor/github.com/jackc/pgconn/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.11.0 (February 7, 2022)
+
+* Support port in ip from LookupFunc to override config (James Hartig)
+* Fix TLS connection timeout (Blake Embrey)
+* Add support for read-only, primary, standby, prefer-standby target_session_attributes (Oscar)
+* Fix connect when receiving NoticeResponse
+
 # 1.10.1 (November 20, 2021)
 
 * Close without waiting for response (Kei Kamikawa)

vendor/github.com/jackc/pgconn/config.go

@@ -248,21 +248,21 @@ func ParseConfig(connString string) (*Config, error) {
 	config.LookupFunc = makeDefaultResolver().LookupHost
 
 	notRuntimeParams := map[string]struct{}{
-		"host":                 struct{}{},
-		"port":                 struct{}{},
-		"database":             struct{}{},
-		"user":                 struct{}{},
-		"password":             struct{}{},
-		"passfile":             struct{}{},
-		"connect_timeout":      struct{}{},
-		"sslmode":              struct{}{},
-		"sslkey":               struct{}{},
-		"sslcert":              struct{}{},
-		"sslrootcert":          struct{}{},
-		"target_session_attrs": struct{}{},
-		"min_read_buffer_size": struct{}{},
-		"service":              struct{}{},
-		"servicefile":          struct{}{},
+		"host":                 {},
+		"port":                 {},
+		"database":             {},
+		"user":                 {},
+		"password":             {},
+		"passfile":             {},
+		"connect_timeout":      {},
+		"sslmode":              {},
+		"sslkey":               {},
+		"sslcert":              {},
+		"sslrootcert":          {},
+		"target_session_attrs": {},
+		"min_read_buffer_size": {},
+		"service":              {},
+		"servicefile":          {},
 	}
 
 	for k, v := range settings {
@@ -329,10 +329,19 @@ func ParseConfig(connString string) (*Config, error) {
 		}
 	}
 
-	if settings["target_session_attrs"] == "read-write" {
+	switch tsa := settings["target_session_attrs"]; tsa {
+	case "read-write":
 		config.ValidateConnect = ValidateConnectTargetSessionAttrsReadWrite
-	} else if settings["target_session_attrs"] != "any" {
-		return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", settings["target_session_attrs"])}
+	case "read-only":
+		config.ValidateConnect = ValidateConnectTargetSessionAttrsReadOnly
+	case "primary":
+		config.ValidateConnect = ValidateConnectTargetSessionAttrsPrimary
+	case "standby":
+		config.ValidateConnect = ValidateConnectTargetSessionAttrsStandby
+	case "any", "prefer-standby":
+		// do nothing
+	default:
+		return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", tsa)}
 	}
 
 	return config, nil
@@ -727,3 +736,48 @@ func ValidateConnectTargetSessionAttrsReadWrite(ctx context.Context, pgConn *PgC
 
 	return nil
 }
+
+// ValidateConnectTargetSessionAttrsReadOnly is an ValidateConnectFunc that implements libpq compatible
+// target_session_attrs=read-only.
+func ValidateConnectTargetSessionAttrsReadOnly(ctx context.Context, pgConn *PgConn) error {
+	result := pgConn.ExecParams(ctx, "show transaction_read_only", nil, nil, nil, nil).Read()
+	if result.Err != nil {
+		return result.Err
+	}
+
+	if string(result.Rows[0][0]) != "on" {
+		return errors.New("connection is not read only")
+	}
+
+	return nil
+}
+
+// ValidateConnectTargetSessionAttrsStandby is an ValidateConnectFunc that implements libpq compatible
+// target_session_attrs=standby.
+func ValidateConnectTargetSessionAttrsStandby(ctx context.Context, pgConn *PgConn) error {
+	result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
+	if result.Err != nil {
+		return result.Err
+	}
+
+	if string(result.Rows[0][0]) != "t" {
+		return errors.New("server is not in hot standby mode")
+	}
+
+	return nil
+}
+
+// ValidateConnectTargetSessionAttrsPrimary is an ValidateConnectFunc that implements libpq compatible
+// target_session_attrs=primary.
+func ValidateConnectTargetSessionAttrsPrimary(ctx context.Context, pgConn *PgConn) error {
+	result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
+	if result.Err != nil {
+		return result.Err
+	}
+
+	if string(result.Rows[0][0]) == "t" {
+		return errors.New("server is in standby mode")
+	}
+
+	return nil
+}

vendor/github.com/jackc/pgconn/pgconn.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"math"
 	"net"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -44,7 +45,8 @@ type Notification struct {
 // DialFunc is a function that can be used to connect to a PostgreSQL server.
 type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
 
-// LookupFunc is a function that can be used to lookup IPs addrs from host.
+// LookupFunc is a function that can be used to lookup IPs addrs from host. Optionally an ip:port combination can be
+// returned in order to override the connection string's port.
 type LookupFunc func(ctx context.Context, host string) (addrs []string, err error)
 
 // BuildFrontendFunc is a function that can be used to create Frontend implementation for connection.
@@ -196,11 +198,24 @@ func expandWithIPs(ctx context.Context, lookupFn LookupFunc, fallbacks []*Fallba
 		}
 
 		for _, ip := range ips {
-			configs = append(configs, &FallbackConfig{
-				Host:      ip,
-				Port:      fb.Port,
-				TLSConfig: fb.TLSConfig,
-			})
+			splitIP, splitPort, err := net.SplitHostPort(ip)
+			if err == nil {
+				port, err := strconv.ParseUint(splitPort, 10, 16)
+				if err != nil {
+					return nil, fmt.Errorf("error parsing port (%s) from lookup: %w", splitPort, err)
+				}
+				configs = append(configs, &FallbackConfig{
+					Host:      splitIP,
+					Port:      uint16(port),
+					TLSConfig: fb.TLSConfig,
+				})
+			} else {
+				configs = append(configs, &FallbackConfig{
+					Host:      ip,
+					Port:      fb.Port,
+					TLSConfig: fb.TLSConfig,
+				})
+			}
 		}
 	}
 
@@ -215,7 +230,7 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 
 	var err error
 	network, address := NetworkAddress(fallbackConfig.Host, fallbackConfig.Port)
-	pgConn.conn, err = config.DialFunc(ctx, network, address)
+	netConn, err := config.DialFunc(ctx, network, address)
 	if err != nil {
 		var netErr net.Error
 		if errors.As(err, &netErr) && netErr.Timeout() {
@@ -224,24 +239,27 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 		return nil, &connectError{config: config, msg: "dial error", err: err}
 	}
 
-	pgConn.parameterStatuses = make(map[string]string)
+	pgConn.conn = netConn
+	pgConn.contextWatcher = newContextWatcher(netConn)
+	pgConn.contextWatcher.Watch(ctx)
 
 	if fallbackConfig.TLSConfig != nil {
-		if err := pgConn.startTLS(fallbackConfig.TLSConfig); err != nil {
-			pgConn.conn.Close()
+		tlsConn, err := startTLS(netConn, fallbackConfig.TLSConfig)
+		pgConn.contextWatcher.Unwatch() // Always unwatch `netConn` after TLS.
+		if err != nil {
+			netConn.Close()
 			return nil, &connectError{config: config, msg: "tls error", err: err}
 		}
+
+		pgConn.conn = tlsConn
+		pgConn.contextWatcher = newContextWatcher(tlsConn)
+		pgConn.contextWatcher.Watch(ctx)
 	}
 
-	pgConn.status = connStatusConnecting
-	pgConn.contextWatcher = ctxwatch.NewContextWatcher(
-		func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
-		func() { pgConn.conn.SetDeadline(time.Time{}) },
-	)
-
-	pgConn.contextWatcher.Watch(ctx)
 	defer pgConn.contextWatcher.Unwatch()
 
+	pgConn.parameterStatuses = make(map[string]string)
+	pgConn.status = connStatusConnecting
 	pgConn.frontend = config.BuildFrontend(pgConn.conn, pgConn.conn)
 
 	startupMsg := pgproto3.StartupMessage{
@@ -317,7 +335,7 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 				}
 			}
 			return pgConn, nil
-		case *pgproto3.ParameterStatus:
+		case *pgproto3.ParameterStatus, *pgproto3.NoticeResponse:
 			// handled by ReceiveMessage
 		case *pgproto3.ErrorResponse:
 			pgConn.conn.Close()
@@ -329,24 +347,29 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 	}
 }
 
-func (pgConn *PgConn) startTLS(tlsConfig *tls.Config) (err error) {
-	err = binary.Write(pgConn.conn, binary.BigEndian, []int32{8, 80877103})
+func newContextWatcher(conn net.Conn) *ctxwatch.ContextWatcher {
+	return ctxwatch.NewContextWatcher(
+		func() { conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
+		func() { conn.SetDeadline(time.Time{}) },
+	)
+}
+
+func startTLS(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) {
+	err := binary.Write(conn, binary.BigEndian, []int32{8, 80877103})
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	response := make([]byte, 1)
-	if _, err = io.ReadFull(pgConn.conn, response); err != nil {
-		return
+	if _, err = io.ReadFull(conn, response); err != nil {
+		return nil, err
 	}
 
 	if response[0] != 'S' {
-		return errors.New("server refused TLS connection")
+		return nil, errors.New("server refused TLS connection")
 	}
 
-	pgConn.conn = tls.Client(pgConn.conn, tlsConfig)
-
-	return nil
+	return tls.Client(conn, tlsConfig), nil
 }
 
 func (pgConn *PgConn) txPasswordMessage(password string) (err error) {
@@ -1694,10 +1717,7 @@ func Construct(hc *HijackedConn) (*PgConn, error) {
 		cleanupDone: make(chan struct{}),
 	}
 
-	pgConn.contextWatcher = ctxwatch.NewContextWatcher(
-		func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
-		func() { pgConn.conn.SetDeadline(time.Time{}) },
-	)
+	pgConn.contextWatcher = newContextWatcher(pgConn.conn)
 
 	return pgConn, nil
 }

vendor/github.com/jackc/pgtype/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.10.0 (February 7, 2022)
+
+* Normalize UTC timestamps to comply with stdlib (Torkel Rogstad)
+* Assign Numeric to *big.Rat (Oleg Lomaka)
+* Fix typo in float8 error message (Pinank Solanki)
+* Scan type aliases for floating point types (Collin Forsyth)
+
 # 1.9.1 (November 28, 2021)
 
 * Fix: binary timestamp is assumed to be in UTC (restored behavior changed in v1.9.0)

vendor/github.com/jackc/pgtype/convert.go

@@ -337,6 +337,10 @@ func float64AssignTo(srcVal float64, srcStatus Status, dst interface{}) error {
 			if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr {
 				el := v.Elem()
 				switch el.Kind() {
+				// if dst is a type alias of a float32 or 64, set dst val
+				case reflect.Float32, reflect.Float64:
+					el.SetFloat(srcVal)
+					return nil
 				// if dst is a pointer to pointer, strip the pointer and try again
 				case reflect.Ptr:
 					if el.IsNil() {

vendor/github.com/jackc/pgtype/float8.go

@@ -204,7 +204,7 @@ func (dst *Float8) DecodeBinary(ci *ConnInfo, src []byte) error {
 	}
 
 	if len(src) != 8 {
-		return fmt.Errorf("invalid length for float4: %v", len(src))
+		return fmt.Errorf("invalid length for float8: %v", len(src))
 	}
 
 	n := int64(binary.BigEndian.Uint64(src))

vendor/github.com/jackc/pgtype/numeric.go

@@ -369,6 +369,12 @@ func (src *Numeric) AssignTo(dst interface{}) error {
 				return fmt.Errorf("%d is greater than maximum value for %T", normalizedInt, *v)
 			}
 			*v = normalizedInt.Uint64()
+		case *big.Rat:
+			rat, err := src.toBigRat()
+			if err != nil {
+				return err
+			}
+			v.Set(rat)
 		default:
 			if nextDst, retry := GetAssignToDstType(dst); retry {
 				return src.AssignTo(nextDst)
@@ -406,6 +412,26 @@ func (dst *Numeric) toBigInt() (*big.Int, error) {
 	return num, nil
 }
 
+func (dst *Numeric) toBigRat() (*big.Rat, error) {
+	if dst.NaN {
+		return nil, fmt.Errorf("%v is not a number", dst)
+	} else if dst.InfinityModifier == Infinity {
+		return nil, fmt.Errorf("%v is infinity", dst)
+	} else if dst.InfinityModifier == NegativeInfinity {
+		return nil, fmt.Errorf("%v is -infinity", dst)
+	}
+
+	num := new(big.Rat).SetInt(dst.Int)
+	if dst.Exp > 0 {
+		mul := new(big.Int).Exp(big10, big.NewInt(int64(dst.Exp)), nil)
+		num.Mul(num, new(big.Rat).SetInt(mul))
+	} else if dst.Exp < 0 {
+		mul := new(big.Int).Exp(big10, big.NewInt(int64(-dst.Exp)), nil)
+		num.Quo(num, new(big.Rat).SetInt(mul))
+	}
+	return num, nil
+}
+
 func (src *Numeric) toFloat64() (float64, error) {
 	if src.NaN {
 		return math.NaN(), nil

vendor/github.com/jackc/pgtype/timestamptz.go

@@ -124,7 +124,7 @@ func (dst *Timestamptz) DecodeText(ci *ConnInfo, src []byte) error {
 			return err
 		}
 
-		*dst = Timestamptz{Time: tim, Status: Present}
+		*dst = Timestamptz{Time: normalizePotentialUTC(tim), Status: Present}
 	}
 
 	return nil
@@ -231,6 +231,9 @@ func (src Timestamptz) Value() (driver.Value, error) {
 		if src.InfinityModifier != None {
 			return src.InfinityModifier.String(), nil
 		}
+		if src.Time.Location().String() == time.UTC.String() {
+			return src.Time.UTC(), nil
+		}
 		return src.Time, nil
 	case Null:
 		return nil, nil
@@ -289,8 +292,23 @@ func (dst *Timestamptz) UnmarshalJSON(b []byte) error {
 			return err
 		}
 
-		*dst = Timestamptz{Time: tim, Status: Present}
+		*dst = Timestamptz{Time: normalizePotentialUTC(tim), Status: Present}
 	}
 
 	return nil
 }
+
+// Normalize timestamps in UTC location to behave similarly to how the Golang
+// standard library does it: UTC timestamps lack a .loc value.
+//
+// Reason for this: when comparing two timestamps with reflect.DeepEqual (generally
+// speaking not a good idea, but several testing libraries (for example testify)
+// does this), their location data needs to be equal for them to be considered
+// equal.
+func normalizePotentialUTC(timestamp time.Time) time.Time {
+	if timestamp.Location().String() != time.UTC.String() {
+		return timestamp
+	}
+
+	return timestamp.UTC()
+}

vendor/github.com/jackc/pgx/v4/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 4.15.0 (February 7, 2022)
+
+* Upgrade to pgconn v1.11.0
+* Upgrade to pgtype v1.10.0
+* Upgrade puddle to v1.2.1
+* Make BatchResults.Close safe to be called multiple times
+
 # 4.14.1 (November 28, 2021)
 
 * Upgrade pgtype to v1.9.1 (fixes unintentional change to timestamp binary decoding)

vendor/github.com/jackc/pgx/v4/batch.go

@@ -3,6 +3,7 @@ package pgx
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/jackc/pgconn"
 )
@@ -46,17 +47,18 @@ type BatchResults interface {
 
 	// Close closes the batch operation. This must be called before the underlying connection can be used again. Any error
 	// that occurred during a batch operation may have made it impossible to resyncronize the connection with the server.
-	// In this case the underlying connection will have been closed.
+	// In this case the underlying connection will have been closed. Close is safe to call multiple times.
 	Close() error
 }
 
 type batchResults struct {
-	ctx  context.Context
-	conn *Conn
-	mrr  *pgconn.MultiResultReader
-	err  error
-	b    *Batch
-	ix   int
+	ctx    context.Context
+	conn   *Conn
+	mrr    *pgconn.MultiResultReader
+	err    error
+	b      *Batch
+	ix     int
+	closed bool
 }
 
 // Exec reads the results from the next query in the batch as if the query has been sent with Exec.
@@ -64,6 +66,9 @@ func (br *batchResults) Exec() (pgconn.CommandTag, error) {
 	if br.err != nil {
 		return nil, br.err
 	}
+	if br.closed {
+		return nil, fmt.Errorf("batch already closed")
+	}
 
 	query, arguments, _ := br.nextQueryAndArgs()
 
@@ -114,6 +119,11 @@ func (br *batchResults) Query() (Rows, error) {
 		return &connRows{err: br.err, closed: true}, br.err
 	}
 
+	if br.closed {
+		alreadyClosedErr := fmt.Errorf("batch already closed")
+		return &connRows{err: alreadyClosedErr, closed: true}, alreadyClosedErr
+	}
+
 	rows := br.conn.getRows(br.ctx, query, arguments)
 
 	if !br.mrr.NextResult() {
@@ -140,6 +150,10 @@ func (br *batchResults) Query() (Rows, error) {
 
 // QueryFunc reads the results from the next query in the batch as if the query has been sent with Conn.QueryFunc.
 func (br *batchResults) QueryFunc(scans []interface{}, f func(QueryFuncRow) error) (pgconn.CommandTag, error) {
+	if br.closed {
+		return nil, fmt.Errorf("batch already closed")
+	}
+
 	rows, err := br.Query()
 	if err != nil {
 		return nil, err
@@ -179,6 +193,11 @@ func (br *batchResults) Close() error {
 		return br.err
 	}
 
+	if br.closed {
+		return nil
+	}
+	br.closed = true
+
 	// log any queries that haven't yet been logged by Exec or Query
 	for {
 		query, args, ok := br.nextQueryAndArgs()

vendor/github.com/jackc/pgx/v4/rows.go

@@ -41,10 +41,13 @@ type Rows interface {
 
 	// Scan reads the values from the current row into dest values positionally.
 	// dest can include pointers to core types, values implementing the Scanner
-	// interface, and nil. nil will skip the value entirely.
+	// interface, and nil. nil will skip the value entirely. It is an error to
+	// call Scan without first calling Next() and checking that it returned true.
 	Scan(dest ...interface{}) error
 
-	// Values returns the decoded row values.
+	// Values returns the decoded row values. As with Scan(), it is an error to
+	// call Values without first calling Next() and checking that it returned
+	// true.
 	Values() ([]interface{}, error)
 
 	// RawValues returns the unparsed bytes of the row values. The returned [][]byte is only valid until the next Next