[feature] Add S3 key prefix (#4200)

Been running these changes on on my live instance without any issues as far as I can tell. It's been playing nice with multiple instances in the same bucket.

# Description

This lets users prefix their object storage files.
Useful for when you want to host multiple GTS instances inside
the same bucket. Providers like Backblaze limit the number of buckets
you can have on your account so grouping by prefix may be more desirable
in this situation.

closes #1371

## Checklist

Please put an x inside each checkbox to indicate that you've read and followed it: `[ ]` -> `[x]`

If this is a documentation change, only the first checkbox must be filled (you can delete the others if you want).

- [x] I/we have read the [GoToSocial contribution guidelines](https://codeberg.org/superseriousbusiness/gotosocial/src/branch/main/CONTRIBUTING.md).
- [x] I/we have discussed the proposed changes already, either in an issue on the repository, or in the Matrix chat.
- [x] I/we have not leveraged AI to create the proposed changes.
- [x] I/we have performed a self-review of added code.
- [x] I/we have written code that is legible and maintainable by others.
- [x] I/we have commented the added code, particularly in hard-to-understand areas.
- [x] I/we have made any necessary changes to documentation.
- [ ] I/we have added tests that cover new code.
- [ ] I/we have run tests and they pass locally with the changes.
- [x] I/we have run `go fmt ./...` and `golangci-lint run`.

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4200
Co-authored-by: vdyotte <vdyotte@gmail.com>
Co-committed-by: vdyotte <vdyotte@gmail.com>

docs/configuration/storage.md

@@ -101,6 +101,20 @@ storage-s3-secret-key: ""
 # Default: ""
 storage-s3-bucket: ""
 
+
+# String. Key prefix to use for the S3 storage.
+# This is optional.
+#
+# Prefix must end with a trailing slash.
+#
+# This is useful if you want to store multiple instances in the same bucket,
+# or if you want to store your data in a subdirectory of the bucket.
+# This has no effect if the storage backend isn't "s3".
+#
+# Examples: ["gts-instance1/", "gts-instance2/"]
+# Default: ""
+storage-s3-key-prefix: ""
+
 # String. Bucket lookup type.
 #
 # If you know what kind of bucket lookup type you need you can specify it here.

internal/cleaner/media.go

@@ -20,6 +20,7 @@ package cleaner
 import (
 	"context"
 	"errors"
+	"strings"
 	"time"
 
 	"code.superseriousbusiness.org/gotosocial/internal/db"
@@ -96,6 +97,7 @@ func (m *Media) PruneOrphaned(ctx context.Context) (int, error) {
 	// All media files in storage will have path fitting: {$account}/{$type}/{$size}/{$id}.{$ext}
 	if err := m.state.Storage.WalkKeys(ctx, func(path string) error {
 		// Check for our expected fileserver path format.
+		path = strings.TrimPrefix(path, m.state.Storage.KeyPrefix)
 		if !regexes.FilePath.MatchString(path) {
 			log.Warnf(ctx, "unexpected storage item: %s", path)
 			return nil

internal/config/config.go

@@ -135,6 +135,7 @@ type Configuration struct {
 	StorageS3Proxy        bool   `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
 	StorageS3RedirectURL  string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."`
 	StorageS3BucketLookup string `name:"storage-s3-bucket-lookup" usage:"S3 bucket lookup type to use. Can be 'auto', 'dns' or 'path'. Defaults to 'auto'."`
+	StorageS3KeyPrefix    string `name:"storage-s3-key-prefix" usage:"Prefix to use for S3 keys. This is useful for separating multiple instances sharing the same S3 bucket."`
 
 	StatusesMaxChars           int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"`
 	StatusesPollMaxOptions     int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"`

internal/config/helpers.gen.go

@@ -104,6 +104,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
 	flags.Bool("storage-s3-proxy", cfg.StorageS3Proxy, "Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL")
 	flags.String("storage-s3-redirect-url", cfg.StorageS3RedirectURL, "Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL.")
 	flags.String("storage-s3-bucket-lookup", cfg.StorageS3BucketLookup, "S3 bucket lookup type to use. Can be 'auto', 'dns' or 'path'. Defaults to 'auto'.")
+	flags.String("storage-s3-key-prefix", cfg.StorageS3KeyPrefix, "Prefix to use for S3 keys. This is useful for separating multiple instances sharing the same S3 bucket.")
 	flags.Int("statuses-max-chars", cfg.StatusesMaxChars, "Max permitted characters for posted statuses, including content warning")
 	flags.Int("statuses-poll-max-options", cfg.StatusesPollMaxOptions, "Max amount of options permitted on a poll")
 	flags.Int("statuses-poll-option-max-chars", cfg.StatusesPollOptionMaxChars, "Max amount of characters for a poll option")
@@ -285,6 +286,7 @@ func (cfg *Configuration) MarshalMap() map[string]any {
 	cfgmap["storage-s3-proxy"] = cfg.StorageS3Proxy
 	cfgmap["storage-s3-redirect-url"] = cfg.StorageS3RedirectURL
 	cfgmap["storage-s3-bucket-lookup"] = cfg.StorageS3BucketLookup
+	cfgmap["storage-s3-key-prefix"] = cfg.StorageS3KeyPrefix
 	cfgmap["statuses-max-chars"] = cfg.StatusesMaxChars
 	cfgmap["statuses-poll-max-options"] = cfg.StatusesPollMaxOptions
 	cfgmap["statuses-poll-option-max-chars"] = cfg.StatusesPollOptionMaxChars
@@ -1029,6 +1031,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
 		}
 	}
 
+	if ival, ok := cfgmap["storage-s3-key-prefix"]; ok {
+		var err error
+		cfg.StorageS3KeyPrefix, err = cast.ToStringE(ival)
+		if err != nil {
+			return fmt.Errorf("error casting %#v -> string for 'storage-s3-key-prefix': %w", ival, err)
+		}
+	}
+
 	if ival, ok := cfgmap["statuses-max-chars"]; ok {
 		var err error
 		cfg.StatusesMaxChars, err = cast.ToIntE(ival)
@@ -3793,6 +3803,31 @@ func GetStorageS3BucketLookup() string { return global.GetStorageS3BucketLookup(
 // SetStorageS3BucketLookup safely sets the value for global configuration 'StorageS3BucketLookup' field
 func SetStorageS3BucketLookup(v string) { global.SetStorageS3BucketLookup(v) }
 
+// StorageS3KeyPrefixFlag returns the flag name for the 'StorageS3KeyPrefix' field
+func StorageS3KeyPrefixFlag() string { return "storage-s3-key-prefix" }
+
+// GetStorageS3KeyPrefix safely fetches the Configuration value for state's 'StorageS3KeyPrefix' field
+func (st *ConfigState) GetStorageS3KeyPrefix() (v string) {
+	st.mutex.RLock()
+	v = st.config.StorageS3KeyPrefix
+	st.mutex.RUnlock()
+	return
+}
+
+// SetStorageS3KeyPrefix safely sets the Configuration value for state's 'StorageS3KeyPrefix' field
+func (st *ConfigState) SetStorageS3KeyPrefix(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.StorageS3KeyPrefix = v
+	st.reloadToViper()
+}
+
+// GetStorageS3KeyPrefix safely fetches the value for global configuration 'StorageS3KeyPrefix' field
+func GetStorageS3KeyPrefix() string { return global.GetStorageS3KeyPrefix() }
+
+// SetStorageS3KeyPrefix safely sets the value for global configuration 'StorageS3KeyPrefix' field
+func SetStorageS3KeyPrefix(v string) { global.SetStorageS3KeyPrefix(v) }
+
 // StatusesMaxCharsFlag returns the flag name for the 'StatusesMaxChars' field
 func StatusesMaxCharsFlag() string { return "statuses-max-chars" }
 

internal/storage/storage.go

@@ -78,27 +78,32 @@ type Driver struct {
 	// S3-only parameters
 	Proxy          bool
 	Bucket         string
+	KeyPrefix      string
 	PresignedCache *ttl.Cache[string, PresignedURL]
 	RedirectURL    string
 }
 
 // Get returns the byte value for key in storage.
 func (d *Driver) Get(ctx context.Context, key string) ([]byte, error) {
+	key = d.KeyPrefix + key
 	return d.Storage.ReadBytes(ctx, key)
 }
 
 // GetStream returns an io.ReadCloser for the value bytes at key in the storage.
 func (d *Driver) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
+	key = d.KeyPrefix + key
 	return d.Storage.ReadStream(ctx, key)
 }
 
 // Put writes the supplied value bytes at key in the storage
 func (d *Driver) Put(ctx context.Context, key string, value []byte) (int, error) {
+	key = d.KeyPrefix + key
 	return d.Storage.WriteBytes(ctx, key, value)
 }
 
 // PutFile moves the contents of file at path, to storage.Driver{} under given key (with content-type if supported).
 func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) (int64, error) {
+	key = d.KeyPrefix + key
 	// Open file at path for reading.
 	file, err := os.Open(filepath)
 	if err != nil {
@@ -144,11 +149,13 @@ func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string)
 
 // Delete attempts to remove the supplied key (and corresponding value) from storage.
 func (d *Driver) Delete(ctx context.Context, key string) error {
+	key = d.KeyPrefix + key
 	return d.Storage.Remove(ctx, key)
 }
 
 // Has checks if the supplied key is in the storage.
 func (d *Driver) Has(ctx context.Context, key string) (bool, error) {
+	key = d.KeyPrefix + key
 	stat, err := d.Storage.Stat(ctx, key)
 	return (stat != nil), err
 }
@@ -156,6 +163,7 @@ func (d *Driver) Has(ctx context.Context, key string) (bool, error) {
 // WalkKeys walks the keys in the storage.
 func (d *Driver) WalkKeys(ctx context.Context, walk func(string) error) error {
 	return d.Storage.WalkKeys(ctx, storage.WalkKeysOpts{
+		Prefix: d.KeyPrefix,
 		Step: func(entry storage.Entry) error {
 			return walk(entry.Key)
 		},
@@ -164,6 +172,7 @@ func (d *Driver) WalkKeys(ctx context.Context, walk func(string) error) error {
 
 // URL will return a presigned GET object URL, but only if running on S3 storage with proxying disabled.
 func (d *Driver) URL(ctx context.Context, key string) *PresignedURL {
+	key = d.KeyPrefix + key
 	// Check whether S3 *without* proxying is enabled
 	s3, ok := d.Storage.(*s3.S3Storage)
 	if !ok || d.Proxy {
@@ -349,6 +358,7 @@ func NewS3Storage() (*Driver, error) {
 	return &Driver{
 		Proxy:          config.GetStorageS3Proxy(),
 		Bucket:         config.GetStorageS3BucketName(),
+		KeyPrefix:      config.GetStorageS3KeyPrefix(),
 		Storage:        s3,
 		PresignedCache: presignedCache,
 		RedirectURL:    redirectURL,

test/envparsing.sh

@@ -187,6 +187,7 @@ EXPECT=$(cat << "EOF"
     "storage-s3-bucket": "gts",
     "storage-s3-bucket-lookup": "auto",
     "storage-s3-endpoint": "localhost:9000",
+    "storage-s3-key-prefix": "",
     "storage-s3-proxy": true,
     "storage-s3-redirect-url": "",
     "storage-s3-secret-key": "miniostorage",
@@ -208,7 +209,7 @@ EXPECT=$(cat << "EOF"
 EOF
 )
 
-# Set all the environment variables to 
+# Set all the environment variables to
 # ensure that these are parsed without panic
 OUTPUT=$(GTS_LOG_LEVEL='info' \
 GTS_LOG_TIMESTAMP_FORMAT="banana" \