[feature] add ability to change s3 bucket lookup type (#4120)

# Description

This pull request implements the ability to specify the S3 bucket lookup type that is passed to the minio-go library. By default, minio-go uses a regex on the endpoint to see if it is a host that supports DNS access or not. Tigris and others are moving away from path style bucket access to DNS style bucket access using subdomains.

This keeps the default as 'auto' to allow existing users to not need to modify their configurations.

closes #3942

## Checklist

- [x] I/we have read the [GoToSocial contribution guidelines](https://codeberg.org/superseriousbusiness/gotosocial/src/branch/main/CONTRIBUTING.md).
- [ ] I/we have discussed the proposed changes already, either in an issue on the repository, or in the Matrix chat.
- [x] I/we have not leveraged AI to create the proposed changes.
- [x] I/we have performed a self-review of added code.
- [x] I/we have written code that is legible and maintainable by others.
- [ ] I/we have commented the added code, particularly in hard-to-understand areas.
- [x] I/we have made any necessary changes to documentation.
- [ ] I/we have added tests that cover new code.
- [x] I/we have run tests and they pass locally with the changes.
- [x] I/we have run `go fmt ./...` and `golangci-lint run`.

Co-authored-by: tobi <tobi.smethurst@protonmail.com>
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4120
Reviewed-by: kim <gruf@noreply.codeberg.org>
Co-authored-by: nopjmp <nop@unearthly.dev>
Co-committed-by: nopjmp <nop@unearthly.dev>

docs/configuration/storage.md

@@ -100,6 +100,19 @@ storage-s3-secret-key: ""
 # Examples: ["gts","cool-instance"]
 # Default: ""
 storage-s3-bucket: ""
+
+# String. Bucket lookup type.
+#
+# If you know what kind of bucket lookup type you need you can specify it here.
+#
+# Many providers are no longer offering path-style bucket access and only subdomain-style. Then,
+# you will need to specify "dns" here instead of the default "auto".
+#
+# If your provider only supports path-style bucket access, specify "path".
+#
+# Examples: ["path", "dns", "auto"]
+# Default: "auto"
+storage-s3-bucket-lookup: "auto"
 ```
 
 ## AWS S3 Configuration

example/config.yaml

@@ -749,6 +749,19 @@ storage-s3-secret-key: ""
 # Default: ""
 storage-s3-bucket: ""
 
+# String. Bucket lookup type.
+#
+# If you know what kind of bucket lookup type you need you can specify it here.
+#
+# Many providers are no longer offering path-style bucket access and only subdomain-style. Then,
+# you will need to specify "dns" here instead of the default "auto".
+#
+# If your provider only supports path-style bucket access, specify "path".
+#
+# Examples: ["path", "dns", "auto"]
+# Default: "auto"
+storage-s3-bucket-lookup: "auto"
+
 ###########################
 ##### STATUSES CONFIG #####
 ###########################

internal/config/config.go

@@ -112,15 +112,16 @@ type Configuration struct {
 	MediaCleanupEvery        time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`
 	MediaFfmpegPoolSize      int           `name:"media-ffmpeg-pool-size" usage:"Number of instances of the embedded ffmpeg WASM binary to add to the media processing pool. 0 or less uses GOMAXPROCS."`
 
-	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
+	StorageBackend        string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
-	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`
+	StorageLocalBasePath  string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`
-	StorageS3Endpoint    string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"`
+	StorageS3Endpoint     string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"`
-	StorageS3AccessKey   string `name:"storage-s3-access-key" usage:"S3 Access Key"`
+	StorageS3AccessKey    string `name:"storage-s3-access-key" usage:"S3 Access Key"`
-	StorageS3SecretKey   string `name:"storage-s3-secret-key" usage:"S3 Secret Key"`
+	StorageS3SecretKey    string `name:"storage-s3-secret-key" usage:"S3 Secret Key"`
-	StorageS3UseSSL      bool   `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
+	StorageS3UseSSL       bool   `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
-	StorageS3BucketName  string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
+	StorageS3BucketName   string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
-	StorageS3Proxy       bool   `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
+	StorageS3Proxy        bool   `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
-	StorageS3RedirectURL string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."`
+	StorageS3RedirectURL  string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."`
+	StorageS3BucketLookup string `name:"storage-s3-bucket-lookup" usage:"S3 bucket lookup type to use. Can be 'auto', 'dns' or 'path'. Defaults to 'auto'."`
 
 	StatusesMaxChars           int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"`
 	StatusesPollMaxOptions     int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"`

internal/config/defaults.go

@@ -87,11 +87,12 @@ var Defaults = Configuration{
 	MediaCleanupEvery:        24 * time.Hour, // 1/day.
 	MediaFfmpegPoolSize:      1,
 
-	StorageBackend:       "local",
+	StorageBackend:        "local",
-	StorageLocalBasePath: "/gotosocial/storage",
+	StorageLocalBasePath:  "/gotosocial/storage",
-	StorageS3UseSSL:      true,
+	StorageS3UseSSL:       true,
-	StorageS3Proxy:       false,
+	StorageS3Proxy:        false,
-	StorageS3RedirectURL: "",
+	StorageS3RedirectURL:  "",
+	StorageS3BucketLookup: "auto",
 
 	StatusesMaxChars:           5000,
 	StatusesPollMaxOptions:     6,

internal/config/helpers.gen.go

@@ -1781,6 +1781,31 @@ func GetStorageS3RedirectURL() string { return global.GetStorageS3RedirectURL()
 // SetStorageS3RedirectURL safely sets the value for global configuration 'StorageS3RedirectURL' field
 func SetStorageS3RedirectURL(v string) { global.SetStorageS3RedirectURL(v) }
 
+// GetStorageS3BucketLookup safely fetches the Configuration value for state's 'StorageS3BucketLookup' field
+func (st *ConfigState) GetStorageS3BucketLookup() (v string) {
+	st.mutex.RLock()
+	v = st.config.StorageS3BucketLookup
+	st.mutex.RUnlock()
+	return
+}
+
+// SetStorageS3BucketLookup safely sets the Configuration value for state's 'StorageS3BucketLookup' field
+func (st *ConfigState) SetStorageS3BucketLookup(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.StorageS3BucketLookup = v
+	st.reloadToViper()
+}
+
+// StorageS3BucketLookupFlag returns the flag name for the 'StorageS3BucketLookup' field
+func StorageS3BucketLookupFlag() string { return "storage-s3-bucket-lookup" }
+
+// GetStorageS3BucketLookup safely fetches the value for global configuration 'StorageS3BucketLookup' field
+func GetStorageS3BucketLookup() string { return global.GetStorageS3BucketLookup() }
+
+// SetStorageS3BucketLookup safely sets the value for global configuration 'StorageS3BucketLookup' field
+func SetStorageS3BucketLookup(v string) { global.SetStorageS3BucketLookup(v) }
+
 // GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
 func (st *ConfigState) GetStatusesMaxChars() (v int) {
 	st.mutex.RLock()

internal/storage/storage.go

@@ -315,11 +315,25 @@ func NewS3Storage() (*Driver, error) {
 	bucket := config.GetStorageS3BucketName()
 	redirectURL := config.GetStorageS3RedirectURL()
 
+	var bucketLookup minio.BucketLookupType
+	switch s := config.GetStorageS3BucketLookup(); s {
+	case "auto":
+		bucketLookup = minio.BucketLookupAuto
+	case "dns":
+		bucketLookup = minio.BucketLookupDNS
+	case "path":
+		bucketLookup = minio.BucketLookupPath
+	default:
+		log.Warnf(nil, "%s set to %s which is not recognized, defaulting to 'auto'", config.StorageS3BucketLookupFlag(), s)
+		bucketLookup = minio.BucketLookupAuto
+	}
+
 	// Open the s3 storage implementation
 	s3, err := s3.Open(endpoint, bucket, &s3.Config{
 		CoreOpts: minio.Options{
-			Creds:  credentials.NewStaticV4(access, secret, ""),
+			Creds:        credentials.NewStaticV4(access, secret, ""),
-			Secure: secure,
+			Secure:       secure,
+			BucketLookup: bucketLookup,
 		},
 		PutChunkSize: 5 * 1024 * 1024, // 5MiB
 		ListSize:     200,

test/envparsing.sh

@@ -185,6 +185,7 @@ EXPECT=$(cat << "EOF"
     "storage-local-base-path": "/root/store",
     "storage-s3-access-key": "minio",
     "storage-s3-bucket": "gts",
+    "storage-s3-bucket-lookup": "auto",
     "storage-s3-endpoint": "localhost:9000",
     "storage-s3-proxy": true,
     "storage-s3-redirect-url": "",
@@ -270,6 +271,7 @@ GTS_STORAGE_S3_ACCESS_KEY='minio' \
 GTS_STORAGE_S3_SECRET_KEY='miniostorage' \
 GTS_STORAGE_S3_ENDPOINT='localhost:9000' \
 GTS_STORAGE_S3_USE_SSL='false' \
+GTS_STORAGE_S3_BUCKET_LOOKUP='auto' \
 GTS_STORAGE_S3_PROXY='true' \
 GTS_STORAGE_S3_REDIRECT_URL='' \
 GTS_STORAGE_S3_BUCKET='gts' \