mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2025-06-05 21:59:39 +02:00
[chore] use kv.KVStore also for S3 storage (#1113)
* replace s3 storage implementation to also use kv.KVStore Signed-off-by: kim <grufwub@gmail.com> * pull in latest `go-store` fix Signed-off-by: kim <grufwub@gmail.com> * pull-in go-store v2.0.9 fixes, update s3 put chunk size to 5MiB Signed-off-by: kim <grufwub@gmail.com> Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
@@ -19,80 +19,32 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/url"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"codeberg.org/gruf/go-store/v2/kv"
|
||||
"codeberg.org/gruf/go-store/v2/storage"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
mc *minio.Client
|
||||
bucket string
|
||||
proxy bool
|
||||
}
|
||||
|
||||
func NewS3(mc *minio.Client, bucket string, proxy bool) *S3 {
|
||||
return &S3{
|
||||
mc: mc,
|
||||
bucket: bucket,
|
||||
proxy: proxy,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
|
||||
r, err := s.GetStream(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
b, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading data from s3: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
err = fmt.Errorf("retrieving object from s3: %w", err)
|
||||
}
|
||||
return o, err
|
||||
}
|
||||
|
||||
func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error {
|
||||
if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil {
|
||||
return fmt.Errorf("uploading data stream: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3) Put(ctx context.Context, key string, value []byte) error {
|
||||
if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil {
|
||||
return fmt.Errorf("uploading data slice: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3) Delete(ctx context.Context, key string) error {
|
||||
return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
Proxy bool
|
||||
Bucket string
|
||||
Storage *storage.S3Storage
|
||||
*kv.KVStore
|
||||
}
|
||||
|
||||
func (s *S3) URL(ctx context.Context, key string) *url.URL {
|
||||
if s.proxy {
|
||||
if s.Proxy {
|
||||
return nil
|
||||
}
|
||||
|
||||
// it's safe to ignore the error here, as we just fall back to fetching the
|
||||
// file if the url request fails
|
||||
url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{
|
||||
// it's safe to ignore the error here, as we just fall back to fetching the file if URL request fails
|
||||
url, _ := s.Storage.Client().PresignedGetObject(ctx, s.Bucket, key, time.Hour, url.Values{
|
||||
"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
|
||||
})
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -52,20 +52,38 @@ type Driver interface {
|
||||
func AutoConfig() (Driver, error) {
|
||||
switch config.GetStorageBackend() {
|
||||
case "s3":
|
||||
mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{
|
||||
Creds: credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""),
|
||||
Secure: config.GetStorageS3UseSSL(),
|
||||
endpoint := config.GetStorageS3Endpoint()
|
||||
access := config.GetStorageS3AccessKey()
|
||||
secret := config.GetStorageS3SecretKey()
|
||||
secure := config.GetStorageS3UseSSL()
|
||||
bucket := config.GetStorageS3BucketName()
|
||||
proxy := config.GetStorageS3Proxy()
|
||||
|
||||
s3, err := storage.OpenS3(endpoint, bucket, &storage.S3Config{
|
||||
CoreOpts: minio.Options{
|
||||
Creds: credentials.NewStaticV4(access, secret, ""),
|
||||
Secure: secure,
|
||||
},
|
||||
GetOpts: minio.GetObjectOptions{},
|
||||
PutOpts: minio.PutObjectOptions{},
|
||||
PutChunkSize: 5 * 1024 * 1024, // 5MiB
|
||||
StatOpts: minio.StatObjectOptions{},
|
||||
RemoveOpts: minio.RemoveObjectOptions{},
|
||||
ListSize: 200,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating minio client: %w", err)
|
||||
return nil, fmt.Errorf("error opening s3 storage: %w", err)
|
||||
}
|
||||
return NewS3(
|
||||
mc,
|
||||
config.GetStorageS3BucketName(),
|
||||
config.GetStorageS3Proxy(),
|
||||
), nil
|
||||
|
||||
return &S3{
|
||||
Proxy: proxy,
|
||||
Bucket: bucket,
|
||||
Storage: s3,
|
||||
KVStore: kv.New(s3),
|
||||
}, nil
|
||||
case "local":
|
||||
basePath := config.GetStorageLocalBasePath()
|
||||
|
||||
disk, err := storage.OpenDisk(basePath, &storage.DiskConfig{
|
||||
// Put the store lockfile in the storage dir itself.
|
||||
// Normally this would not be safe, since we could end up
|
||||
@@ -75,8 +93,9 @@ func AutoConfig() (Driver, error) {
|
||||
LockFile: path.Join(basePath, "store.lock"),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error openingdisk storage: %v", err)
|
||||
return nil, fmt.Errorf("error opening disk storage: %w", err)
|
||||
}
|
||||
|
||||
return &Local{kv.New(disk)}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend())
|
||||
|
||||
Reference in New Issue
Block a user