feat: pre-signed URL for S3 storage (#2855)

Adds automatically background refresh of all external links if they are belongs to the current blob (S3) storage. The feature is disabled by default in order to keep backward compatibility.

The background go-routine spawns once during startup and periodically signs and updates external links if that links belongs to current S3 storage.

The original idea was to sign external links on-demand, however, with current architecture it will require duplicated code in plenty of places. If do it, the changes will be quite invasive and in the end pointless: I believe, the architecture will be eventually updated to give more scalable way for pluggable storage. For example - Upload/Download interface without hard dependency on external link. There are stubs already, but I don't feel confident enough to change significant part of the application architecture.

plugin/storage/s3/s3.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	s3config "github.com/aws/aws-sdk-go-v2/config"
@@ -14,8 +15,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	errors2 "github.com/pkg/errors"
 )
 
+const LinkLifetime = 24 * time.Hour
+
 type Config struct {
 	AccessKey string
 	SecretKey string
@@ -24,6 +28,7 @@ type Config struct {
 	Region    string
 	URLPrefix string
 	URLSuffix string
+	PreSign   bool
 }
 
 type Client struct {
@@ -93,5 +98,44 @@ func (client *Client) UploadFile(ctx context.Context, filename string, fileType
 	if link == "" {
 		return "", errors.New("failed to get file link")
 	}
+	if client.Config.PreSign {
+		return client.PreSignLink(ctx, link)
+	}
 	return link, nil
 }
+
+// PreSignLink generates a pre-signed URL for the given sourceLink.
+// If the link does not belong to the configured storage endpoint, it is returned as-is.
+// If the link belongs to the storage, the function generates a pre-signed URL using the AWS S3 client.
+func (client *Client) PreSignLink(ctx context.Context, sourceLink string) (string, error) {
+	u, err := url.Parse(sourceLink)
+	if err != nil {
+		return "", errors2.Wrapf(err, "parse URL")
+	}
+	// if link doesn't belong to storage, then return as-is.
+	// the empty hostname is corner-case for AWS native endpoint.
+	if client.Config.EndPoint != "" && !strings.Contains(client.Config.EndPoint, u.Hostname()) {
+		return sourceLink, nil
+	}
+
+	filename := u.Path
+	if prefixLen := len(client.Config.URLPrefix); len(filename) >= prefixLen {
+		filename = filename[prefixLen:]
+	}
+	if suffixLen := len(client.Config.URLSuffix); len(filename) >= suffixLen {
+		filename = filename[:len(filename)-suffixLen]
+	}
+	filename = strings.Trim(filename, "/")
+	if strings.HasPrefix(filename, client.Config.Bucket) {
+		filename = strings.Trim(filename[len(client.Config.Bucket):], "/")
+	}
+
+	req, err := awss3.NewPresignClient(client.Client).PresignGetObject(ctx, &awss3.GetObjectInput{
+		Bucket: aws.String(client.Config.Bucket),
+		Key:    aws.String(filename),
+	}, awss3.WithPresignExpires(LinkLifetime))
+	if err != nil {
+		return "", errors2.Wrapf(err, "pre-sign link")
+	}
+	return req.URL, nil
+}