feat: pre-signed URL for S3 storage (#2855)

Adds automatically background refresh of all external links if they are belongs to the current blob (S3) storage. The feature is disabled by default in order to keep backward compatibility.

The background go-routine spawns once during startup and periodically signs and updates external links if that links belongs to current S3 storage.

The original idea was to sign external links on-demand, however, with current architecture it will require duplicated code in plenty of places. If do it, the changes will be quite invasive and in the end pointless: I believe, the architecture will be eventually updated to give more scalable way for pluggable storage. For example - Upload/Download interface without hard dependency on external link. There are stubs already, but I don't feel confident enough to change significant part of the application architecture.

api/v1/resource.go

@@ -480,6 +480,7 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
 		Bucket:    s3Config.Bucket,
 		URLPrefix: s3Config.URLPrefix,
 		URLSuffix: s3Config.URLSuffix,
+		PreSign:   s3Config.PreSign,
 	})
 	if err != nil {
 		return errors.Wrap(err, "Failed to create s3 client")

api/v1/storage.go

@@ -43,6 +43,7 @@ type StorageS3Config struct {
 	Bucket    string `json:"bucket"`
 	URLPrefix string `json:"urlPrefix"`
 	URLSuffix string `json:"urlSuffix"`
+	PreSign   bool   `json:"presign"`
 }
 
 type Storage struct {

api/v1/swagger.yaml

@@ -238,6 +238,8 @@ definitions:
         type: string
       urlSuffix:
         type: string
+      presign:
+        type: boolean
     type: object
   api_v1.StorageType:
     enum:
@@ -668,6 +670,8 @@ definitions:
         type: string
       urlSuffix:
         type: string
+      presign:
+        type: boolean
     type: object
   github_com_usememos_memos_api_v1.StorageType:
     enum:

bin/memos/main.go

@@ -12,6 +12,8 @@ import (
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
 
+	"github.com/usememos/memos/internal/jobs"
+
 	"github.com/usememos/memos/internal/log"
 	"github.com/usememos/memos/server"
 	_profile "github.com/usememos/memos/server/profile"
@@ -91,6 +93,9 @@ var (
 
 			printGreetings()
 
+			// update (pre-sign) object storage links if applicable
+			go jobs.RunPreSignLinks(ctx, storeInstance)
+
 			if err := s.Start(ctx); err != nil {
 				if err != http.ErrServerClosed {
 					log.Error("failed to start server", zap.Error(err))

docs/api/v1.md

@@ -1255,6 +1255,7 @@ Get GetImage from URL
 | secretKey | string |             | No       |
 | urlPrefix | string |             | No       |
 | urlSuffix | string |             | No       |
+| presign   | boolean |             | No       |
 
 #### api_v1.StorageType
 
@@ -1541,7 +1542,7 @@ Get GetImage from URL
 #### github_com_usememos_memos_api_v1.StorageS3Config
 
 | Name      | Type    | Description | Required |
-| --------- | ------ | ----------- | -------- |
+|-----------|---------| ----------- | -------- |
 | accessKey | string  |             | No       |
 | bucket    | string  |             | No       |
 | endPoint  | string  |             | No       |
@@ -1550,6 +1551,8 @@ Get GetImage from URL
 | secretKey | string  |             | No       |
 | urlPrefix | string  |             | No       |
 | urlSuffix | string  |             | No       |
+| presign   | boolean |             | No       |
+
 
 #### github_com_usememos_memos_api_v1.StorageType
 

internal/jobs/presign_link.go

@@ -0,0 +1,140 @@
+package jobs
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	apiv1 "github.com/usememos/memos/api/v1"
+	"github.com/usememos/memos/internal/log"
+	"github.com/usememos/memos/plugin/storage/s3"
+	"github.com/usememos/memos/store"
+)
+
+// RunPreSignLinks is a background job that pre-signs external links stored in the database.
+// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.
+func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {
+	for {
+		started := time.Now()
+		if err := signExternalLinks(ctx, dataStore); err != nil {
+			log.Warn("failed sign external links", zap.Error(err))
+		} else {
+			log.Info("links pre-signed", zap.Duration("duration", time.Since(started)))
+		}
+		select {
+		case <-time.After(s3.LinkLifetime / 2):
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func signExternalLinks(ctx context.Context, dataStore *store.Store) error {
+	const pageSize = 32
+
+	objectStore, err := findObjectStorage(ctx, dataStore)
+	if err != nil {
+		return errors.Wrapf(err, "find object storage")
+	}
+	if objectStore == nil || !objectStore.Config.PreSign {
+		// object storage not set or not supported
+		return nil
+	}
+
+	var offset int
+	var limit = pageSize
+	for {
+		resources, err := dataStore.ListResources(ctx, &store.FindResource{
+			GetBlob: false,
+			Limit:   &limit,
+			Offset:  &offset,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "list resources, offset %d", offset)
+		}
+
+		for _, res := range resources {
+			if res.ExternalLink == "" {
+				// not for object store
+				continue
+			}
+			if strings.Contains(res.ExternalLink, "?") && time.Since(time.Unix(res.UpdatedTs, 0)) < s3.LinkLifetime/2 {
+				// resource not signed (hack for migration)
+				// resource was recently updated - skipping
+				continue
+			}
+			newLink, err := objectStore.PreSignLink(ctx, res.ExternalLink)
+			if err != nil {
+				log.Warn("failed pre-sign link", zap.Int32("resource", res.ID), zap.String("link", res.ExternalLink), zap.Error(err))
+				continue // do not fail - we may want update left over links too
+			}
+			now := time.Now().Unix()
+			// we may want to use here transaction and batch update in the future
+			_, err = dataStore.UpdateResource(ctx, &store.UpdateResource{
+				ID:           res.ID,
+				UpdatedTs:    &now,
+				ExternalLink: &newLink,
+			})
+			if err != nil {
+				// something with DB - better to stop here
+				return errors.Wrapf(err, "update resource %d link to %q", res.ID, newLink)
+			}
+		}
+
+		offset += limit
+		if len(resources) < limit {
+			break
+		}
+	}
+	return nil
+}
+
+// findObjectStorage returns current default storage if it's S3-compatible or nil otherwise.
+// Returns error only in case of internal problems (ie: database or configuration issues).
+// May return nil client and nil error.
+func findObjectStorage(ctx context.Context, dataStore *store.Store) (*s3.Client, error) {
+	systemSettingStorageServiceID, err := dataStore.GetSystemSetting(ctx, &store.FindSystemSetting{Name: apiv1.SystemSettingStorageServiceIDName.String()})
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to find SystemSettingStorageServiceIDName")
+	}
+
+	storageServiceID := apiv1.DefaultStorage
+	if systemSettingStorageServiceID != nil {
+		err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID)
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed to unmarshal storage service id")
+		}
+	}
+	storage, err := dataStore.GetStorage(ctx, &store.FindStorage{ID: &storageServiceID})
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to find StorageServiceID")
+	}
+
+	if storage == nil {
+		return nil, nil // storage not configured - not an error, just return empty ref
+	}
+	storageMessage, err := apiv1.ConvertStorageFromStore(storage)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to ConvertStorageFromStore")
+	}
+	if storageMessage.Type != apiv1.StorageS3 {
+		return nil, nil
+	}
+
+	s3Config := storageMessage.Config.S3Config
+	return s3.NewClient(ctx, &s3.Config{
+		AccessKey: s3Config.AccessKey,
+		SecretKey: s3Config.SecretKey,
+		EndPoint:  s3Config.EndPoint,
+		Region:    s3Config.Region,
+		Bucket:    s3Config.Bucket,
+		URLPrefix: s3Config.URLPrefix,
+		URLSuffix: s3Config.URLSuffix,
+		PreSign:   s3Config.PreSign,
+	})
+}

plugin/storage/s3/s3.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	s3config "github.com/aws/aws-sdk-go-v2/config"
@@ -14,8 +15,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	errors2 "github.com/pkg/errors"
 )
 
+const LinkLifetime = 24 * time.Hour
+
 type Config struct {
 	AccessKey string
 	SecretKey string
@@ -24,6 +28,7 @@ type Config struct {
 	Region    string
 	URLPrefix string
 	URLSuffix string
+	PreSign   bool
 }
 
 type Client struct {
@@ -93,5 +98,44 @@ func (client *Client) UploadFile(ctx context.Context, filename string, fileType
 	if link == "" {
 		return "", errors.New("failed to get file link")
 	}
+	if client.Config.PreSign {
+		return client.PreSignLink(ctx, link)
+	}
 	return link, nil
 }
+
+// PreSignLink generates a pre-signed URL for the given sourceLink.
+// If the link does not belong to the configured storage endpoint, it is returned as-is.
+// If the link belongs to the storage, the function generates a pre-signed URL using the AWS S3 client.
+func (client *Client) PreSignLink(ctx context.Context, sourceLink string) (string, error) {
+	u, err := url.Parse(sourceLink)
+	if err != nil {
+		return "", errors2.Wrapf(err, "parse URL")
+	}
+	// if link doesn't belong to storage, then return as-is.
+	// the empty hostname is corner-case for AWS native endpoint.
+	if client.Config.EndPoint != "" && !strings.Contains(client.Config.EndPoint, u.Hostname()) {
+		return sourceLink, nil
+	}
+
+	filename := u.Path
+	if prefixLen := len(client.Config.URLPrefix); len(filename) >= prefixLen {
+		filename = filename[prefixLen:]
+	}
+	if suffixLen := len(client.Config.URLSuffix); len(filename) >= suffixLen {
+		filename = filename[:len(filename)-suffixLen]
+	}
+	filename = strings.Trim(filename, "/")
+	if strings.HasPrefix(filename, client.Config.Bucket) {
+		filename = strings.Trim(filename[len(client.Config.Bucket):], "/")
+	}
+
+	req, err := awss3.NewPresignClient(client.Client).PresignGetObject(ctx, &awss3.GetObjectInput{
+		Bucket: aws.String(client.Config.Bucket),
+		Key:    aws.String(filename),
+	}, awss3.WithPresignExpires(LinkLifetime))
+	if err != nil {
+		return "", errors2.Wrapf(err, "pre-sign link")
+	}
+	return req.URL, nil
+}

store/db/mysql/resource.go

@@ -133,6 +133,9 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
 	if v := update.InternalPath; v != nil {
 		set, args = append(set, "`internal_path` = ?"), append(args, *v)
 	}
+	if v := update.ExternalLink; v != nil {
+		set, args = append(set, "`external_link` = ?"), append(args, *v)
+	}
 	if v := update.MemoID; v != nil {
 		set, args = append(set, "`memo_id` = ?"), append(args, *v)
 	}

store/db/postgres/resource.go

@@ -118,6 +118,9 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
 	if v := update.InternalPath; v != nil {
 		set, args = append(set, "internal_path = "+placeholder(len(args)+1)), append(args, *v)
 	}
+	if v := update.ExternalLink; v != nil {
+		set, args = append(set, "external_link = "+placeholder(len(args)+1)), append(args, *v)
+	}
 	if v := update.MemoID; v != nil {
 		set, args = append(set, "memo_id = "+placeholder(len(args)+1)), append(args, *v)
 	}

store/db/sqlite/resource.go

@@ -114,6 +114,9 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
 	if v := update.InternalPath; v != nil {
 		set, args = append(set, "`internal_path` = ?"), append(args, *v)
 	}
+	if v := update.ExternalLink; v != nil {
+		set, args = append(set, "`external_link` = ?"), append(args, *v)
+	}
 	if v := update.MemoID; v != nil {
 		set, args = append(set, "`memo_id` = ?"), append(args, *v)
 	}

store/resource.go

@@ -53,6 +53,7 @@ type UpdateResource struct {
 	UpdatedTs    *int64
 	Filename     *string
 	InternalPath *string
+	ExternalLink *string
 	MemoID       *int32
 	Blob         []byte
 }

web/src/components/CreateStorageServiceDialog.tsx

@@ -1,5 +1,5 @@
-import { Button, IconButton, Input, Typography } from "@mui/joy";
+import { Button, IconButton, Input, Checkbox, Typography } from "@mui/joy";
-import { useEffect, useState } from "react";
+import React, { useEffect, useState } from "react";
 import { toast } from "react-hot-toast";
 import * as api from "@/helpers/api";
 import { useTranslate } from "@/utils/i18n";
@@ -29,6 +29,7 @@ const CreateStorageServiceDialog: React.FC<Props> = (props: Props) => {
     bucket: "",
     urlPrefix: "",
     urlSuffix: "",
+    presign: false,
   });
   const isCreating = storage === undefined;
 
@@ -220,6 +221,12 @@ const CreateStorageServiceDialog: React.FC<Props> = (props: Props) => {
           onChange={(e) => setPartialS3Config({ urlSuffix: e.target.value })}
           fullWidth
         />
+        <Checkbox
+          className="mb-2"
+          label={t("setting.storage-section.presign-placeholder")}
+          checked={s3Config.presign}
+          onChange={(e) => setPartialS3Config({ presign: e.target.checked })}
+        />
         <div className="mt-2 w-full flex flex-row justify-end items-center space-x-1">
           <Button variant="plain" color="neutral" onClick={handleCloseBtnClick}>
             {t("common.cancel")}

web/src/locales/en.json

@@ -228,7 +228,8 @@
       "url-prefix": "URL prefix",
       "url-prefix-placeholder": "Custom URL prefix, optional",
       "url-suffix": "URL suffix",
-      "url-suffix-placeholder": "Custom URL suffix, optional"
+      "url-suffix-placeholder": "Custom URL suffix, optional",
+      "presign-placeholder": "Pre-sign URL, optional"
     },
     "member-section": {
       "create-a-member": "Create a member",

web/src/locales/ru.json

@@ -302,6 +302,7 @@
       "update-storage": "Обновить хранилище",
       "url-prefix": "Префикс URL",
       "url-prefix-placeholder": "Пользовательский префикс URL, необязательно",
+      "presign-placeholder": "Генерировать временную публичную ссылку, необязательно",
       "url-suffix": "суффикс URL",
       "url-suffix-placeholder": "Пользовательский суффикс URL, необязательно",
       "warning-text": "Вы уверены, что хотите удалить это хранилище?\nЭТО ДЕЙСТВИЕ НЕВОЗМОЖНО ОТМЕНИТЬ❗"

web/src/types/modules/storage.d.ts

@@ -11,6 +11,7 @@ interface StorageS3Config {
   bucket: string;
   urlPrefix: string;
   urlSuffix: string;
+  presign: boolean;
 }
 
 interface StorageConfig {