YourSelfHosted/memos
1
0
Fork 0
mirror of https://github.com/usememos/memos.git synced 2025-06-05 22:09:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: implement s3 storage

Browse Source
This commit is contained in:
Steven
2024-05-02 21:28:06 +08:00
parent 355ea352aa
commit 26545c855c
55 changed files with 842 additions and 858 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
store/db/postgres/inbox.go
View File

@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"strings"
"github.com/pkg/errors"
@@ -133,13 +132,3 @@ func (d *DB) DeleteInbox(ctx context.Context, delete *store.DeleteInbox) error {
}
return nil
}
func vacuumInbox(ctx context.Context, tx *sql.Tx) error {
stmt := `DELETE FROM inbox WHERE sender_id NOT IN (SELECT id FROM "user")`
_, err := tx.ExecContext(ctx, stmt)
if err != nil {
return err
}
return nil
}

7
store/db/postgres/memo.go
View File

@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
@@ -203,9 +202,3 @@ func (d *DB) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error {
}
return nil
}
func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
stmt := `DELETE FROM memo WHERE creator_id NOT IN (SELECT id FROM "user")`
_, err := tx.ExecContext(ctx, stmt)
return err
}

16
store/db/postgres/memo_organizer.go
View File

@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
@@ -90,18 +89,3 @@ func (d *DB) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMemoOr
}
return nil
}
func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
stmt := `
DELETE FROM
memo_organizer
WHERE
memo_id NOT IN (SELECT id FROM memo)
OR user_id NOT IN (SELECT id FROM "user")`
_, err := tx.ExecContext(ctx, stmt)
if err != nil {
return err
}
return nil
}

11
store/db/postgres/memo_relation.go
View File

@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"strings"
"github.com/usememos/memos/store"
@@ -101,13 +100,3 @@ func (d *DB) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMemoRel
}
return nil
}
func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
if _, err := tx.ExecContext(ctx, `
DELETE FROM memo_relation
WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
`); err != nil {
return err
}
return nil
}

15
store/db/postgres/migration/dev/LATEST__SCHEMA.sql
View File

@@ -71,11 +71,12 @@ CREATE TABLE resource (
updated_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
filename TEXT NOT NULL,
blob BYTEA,
external_link TEXT NOT NULL,
type TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
internal_path TEXT NOT NULL DEFAULT '',
memo_id INTEGER DEFAULT NULL
memo_id INTEGER DEFAULT NULL,
storage_type TEXT NOT NULL DEFAULT '',
reference TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL DEFAULT '{}'
);
-- tag
@@ -95,14 +96,6 @@ CREATE TABLE activity (
payload JSONB NOT NULL DEFAULT '{}'
);
-- storage
CREATE TABLE storage (
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
config JSONB NOT NULL DEFAULT '{}'
);
-- idp
CREATE TABLE idp (
id SERIAL PRIMARY KEY,

16
store/db/postgres/migration/prod/0.22/00__resource_storage_type.sql Normal file
View File

@@ -0,0 +1,16 @@
ALTER TABLE `resource`
ADD COLUMN `storage_type` VARCHAR(256) NOT NULL DEFAULT '',
ADD COLUMN `reference` VARCHAR(256) NOT NULL DEFAULT '',
ADD COLUMN `payload` TEXT NOT NULL;
UPDATE `resource`
SET `storage_type` = 'LOCAL', `reference` = `internal_path`
WHERE `internal_path` IS NOT NULL AND `internal_path` != '';
UPDATE `resource`
SET `storage_type` = 'EXTERNAL', `reference` = `external_link`
WHERE `external_link` IS NOT NULL AND `external_link` != '';
ALTER TABLE `resource`
DROP COLUMN `internal_path`,
DROP COLUMN `external_link`;

33
store/db/postgres/postgres.go
View File

@@ -44,39 +44,6 @@ func (d *DB) GetDB() *sql.DB {
return d.db
}
func (d *DB) Vacuum(ctx context.Context) error {
tx, err := d.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := vacuumMemo(ctx, tx); err != nil {
return err
}
if err := vacuumResource(ctx, tx); err != nil {
return err
}
if err := vacuumUserSetting(ctx, tx); err != nil {
return err
}
if err := vacuumMemoOrganizer(ctx, tx); err != nil {
return err
}
if err := vacuumMemoRelations(ctx, tx); err != nil {
return err
}
if err := vacuumInbox(ctx, tx); err != nil {
return err
}
if err := vacuumTag(ctx, tx); err != nil {
// Prevent revive warning.
return err
}
return tx.Commit()
}
func (*DB) GetCurrentDBSize(context.Context) (int64, error) {
return 0, errors.New("unimplemented")
}

84
store/db/postgres/resource.go
View File

@@ -6,12 +6,28 @@ import (
"fmt"
"strings"
"github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
fields := []string{"uid", "filename", "blob", "external_link", "type", "size", "creator_id", "internal_path", "memo_id"}
args := []any{create.UID, create.Filename, create.Blob, create.ExternalLink, create.Type, create.Size, create.CreatorID, create.InternalPath, create.MemoID}
fields := []string{"uid", "filename", "blob", "type", "size", "creator_id", "memo_id", "storage_type", "reference", "payload"}
storageType := ""
if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
storageType = create.StorageType.String()
}
payloadString := "{}"
if create.Payload != nil {
bytes, err := protojson.Marshal(create.Payload)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal resource payload")
}
payloadString = string(bytes)
}
args := []any{create.UID, create.Filename, create.Blob, create.Type, create.Size, create.CreatorID, create.MemoID, storageType, create.Reference, payloadString}
stmt := "INSERT INTO resource (" + strings.Join(fields, ", ") + ") VALUES (" + placeholders(len(args)) + ") RETURNING id, created_ts, updated_ts"
if err := d.db.QueryRowContext(ctx, stmt, args...).Scan(&create.ID, &create.CreatedTs, &create.UpdatedTs); err != nil {
@@ -42,7 +58,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
where = append(where, "memo_id IS NOT NULL")
}
fields := []string{"id", "uid", "filename", "external_link", "type", "size", "creator_id", "created_ts", "updated_ts", "internal_path", "memo_id"}
fields := []string{"id", "uid", "filename", "type", "size", "creator_id", "created_ts", "updated_ts", "memo_id", "storage_type", "reference", "payload"}
if find.GetBlob {
fields = append(fields, "blob")
}
@@ -52,7 +68,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
%s
FROM resource
WHERE %s
ORDER BY updated_ts DESC, created_ts DESC
ORDER BY created_ts DESC
`, strings.Join(fields, ", "), strings.Join(where, " AND "))
if find.Limit != nil {
query = fmt.Sprintf("%s LIMIT %d", query, *find.Limit)
@@ -71,18 +87,21 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
for rows.Next() {
resource := store.Resource{}
var memoID sql.NullInt32
var storageType string
var payloadBytes []byte
dests := []any{
&resource.ID,
&resource.UID,
&resource.Filename,
&resource.ExternalLink,
&resource.Type,
&resource.Size,
&resource.CreatorID,
&resource.CreatedTs,
&resource.UpdatedTs,
&resource.InternalPath,
&memoID,
&storageType,
&resource.Reference,
&payloadBytes,
}
if find.GetBlob {
dests = append(dests, &resource.Blob)
@@ -90,9 +109,16 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
if err := rows.Scan(dests...); err != nil {
return nil, err
}
if memoID.Valid {
resource.MemoID = &memoID.Int32
}
resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
payload := &storepb.ResourcePayload{}
if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
return nil, err
}
resource.Payload = payload
list = append(list, &resource)
}
@@ -103,7 +129,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
return list, nil
}
func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (*store.Resource, error) {
func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
set, args := []string{}, []any{}
if v := update.UID; v != nil {
@@ -115,40 +141,20 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
if v := update.Filename; v != nil {
set, args = append(set, "filename = "+placeholder(len(args)+1)), append(args, *v)
}
if v := update.InternalPath; v != nil {
set, args = append(set, "internal_path = "+placeholder(len(args)+1)), append(args, *v)
}
if v := update.ExternalLink; v != nil {
set, args = append(set, "external_link = "+placeholder(len(args)+1)), append(args, *v)
}
if v := update.MemoID; v != nil {
set, args = append(set, "memo_id = "+placeholder(len(args)+1)), append(args, *v)
}
if v := update.Blob; v != nil {
set, args = append(set, "blob = "+placeholder(len(args)+1)), append(args, v)
}
fields := []string{"id", "uid", "filename", "external_link", "type", "size", "creator_id", "created_ts", "updated_ts", "internal_path"}
stmt := `UPDATE resource SET ` + strings.Join(set, ", ") + ` WHERE id = ` + placeholder(len(args)+1) + ` RETURNING ` + strings.Join(fields, ", ")
stmt := `UPDATE resource SET ` + strings.Join(set, ", ") + ` WHERE id = ` + placeholder(len(args)+1)
args = append(args, update.ID)
resource := store.Resource{}
dests := []any{
&resource.ID,
&resource.UID,
&resource.Filename,
&resource.ExternalLink,
&resource.Type,
&resource.Size,
&resource.CreatorID,
&resource.CreatedTs,
&resource.UpdatedTs,
&resource.InternalPath,
result, err := d.db.ExecContext(ctx, stmt, args...)
if err != nil {
return err
}
if err := d.db.QueryRowContext(ctx, stmt, args...).Scan(dests...); err != nil {
return nil, err
if _, err := result.RowsAffected(); err != nil {
return err
}
return &resource, nil
return nil
}
func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
@@ -162,13 +168,3 @@ func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) e
}
return nil
}
func vacuumResource(ctx context.Context, tx *sql.Tx) error {
stmt := `DELETE FROM resource WHERE creator_id NOT IN (SELECT id FROM "user")`
_, err := tx.ExecContext(ctx, stmt)
if err != nil {
return err
}
return nil
}

11
store/db/postgres/tag.go
View File

@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"strings"
"github.com/usememos/memos/store"
@@ -69,13 +68,3 @@ func (d *DB) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {
}
return nil
}
func vacuumTag(ctx context.Context, tx *sql.Tx) error {
stmt := `DELETE FROM tag WHERE creator_id NOT IN (SELECT id FROM "user")`
_, err := tx.ExecContext(ctx, stmt)
if err != nil {
return err
}
return nil
}

6
store/db/postgres/user.go
View File

@@ -165,11 +165,5 @@ func (d *DB) DeleteUser(ctx context.Context, delete *store.DeleteUser) error {
if _, err := result.RowsAffected(); err != nil {
return err
}
if err := d.Vacuum(ctx); err != nil {
// Prevent linter warning.
return err
}
return nil
}

11
store/db/postgres/user_setting.go
View File

@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"strings"
storepb "github.com/usememos/memos/proto/gen/store"
@@ -68,13 +67,3 @@ func (d *DB) ListUserSettings(ctx context.Context, find *store.FindUserSetting)
return userSettingList, nil
}
func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
stmt := `DELETE FROM user_setting WHERE user_id NOT IN (SELECT id FROM "user")`
_, err := tx.ExecContext(ctx, stmt)
if err != nil {
return err
}
return nil
}