diff --git a/server/router/api/v1/memo_service.go b/server/router/api/v1/memo_service.go index d51eabe3..8e552272 100644 --- a/server/router/api/v1/memo_service.go +++ b/server/router/api/v1/memo_service.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "log/slog" - "slices" "time" "unicode/utf8" @@ -28,6 +27,7 @@ import ( "github.com/usememos/memos/plugin/webhook" v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" + memoproperty "github.com/usememos/memos/server/runner/memo_property" "github.com/usememos/memos/store" ) @@ -60,8 +60,9 @@ func (s *APIV1Service) CreateMemo(ctx context.Context, request *v1pb.CreateMemoR } if len(create.Content) > contentLengthLimit { return nil, status.Errorf(codes.InvalidArgument, "content too long (max %d characters)", contentLengthLimit) + } - property, err := getMemoPropertyFromContent(create.Content) + property, err := memoproperty.GetMemoPropertyFromContent(create.Content) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err) } @@ -247,7 +248,7 @@ func (s *APIV1Service) UpdateMemo(ctx context.Context, request *v1pb.UpdateMemoR } update.Content = &request.Memo.Content - property, err := getMemoPropertyFromContent(*update.Content) + property, err := memoproperty.GetMemoPropertyFromContent(*update.Content) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err) } @@ -610,7 +611,7 @@ func (s *APIV1Service) RebuildMemoProperty(ctx context.Context, request *v1pb.Re } for _, memo := range memos { - property, err := getMemoPropertyFromContent(memo.Content) + property, err := memoproperty.GetMemoPropertyFromContent(memo.Content) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err) } @@ -691,14 +692,14 @@ func (s *APIV1Service) RenameMemoTag(ctx context.Context, request *v1pb.RenameMe if err != nil { return nil, status.Errorf(codes.Internal, "failed to parse memo: %v", err) } - TraverseASTNodes(nodes, func(node ast.Node) { + memoproperty.TraverseASTNodes(nodes, func(node ast.Node) { if tag, ok := node.(*ast.Tag); ok && tag.Content == request.OldTag { tag.Content = request.NewTag } }) content := restore.Restore(nodes) - property, err := getMemoPropertyFromContent(content) + property, err := memoproperty.GetMemoPropertyFromContent(content) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err) } @@ -1127,56 +1128,6 @@ func findMemoField(callExpr *expr.Expr_Call, filter *MemoFilter) { } } -func getMemoPropertyFromContent(content string) (*storepb.MemoPayload_Property, error) { - nodes, err := parser.Parse(tokenizer.Tokenize(content)) - if err != nil { - return nil, errors.Wrap(err, "failed to parse content") - } - - property := &storepb.MemoPayload_Property{} - TraverseASTNodes(nodes, func(node ast.Node) { - switch n := node.(type) { - case *ast.Tag: - tag := n.Content - if !slices.Contains(property.Tags, tag) { - property.Tags = append(property.Tags, tag) - } - case *ast.Link, *ast.AutoLink: - property.HasLink = true - case *ast.TaskList: - property.HasTaskList = true - if !n.Complete { - property.HasIncompleteTasks = true - } - case *ast.Code, *ast.CodeBlock: - property.HasCode = true - } - }) - return property, nil -} - -func TraverseASTNodes(nodes []ast.Node, fn func(ast.Node)) { - for _, node := range nodes { - fn(node) - switch n := node.(type) { - case *ast.Paragraph: - TraverseASTNodes(n.Children, fn) - case *ast.Heading: - TraverseASTNodes(n.Children, fn) - case *ast.Blockquote: - TraverseASTNodes(n.Children, fn) - case *ast.OrderedList: - TraverseASTNodes(n.Children, fn) - case *ast.UnorderedList: - TraverseASTNodes(n.Children, fn) - case *ast.TaskList: - TraverseASTNodes(n.Children, fn) - case *ast.Bold: - TraverseASTNodes(n.Children, fn) - } - } -} - // DispatchMemoCreatedWebhook dispatches webhook when memo is created. func (s *APIV1Service) DispatchMemoCreatedWebhook(ctx context.Context, memo *v1pb.Memo) error { return s.dispatchMemoRelatedWebhook(ctx, memo, "memos.memo.created") diff --git a/server/runner/memo_property/runner.go b/server/runner/memo_property/runner.go new file mode 100644 index 00000000..8801e891 --- /dev/null +++ b/server/runner/memo_property/runner.go @@ -0,0 +1,120 @@ +package memoproperty + +import ( + "context" + "log/slog" + "slices" + "time" + + "github.com/pkg/errors" + "github.com/usememos/gomark/ast" + "github.com/usememos/gomark/parser" + "github.com/usememos/gomark/parser/tokenizer" + storepb "github.com/usememos/memos/proto/gen/store" + "github.com/usememos/memos/store" +) + +type Runner struct { + Store *store.Store +} + +func NewRunner(store *store.Store) *Runner { + return &Runner{ + Store: store, + } +} + +// Schedule runner every 12 hours. +const runnerInterval = time.Hour * 12 + +func (r *Runner) Run(ctx context.Context) { + ticker := time.NewTicker(runnerInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + r.RunOnce(ctx) + case <-ctx.Done(): + return + } + } +} + +func (r *Runner) RunOnce(ctx context.Context) { + emptyPayload := "{}" + memos, err := r.Store.ListMemos(ctx, &store.FindMemo{ + PayloadFind: &store.FindMemoPayload{ + Raw: &emptyPayload, + }, + }) + if err != nil { + slog.Error("failed to list memos", "err", err) + return + } + + for _, memo := range memos { + property, err := GetMemoPropertyFromContent(memo.Content) + if err != nil { + slog.Error("failed to get memo property", "err", err) + continue + } + memo.Payload.Property = property + if err := r.Store.UpdateMemo(ctx, &store.UpdateMemo{ + ID: memo.ID, + Payload: memo.Payload, + }); err != nil { + slog.Error("failed to update memo", "err", err) + } + } +} + +func GetMemoPropertyFromContent(content string) (*storepb.MemoPayload_Property, error) { + nodes, err := parser.Parse(tokenizer.Tokenize(content)) + if err != nil { + return nil, errors.Wrap(err, "failed to parse content") + } + + property := &storepb.MemoPayload_Property{} + TraverseASTNodes(nodes, func(node ast.Node) { + switch n := node.(type) { + case *ast.Tag: + tag := n.Content + if !slices.Contains(property.Tags, tag) { + property.Tags = append(property.Tags, tag) + } + case *ast.Link, *ast.AutoLink: + property.HasLink = true + case *ast.TaskList: + property.HasTaskList = true + if !n.Complete { + property.HasIncompleteTasks = true + } + case *ast.Code, *ast.CodeBlock: + property.HasCode = true + } + }) + return property, nil +} + +func TraverseASTNodes(nodes []ast.Node, fn func(ast.Node)) { + for _, node := range nodes { + fn(node) + switch n := node.(type) { + case *ast.Paragraph: + TraverseASTNodes(n.Children, fn) + case *ast.Heading: + TraverseASTNodes(n.Children, fn) + case *ast.Blockquote: + TraverseASTNodes(n.Children, fn) + case *ast.OrderedList: + TraverseASTNodes(n.Children, fn) + case *ast.UnorderedList: + TraverseASTNodes(n.Children, fn) + case *ast.TaskList: + TraverseASTNodes(n.Children, fn) + case *ast.Bold: + TraverseASTNodes(n.Children, fn) + } + } +} diff --git a/server/service/s3_object_presigner/s3_object_presigner.go b/server/runner/s3_presign/runner.go similarity index 74% rename from server/service/s3_object_presigner/s3_object_presigner.go rename to server/runner/s3_presign/runner.go index f4873ee0..342b26c5 100644 --- a/server/service/s3_object_presigner/s3_object_presigner.go +++ b/server/runner/s3_presign/runner.go @@ -1,4 +1,4 @@ -package s3objectpresigner +package s3presign import ( "context" @@ -12,25 +12,45 @@ import ( "github.com/usememos/memos/store" ) -// nolint -type S3ObjectPresigner struct { +type Runner struct { Store *store.Store } -func NewS3ObjectPresigner(store *store.Store) *S3ObjectPresigner { - return &S3ObjectPresigner{ +func NewRunner(store *store.Store) *Runner { + return &Runner{ Store: store, } } -func (p *S3ObjectPresigner) CheckAndPresign(ctx context.Context) { - workspaceStorageSetting, err := p.Store.GetWorkspaceStorageSetting(ctx) +// Schedule runner every 12 hours. +const runnerInterval = time.Hour * 12 + +func (r *Runner) Run(ctx context.Context) { + ticker := time.NewTicker(runnerInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + r.RunOnce(ctx) + case <-ctx.Done(): + return + } + } +} + +func (r *Runner) RunOnce(ctx context.Context) { + r.CheckAndPresign(ctx) +} + +func (r *Runner) CheckAndPresign(ctx context.Context) { + workspaceStorageSetting, err := r.Store.GetWorkspaceStorageSetting(ctx) if err != nil { return } s3StorageType := storepb.ResourceStorageType_S3 - resources, err := p.Store.ListResources(ctx, &store.FindResource{ + resources, err := r.Store.ListResources(ctx, &store.FindResource{ GetBlob: false, StorageType: &s3StorageType, }) @@ -73,7 +93,7 @@ func (p *S3ObjectPresigner) CheckAndPresign(ctx context.Context) { } s3ObjectPayload.S3Config = s3Config s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now()) - if err := p.Store.UpdateResource(ctx, &store.UpdateResource{ + if err := r.Store.UpdateResource(ctx, &store.UpdateResource{ ID: resource.ID, Reference: &presignURL, Payload: &storepb.ResourcePayload{ @@ -86,21 +106,3 @@ func (p *S3ObjectPresigner) CheckAndPresign(ctx context.Context) { } } } - -func (p *S3ObjectPresigner) Start(ctx context.Context) { - p.CheckAndPresign(ctx) - - // Schedule runner every 24 hours. - ticker := time.NewTicker(24 * time.Hour) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - - p.CheckAndPresign(ctx) - } -} diff --git a/server/service/version_checker/version_checker.go b/server/runner/version/runner.go similarity index 72% rename from server/service/version_checker/version_checker.go rename to server/runner/version/runner.go index 40e94a74..b770df6f 100644 --- a/server/service/version_checker/version_checker.go +++ b/server/runner/version/runner.go @@ -1,4 +1,5 @@ -package versionchecker +// Packge version provides a runner to check the latest version of the application. +package version import ( "bytes" @@ -16,50 +17,50 @@ import ( "github.com/usememos/memos/store" ) -// nolint -type VersionChecker struct { +type Runner struct { Store *store.Store Profile *profile.Profile } -func NewVersionChecker(store *store.Store, profile *profile.Profile) *VersionChecker { - return &VersionChecker{ +func NewRunner(store *store.Store, profile *profile.Profile) *Runner { + return &Runner{ Store: store, Profile: profile, } } -func (*VersionChecker) GetLatestVersion() (string, error) { - response, err := http.Get("https://www.usememos.com/api/version") - if err != nil { - return "", errors.Wrap(err, "failed to make http request") - } - defer response.Body.Close() +// Schedule checker every 8 hours. +const runnerInterval = time.Hour * 8 - buf := &bytes.Buffer{} - _, err = buf.ReadFrom(response.Body) - if err != nil { - return "", errors.Wrap(err, "fail to read response body") - } +func (r *Runner) Run(ctx context.Context) { + ticker := time.NewTicker(runnerInterval) + defer ticker.Stop() - version := "" - if err = json.Unmarshal(buf.Bytes(), &version); err != nil { - return "", errors.Wrap(err, "fail to unmarshal get version response") + for { + select { + case <-ticker.C: + r.RunOnce(ctx) + case <-ctx.Done(): + return + } } - return version, nil } -func (c *VersionChecker) Check(ctx context.Context) { - latestVersion, err := c.GetLatestVersion() +func (r *Runner) RunOnce(ctx context.Context) { + r.Check(ctx) +} + +func (r *Runner) Check(ctx context.Context) { + latestVersion, err := r.GetLatestVersion() if err != nil { return } - if !version.IsVersionGreaterThan(latestVersion, version.GetCurrentVersion(c.Profile.Mode)) { + if !version.IsVersionGreaterThan(latestVersion, version.GetCurrentVersion(r.Profile.Mode)) { return } versionUpdateActivityType := store.ActivityTypeVersionUpdate - list, err := c.Store.ListActivities(ctx, &store.FindActivity{ + list, err := r.Store.ListActivities(ctx, &store.FindActivity{ Type: &versionUpdateActivityType, }) if err != nil { @@ -89,12 +90,12 @@ func (c *VersionChecker) Check(ctx context.Context) { }, }, } - if _, err := c.Store.CreateActivity(ctx, activity); err != nil { + if _, err := r.Store.CreateActivity(ctx, activity); err != nil { return } hostUserRole := store.RoleHost - users, err := c.Store.ListUsers(ctx, &store.FindUser{ + users, err := r.Store.ListUsers(ctx, &store.FindUser{ Role: &hostUserRole, }) if err != nil { @@ -105,7 +106,7 @@ func (c *VersionChecker) Check(ctx context.Context) { } hostUser := users[0] - if _, err := c.Store.CreateInbox(ctx, &store.Inbox{ + if _, err := r.Store.CreateInbox(ctx, &store.Inbox{ SenderID: store.SystemBotID, ReceiverID: hostUser.ID, Status: store.UNREAD, @@ -118,20 +119,22 @@ func (c *VersionChecker) Check(ctx context.Context) { } } -func (c *VersionChecker) Start(ctx context.Context) { - c.Check(ctx) - - // Schedule checker every 8 hours. - ticker := time.NewTicker(8 * time.Hour) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - - c.Check(ctx) +func (*Runner) GetLatestVersion() (string, error) { + response, err := http.Get("https://www.usememos.com/api/version") + if err != nil { + return "", errors.Wrap(err, "failed to make http request") } + defer response.Body.Close() + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(response.Body) + if err != nil { + return "", errors.Wrap(err, "fail to read response body") + } + + version := "" + if err = json.Unmarshal(buf.Bytes(), &version); err != nil { + return "", errors.Wrap(err, "fail to unmarshal get version response") + } + return version, nil } diff --git a/server/service/version_checker/version_checker_test.go b/server/runner/version/runner_test.go similarity index 62% rename from server/service/version_checker/version_checker_test.go rename to server/runner/version/runner_test.go index 19a7dc35..25824aab 100644 --- a/server/service/version_checker/version_checker_test.go +++ b/server/runner/version/runner_test.go @@ -1,4 +1,4 @@ -package versionchecker +package version import ( "testing" @@ -7,6 +7,6 @@ import ( ) func TestGetLatestVersion(t *testing.T) { - _, err := NewVersionChecker(nil, nil).GetLatestVersion() + _, err := NewRunner(nil, nil).GetLatestVersion() require.NoError(t, err) } diff --git a/server/server.go b/server/server.go index cf5854bb..4e7ee6da 100644 --- a/server/server.go +++ b/server/server.go @@ -21,8 +21,9 @@ import ( apiv1 "github.com/usememos/memos/server/router/api/v1" "github.com/usememos/memos/server/router/frontend" "github.com/usememos/memos/server/router/rss" - s3objectpresigner "github.com/usememos/memos/server/service/s3_object_presigner" - versionchecker "github.com/usememos/memos/server/service/version_checker" + memoproperty "github.com/usememos/memos/server/runner/memo_property" + s3presign "github.com/usememos/memos/server/runner/s3_presign" + "github.com/usememos/memos/server/runner/version" "github.com/usememos/memos/store" ) @@ -140,8 +141,16 @@ func (s *Server) Shutdown(ctx context.Context) { } func (s *Server) StartBackgroundRunners(ctx context.Context) { - go versionchecker.NewVersionChecker(s.Store, s.Profile).Start(ctx) - go s3objectpresigner.NewS3ObjectPresigner(s.Store).Start(ctx) + s3presignRunner := s3presign.NewRunner(s.Store) + s3presignRunner.RunOnce(ctx) + versionRunner := version.NewRunner(s.Store, s.Profile) + versionRunner.RunOnce(ctx) + memopropertyRunner := memoproperty.NewRunner(s.Store) + memopropertyRunner.RunOnce(ctx) + + go s3presignRunner.Run(ctx) + go versionRunner.Run(ctx) + go memopropertyRunner.Run(ctx) } func (s *Server) getOrUpsertWorkspaceBasicSetting(ctx context.Context) (*storepb.WorkspaceBasicSetting, error) {