package cleaner_test import ( "context" "errors" "time" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/util" ) func copyMap(in map[string]*gtsmodel.Emoji) map[string]*gtsmodel.Emoji { out := make(map[string]*gtsmodel.Emoji, len(in)) for k, v1 := range in { v2 := new(gtsmodel.Emoji) *v2 = *v1 out[k] = v2 } return out } func (suite *CleanerTestSuite) TestEmojiUncacheRemote() { suite.testEmojiUncacheRemote( context.Background(), mapvals(suite.emojis), ) } func (suite *CleanerTestSuite) TestEmojiUncacheRemoteDryRun() { suite.testEmojiUncacheRemote( gtscontext.SetDryRun(context.Background()), mapvals(suite.emojis), ) } func (suite *CleanerTestSuite) TestEmojiFixBroken() { suite.testEmojiFixBroken( context.Background(), mapvals(suite.emojis), ) } func (suite *CleanerTestSuite) TestEmojiFixBrokenDryRun() { suite.testEmojiFixBroken( gtscontext.SetDryRun(context.Background()), mapvals(suite.emojis), ) } func (suite *CleanerTestSuite) TestEmojiPruneUnused() { suite.testEmojiPruneUnused( context.Background(), mapvals(suite.emojis), ) } func (suite *CleanerTestSuite) TestEmojiPruneUnusedDryRun() { suite.testEmojiPruneUnused( gtscontext.SetDryRun(context.Background()), mapvals(suite.emojis), ) } func (suite *CleanerTestSuite) TestEmojiFixCacheStates() { // Copy testrig emojis + mark // rainbow emoji as uncached // so there's something to fix. emojis := copyMap(suite.emojis) emojis["rainbow"].Cached = util.Ptr(false) suite.testEmojiFixCacheStates( context.Background(), mapvals(emojis), ) } func (suite *CleanerTestSuite) TestEmojiFixCacheStatesDryRun() { // Copy testrig emojis + mark // rainbow emoji as uncached // so there's something to fix. emojis := copyMap(suite.emojis) emojis["rainbow"].Cached = util.Ptr(false) suite.testEmojiFixCacheStates( gtscontext.SetDryRun(context.Background()), mapvals(emojis), ) } func (suite *CleanerTestSuite) testEmojiUncacheRemote(ctx context.Context, emojis []*gtsmodel.Emoji) { var uncacheIDs []string // Test state. t := suite.T() // Get max remote cache days to keep. days := config.GetMediaRemoteCacheDays() olderThan := time.Now().Add(-24 * time.Hour * time.Duration(days)) for _, emoji := range emojis { // Check whether this emoji should be uncached. ok, err := suite.shouldUncacheEmoji(ctx, emoji, olderThan) if err != nil { t.Fatalf("error checking whether emoji should be uncached: %v", err) } if ok { // Mark this emoji ID as to be uncached. uncacheIDs = append(uncacheIDs, emoji.ID) } } // Attempt to uncache remote emojis. found, err := suite.cleaner.Emoji().UncacheRemote(ctx, olderThan) if err != nil { t.Errorf("error uncaching remote emojis: %v", err) return } // Check expected were uncached. if found != len(uncacheIDs) { t.Errorf("expected %d emojis to be uncached, %d were", len(uncacheIDs), found) return } if gtscontext.DryRun(ctx) { // nothing else to test. return } for _, id := range uncacheIDs { // Fetch the emoji by ID that should now be uncached. emoji, err := suite.state.DB.GetEmojiByID(ctx, id) if err != nil { t.Fatalf("error fetching emoji from database: %v", err) } // Check cache state. if *emoji.Cached { t.Errorf("emoji %s@%s should have been uncached", emoji.Shortcode, emoji.Domain) } // Check that the emoji files in storage have been deleted. if ok, err := suite.state.Storage.Has(ctx, emoji.ImagePath); err != nil { t.Fatalf("error checking storage for emoji: %v", err) } else if ok { t.Errorf("emoji %s@%s image path should not exist", emoji.Shortcode, emoji.Domain) } else if ok, err := suite.state.Storage.Has(ctx, emoji.ImageStaticPath); err != nil { t.Fatalf("error checking storage for emoji: %v", err) } else if ok { t.Errorf("emoji %s@%s image static path should not exist", emoji.Shortcode, emoji.Domain) } } } func (suite *CleanerTestSuite) shouldUncacheEmoji(ctx context.Context, emoji *gtsmodel.Emoji, after time.Time) (bool, error) { if emoji.ImageRemoteURL == "" { // Local emojis are never uncached. return false, nil } if emoji.Cached == nil || !*emoji.Cached { // Emoji is already uncached. return false, nil } // Get related accounts using this emoji (if any). accounts, err := suite.state.DB.GetAccountsUsingEmoji(ctx, emoji.ID) if err != nil { return false, err } // Check if accounts are recently updated. for _, account := range accounts { if account.FetchedAt.After(after) { return false, nil } } // Get related statuses using this emoji (if any). statuses, err := suite.state.DB.GetStatusesUsingEmoji(ctx, emoji.ID) if err != nil { return false, err } // Check if statuses are recently updated. for _, status := range statuses { if status.FetchedAt.After(after) { return false, nil } } return true, nil } func (suite *CleanerTestSuite) testEmojiFixBroken(ctx context.Context, emojis []*gtsmodel.Emoji) { var fixIDs []string // Test state. t := suite.T() for _, emoji := range emojis { // Check whether this emoji should be fixed. ok, err := suite.shouldFixBrokenEmoji(ctx, emoji) if err != nil { t.Fatalf("error checking whether emoji should be fixed: %v", err) } if ok { // Mark this emoji ID as to be fixed. fixIDs = append(fixIDs, emoji.ID) } } // Attempt to fix broken emojis. found, err := suite.cleaner.Emoji().FixBroken(ctx) if err != nil { t.Errorf("error fixing broken emojis: %v", err) return } // Check expected were fixed. if found != len(fixIDs) { t.Errorf("expected %d emojis to be fixed, %d were", len(fixIDs), found) return } if gtscontext.DryRun(ctx) { // nothing else to test. return } for _, id := range fixIDs { // Fetch the emoji by ID that should now be fixed. emoji, err := suite.state.DB.GetEmojiByID(ctx, id) if err != nil { t.Fatalf("error fetching emoji from database: %v", err) } // Ensure category was cleared. if emoji.CategoryID != "" { t.Errorf("emoji %s@%s should have empty category", emoji.Shortcode, emoji.Domain) } } } func (suite *CleanerTestSuite) shouldFixBrokenEmoji(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { if emoji.CategoryID == "" { // no category issue. return false, nil } // Get the related category for this emoji. category, err := suite.state.DB.GetEmojiCategory(ctx, emoji.CategoryID) if err != nil && !errors.Is(err, db.ErrNoEntries) { return false, nil } return (category == nil), nil } func (suite *CleanerTestSuite) testEmojiPruneUnused(ctx context.Context, emojis []*gtsmodel.Emoji) { var pruneIDs []string // Test state. t := suite.T() for _, emoji := range emojis { // Check whether this emoji should be pruned. ok, err := suite.shouldPruneEmoji(ctx, emoji) if err != nil { t.Fatalf("error checking whether emoji should be pruned: %v", err) } if ok { // Mark this emoji ID as to be pruned. pruneIDs = append(pruneIDs, emoji.ID) } } // Attempt to prune emojis. found, err := suite.cleaner.Emoji().PruneUnused(ctx) if err != nil { t.Errorf("error fixing broken emojis: %v", err) return } // Check expected were pruned. if found != len(pruneIDs) { t.Errorf("expected %d emojis to be pruned, %d were", len(pruneIDs), found) return } if gtscontext.DryRun(ctx) { // nothing else to test. return } for _, id := range pruneIDs { // Fetch the emoji by ID that should now be pruned. emoji, err := suite.state.DB.GetEmojiByID(ctx, id) if err != nil && !errors.Is(err, db.ErrNoEntries) { t.Fatalf("error fetching emoji from database: %v", err) } // Ensure gone. if emoji != nil { t.Errorf("emoji %s@%s should have been pruned", emoji.Shortcode, emoji.Domain) } } } func (suite *CleanerTestSuite) shouldPruneEmoji(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { if emoji.ImageRemoteURL == "" { // Local emojis are never pruned. return false, nil } // Get related accounts using this emoji (if any). accounts, err := suite.state.DB.GetAccountsUsingEmoji(ctx, emoji.ID) if err != nil { return false, err } else if len(accounts) > 0 { return false, nil } // Get related statuses using this emoji (if any). statuses, err := suite.state.DB.GetStatusesUsingEmoji(ctx, emoji.ID) if err != nil { return false, err } else if len(statuses) > 0 { return false, nil } return true, nil } func (suite *CleanerTestSuite) testEmojiFixCacheStates(ctx context.Context, emojis []*gtsmodel.Emoji) { var fixIDs []string // Test state. t := suite.T() for _, emoji := range emojis { // Check whether this emoji should be fixed. ok, err := suite.shouldFixEmojiCacheState(ctx, emoji) if err != nil { t.Fatalf("error checking whether emoji should be fixed: %v", err) } if ok { // Mark this emoji ID as to be fixed. fixIDs = append(fixIDs, emoji.ID) } } // Attempt to fix broken emoji cache states. found, err := suite.cleaner.Emoji().FixCacheStates(ctx) if err != nil { t.Errorf("error fixing broken emojis: %v", err) return } // Check expected were fixed. if found != len(fixIDs) { t.Errorf("expected %d emojis to be fixed, %d were", len(fixIDs), found) return } if gtscontext.DryRun(ctx) { // nothing else to test. return } for _, id := range fixIDs { // Fetch the emoji by ID that should now be fixed. emoji, err := suite.state.DB.GetEmojiByID(ctx, id) if err != nil { t.Fatalf("error fetching emoji from database: %v", err) } // Ensure emoji cache state has been fixed. ok, err := suite.shouldFixEmojiCacheState(ctx, emoji) if err != nil { t.Fatalf("error checking whether emoji should be fixed: %v", err) } else if ok { t.Errorf("emoji %s@%s cache state should have been fixed", emoji.Shortcode, emoji.Domain) } } } func (suite *CleanerTestSuite) shouldFixEmojiCacheState(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { // Check whether emoji image path exists. haveImage, err := suite.state.Storage.Has(ctx, emoji.ImagePath) if err != nil { return false, err } // Check whether emoji static path exists. haveStatic, err := suite.state.Storage.Has(ctx, emoji.ImageStaticPath) if err != nil { return false, err } switch exists := (haveImage && haveStatic); { case emoji.Cached != nil && *emoji.Cached && !exists: // (cached can be nil in tests) // Cached but missing files. return true, nil case emoji.Cached != nil && !*emoji.Cached && exists: // (cached can be nil in tests) // Uncached but unexpected files. return true, nil default: // No cache state issue. return false, nil } }