From 1c3da007fd3e52335267cdcd053b5c975f7d3bc5 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 24 Dec 2022 18:44:41 -0500 Subject: [PATCH] don't retrieve deleted users --- src/BirdsiteLive.Domain/MigrationService.cs | 2 + .../TwitterAccountModerationProcessor.cs | 2 +- .../RetrieveTwitterUsersProcessor.cs | 2 +- .../TwitterUserPostgresDal.cs | 14 ++-- .../Contracts/ITwitterUserDal.cs | 4 +- .../TwitterUserPostgresDalTests.cs | 71 +++++++++++++++++-- .../TwitterAccountModerationProcessorTests.cs | 8 +-- .../RetrieveTwitterUsersProcessorTests.cs | 15 ++-- 8 files changed, 93 insertions(+), 25 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index e7e9ceb..844cedf 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -117,6 +117,7 @@ namespace BirdsiteLive.Domain twitterAccount.MovedTo = validatedUser.ObjectId; twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + twitterAccount.LastSync = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers @@ -180,6 +181,7 @@ namespace BirdsiteLive.Domain } twitterAccount.Deleted = true; + twitterAccount.LastSync = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers diff --git a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs index 91e3931..2f4d50e 100644 --- a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs +++ b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs @@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors { if (type == ModerationTypeEnum.None) return; - var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(); + var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false); foreach (var user in twitterUsers) { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index 973b672..d9d0ffb 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors try { var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); - var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 92dd41f..d542a76 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -69,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -82,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetFailingTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -93,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync(int maxNumber) + public async Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; using (var dbConnection = Connection) { @@ -106,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; + if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}"; using (var dbConnection = Connection) { diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index 46a59e4..0c58881 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -10,8 +10,8 @@ namespace BirdsiteLive.DAL.Contracts string movedToAcct = null); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); - Task GetAllTwitterUsersAsync(int maxNumber); - Task GetAllTwitterUsersAsync(); + Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser); + Task GetAllTwitterUsersAsync(bool retrieveDisabledUser); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 3149cc2..936bb73 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -348,12 +348,71 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); } - var result = await dal.GetAllTwitterUsersAsync(1100); + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].LastTweetPostedId == default); Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var user in result) + { + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo)); + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct)); + Assert.IsFalse(user.Deleted); + } + } + + [TestMethod] + public async Task GetAllTwitterUsers_Top_RetrieveDeleted() + { + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 1000; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, true); + Assert.AreEqual(1020, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } [TestMethod] @@ -371,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers // Update accounts var now = DateTime.UtcNow; - var allUsers = await dal.GetAllTwitterUsersAsync(); + var allUsers = await dal.GetAllTwitterUsersAsync(false); foreach (var acc in allUsers) { var lastSync = now.AddDays(acc.LastTweetPostedId); @@ -382,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers // Create a not init account await dal.CreateTwitterUserAsync("not_init", -1); - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.IsTrue(result.Any(x => x.Acct == "myid0")); Assert.IsTrue(result.Any(x => x.Acct == "myid8")); @@ -405,7 +464,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var allUsers = await dal.GetAllTwitterUsersAsync(100); + var allUsers = await dal.GetAllTwitterUsersAsync(100, false); for (var i = 0; i < 20; i++) { var user = allUsers[i]; @@ -413,7 +472,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false); } - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.AreEqual(10, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -444,7 +503,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); } - var result = await dal.GetAllTwitterUsersAsync(); + var result = await dal.GetAllTwitterUsersAsync(false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs index 21d1288..8473424 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 4d0e465..daf0bfa 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users); var loggerMock = new Mock>(); @@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); @@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>();