From 76b2e659abb664ed94906f0622243dc1924adb3b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Nov 2022 20:02:37 -0400 Subject: [PATCH] added movedTo support in db --- src/BirdsiteLive.ActivityPub/Models/Actor.cs | 1 + src/BirdsiteLive.Domain/UserService.cs | 1 + .../Processors/RetrieveTweetsProcessor.cs | 4 +- .../Processors/SaveProgressionProcessor.cs | 2 +- .../DbInitializerPostgresDal.cs | 13 ++++++- .../TwitterUserPostgresDal.cs | 38 +++++++++++++------ .../Contracts/ITwitterUserDal.cs | 5 ++- .../Models/SyncTwitterUser.cs | 5 ++- .../TwitterUserPostgresDalTests.cs | 16 ++++---- .../ProcessFollowUserTests.cs | 4 +- .../RetrieveTweetsProcessorTests.cs | 4 +- .../SaveProgressionProcessorTests.cs | 16 ++++++-- 12 files changed, 77 insertions(+), 32 deletions(-) diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs index 713ea89..ea4f8a3 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs @@ -17,6 +17,7 @@ namespace BirdsiteLive.ActivityPub public string name { get; set; } public string summary { get; set; } public string url { get; set; } + public string movedTo { get; set; } public bool manuallyApprovesFollowers { get; set; } public string inbox { get; set; } public bool? discoverable { get; set; } = true; diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 42cf6d3..f5ae123 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -90,6 +90,7 @@ namespace BirdsiteLive.Domain summary = description, url = actorUrl, manuallyApprovesFollowers = twitterUser.Protected, + discoverable = false, publicKey = new PublicKey() { id = $"{actorUrl}#main-key", diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 58d35d0..e6608d7 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -49,12 +49,12 @@ namespace BirdsiteLive.Pipeline.Processors { var tweetId = tweets.Last().Id; var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct); } else { var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index f262f39..109f751 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Pipeline.Processors var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = followingSyncStatuses.Min(); var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct); } catch (Exception e) { diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 2f9cb54..a66af72 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 4); + private readonly Version _currentVersion = new Version(2, 5); private const string DbVersionType = "db-version"; #region Ctor @@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers new Tuple(new Version(2,0), new Version(2,1)), new Tuple(new Version(2,1), new Version(2,2)), new Tuple(new Version(2,2), new Version(2,3)), - new Tuple(new Version(2,3), new Version(2,4)) + new Tuple(new Version(2,3), new Version(2,4)), + new Tuple(new Version(2,4), new Version(2,5)) }; } @@ -172,6 +173,14 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER"; await _tools.ExecuteRequestAsync(alterPostingError); } + else if (from == new Version(2, 4) && to == new Version(2, 5)) + { + var addMovedTo = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedTo VARCHAR(2048)"; + await _tools.ExecuteRequestAsync(addMovedTo); + + var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)"; + await _tools.ExecuteRequestAsync(addMovedToAcct); + } else { throw new NotImplementedException(); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 11214d4..a649c7b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } #endregion - public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId) + public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, string movedToAcct = null) { acct = acct.ToLowerInvariant(); @@ -27,8 +27,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers dbConnection.Open(); await dbConnection.ExecuteAsync( - $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)", - new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId }); + $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId, movedTo, movedToAcct) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId,@movedTo,@movedToAcct)", + new + { + acct, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId = lastTweetPostedId, + movedTo, + movedToAcct + }); } } @@ -62,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; using (var dbConnection = Connection) { @@ -75,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetFailingTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE"; using (var dbConnection = Connection) { @@ -88,7 +95,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetAllTwitterUsersAsync(int maxNumber) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; using (var dbConnection = Connection) { @@ -101,7 +108,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetAllTwitterUsersAsync() { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; using (var dbConnection = Connection) { @@ -112,26 +119,35 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); if(lastSync == default) throw new ArgumentException("lastSync"); - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id"; + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() }); + await dbConnection.QueryAsync(query, new + { + id, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId, + fetchingErrorCount, + lastSync = lastSync.ToUniversalTime(), + movedTo, + movedToAcct + }); } } public async Task UpdateTwitterUserAsync(SyncTwitterUser user) { - await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync); + await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct); } public async Task DeleteTwitterUserAsync(string acct) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index ef2cc36..a29463b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -6,12 +6,13 @@ namespace BirdsiteLive.DAL.Contracts { public interface ITwitterUserDal { - Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); + Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, + string movedToAcct = null); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); Task GetAllTwitterUsersAsync(int maxNumber); Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8b18ba1..224d981 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -12,6 +12,9 @@ namespace BirdsiteLive.DAL.Models public DateTime LastSync { get; set; } - public int FetchingErrorCount { get; set; } //TODO: update DAL + public int FetchingErrorCount { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index c9bc746..c6932ce 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -87,7 +87,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var updatedLastSyncId = 1550L; var now = DateTime.Now; var errors = 15; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now); + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null); result = await dal.GetTwitterUserAsync(acct); @@ -96,6 +96,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); Assert.AreEqual(errors, result.FetchingErrorCount); Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); } [TestMethod] @@ -167,7 +169,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null); } [TestMethod] @@ -175,7 +177,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetPostedId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null); } [TestMethod] @@ -183,7 +185,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetSynchronizedForAllFollowersId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null); } [TestMethod] @@ -191,7 +193,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastSync() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default); + await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null); } [TestMethod] @@ -318,7 +320,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { var user = allUsers[i]; var date = i % 2 == 0 ? oldest : newest; - await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date); + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null); } var result = await dal.GetAllTwitterUsersAsync(10); @@ -382,7 +384,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers if (i == 0 || i == 2 || i == 3) { var t = await dal.GetTwitterUserAsync(acct); - await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now); + await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null); } } diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs index 0fb03ae..8f2f393 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs @@ -77,7 +77,9 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases twitterUserDalMock .Setup(x => x.CreateTwitterUserAsync( It.Is(y => y == twitterName), - It.Is(y => y == -1))) + It.Is(y => y == -1), + It.Is(y => y == null), + It.Is(y => y == null))) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 17a3aa2..b82ca5e 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -64,7 +64,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweets.Last().Id), It.Is(y => y == tweets.Last().Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index feffbff..23a11ec 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -66,7 +66,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet2.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); @@ -133,7 +135,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet2.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Throws(new ArgumentException()); @@ -202,7 +206,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); @@ -281,7 +287,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask);