From 3460c728951aafb8c865be00b7f2551daa2ae459 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 5 Jan 2023 23:55:50 -0500 Subject: [PATCH] saving synchronized tweets --- .../Processors/SubTasks/SendTweetsTaskBase.cs | 31 +++++++ .../SubTasks/SendTweetsToInboxTask.cs | 8 +- .../SubTasks/SendTweetsToSharedInboxTask.cs | 6 +- .../SubTasks/SendTweetsToInboxTaskTests.cs | 80 +++++++++++++++++-- .../SubTasks/SendTweetsToSharedInboxTests.cs | 78 ++++++++++++++++-- 5 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs new file mode 100644 index 0000000..79fd5ef --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Pipeline.Processors.SubTasks; + +public class SendTweetsTaskBase +{ + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + + #region Ctor + protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + } + #endregion + + protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox) + { + var inboxUrl = $"https://{host}/{inbox.Trim('/')}"; + var tweet = new SyncTweet + { + Acct = acct, + TweetId = tweetId, + PublishedAt = DateTime.UtcNow, + Inbox = inboxUrl + }; + await _syncTweetsPostgresDal.SaveTweetAsync(tweet); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index a6f6982..0c91750 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -17,17 +17,16 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks Task ExecuteAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user); } - public class SendTweetsToInboxTask : ISendTweetsToInboxTask + public class SendTweetsToInboxTask : SendTweetsTaskBase, ISendTweetsToInboxTask { private readonly IActivityPubService _activityPubService; private readonly IStatusService _statusService; private readonly IFollowersDal _followersDal; private readonly InstanceSettings _settings; private readonly ILogger _logger; - - + #region Ctor - public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) + public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal) { _activityPubService = activityPubService; _statusService = statusService; @@ -61,6 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { var note = _statusService.GetStatus(user.Acct, tweet); await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + await SaveSyncTweetAsync(user.Acct, tweet.Id, follower.Host, inbox); } } catch (ArgumentException e) diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index 1abe183..95f32fd 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; @@ -16,7 +17,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance); } - public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask + public class SendTweetsToSharedInboxTask : SendTweetsTaskBase, ISendTweetsToSharedInboxTask { private readonly IStatusService _statusService; private readonly IActivityPubService _activityPubService; @@ -25,7 +26,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks private readonly ILogger _logger; #region Ctor - public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) + public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal) { _activityPubService = activityPubService; _statusService = statusService; @@ -61,6 +62,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { var note = _statusService.GetStatus(user.Acct, tweet); await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + await SaveSyncTweetAsync(user.Acct, tweet.Id, host, inbox); } } catch (ArgumentException e) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs index 367a642..81361ed 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; @@ -87,15 +88,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -155,15 +166,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -237,15 +251,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -319,15 +343,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -402,15 +436,28 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); + } #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -493,9 +540,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId2 + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); + #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -507,6 +564,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } @@ -571,15 +629,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -634,9 +695,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -649,6 +712,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs index 7ab06a2..5e61379 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs @@ -109,15 +109,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -199,15 +209,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -302,15 +315,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -405,15 +428,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -509,15 +542,28 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); + } #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -621,9 +667,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId2 + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -635,6 +690,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } @@ -720,15 +776,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -800,9 +859,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -815,6 +876,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } }