refactoring + init workerservice to delete tweets

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Folder Include="Processors\TweetsCleanUp\" />
     <Folder Include="Tools\" />
   </ItemGroup>
 

src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRefreshTwitterUserStatusProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveFollowersProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTweetsProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTwitterUsersProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs

@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISaveProgressionProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs

@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISendTweetsToFollowersProcessor
     {

src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs

@@ -6,12 +6,12 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
     {

src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs

@@ -2,10 +2,10 @@
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor
     {

src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs

@@ -5,14 +5,14 @@ using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
@@ -64,7 +64,7 @@ namespace BirdsiteLive.Pipeline.Processors
         private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
         {
             var tweets = new ExtractedTweet[0];
-            
+
             try
             {
                 if (user.LastTweetPostedId == -1)

src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs

@@ -7,18 +7,18 @@ using BirdsiteLive.Common.Extensions;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
-        
+
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
@@ -42,7 +42,7 @@ namespace BirdsiteLive.Pipeline.Processors
                     var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
 
                     var userCount = users.Any() ? users.Length : 1;
-                    var splitNumber = (int) Math.Ceiling(userCount / 15d);
+                    var splitNumber = (int)Math.Ceiling(userCount / 15d);
                     var splitUsers = users.Split(splitNumber).ToList();
 
                     foreach (var u in splitUsers)

src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs

@@ -5,11 +5,11 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using Microsoft.Extensions.Logging;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class SaveProgressionProcessor : ISaveProgressionProcessor
     {
@@ -36,13 +36,13 @@ namespace BirdsiteLive.Pipeline.Processors
                     await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
                     return;
                 }
-                if(userWithTweetsToSync.Followers.Length == 0)
+                if (userWithTweetsToSync.Followers.Length == 0)
                 {
                     _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
                     await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
                     return;
                 }
-            
+
                 var userId = userWithTweetsToSync.User.Id;
                 var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
                 var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();

src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs

@@ -10,7 +10,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter;
@@ -18,7 +18,7 @@ using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor
     {
@@ -83,7 +83,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 }
             }
         }
-        
+
         private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
         {
             foreach (var follower in followerWtInbox)
@@ -114,7 +114,7 @@ namespace BirdsiteLive.Pipeline.Processors
         {
             follower.PostingErrorCount++;
 
-            if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold 
+            if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold
                 && _instanceSettings.FailingFollowerCleanUpThreshold > 0
                 || follower.PostingErrorCount > 2147483600)
             {

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
+{
+    public class DeleteTweetsProcessor
+    {
+    
+    }
+}

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
+{
+    public class RetrieveTweetsToDeleteProcessor
+    {
+    
+    }
+}

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
+{
+    public class SaveDeletedTweetStatusProcessor
+    {
+    
+    }
+}

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -4,7 +4,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using Microsoft.Extensions.Logging;
 

src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs

@@ -0,0 +1,18 @@
+using System.Threading.Tasks;
+using System.Threading;
+
+namespace BirdsiteLive.Pipeline
+{
+    public interface ITweetCleanUpPipeline
+    {
+        Task ExecuteAsync(CancellationToken ct);
+    }
+
+    public class TweetCleanUpPipeline : ITweetCleanUpPipeline
+    {
+        public async Task ExecuteAsync(CancellationToken ct)
+        {
+            throw new System.NotImplementedException();
+        }
+    }
+}

src/BirdsiteLive/Program.cs

@@ -29,6 +29,7 @@ namespace BirdsiteLive
                 .ConfigureServices(services =>
                 {
                     services.AddHostedService<FederationService>();
+                    services.AddHostedService<TweetCleanUpService>();
                 });
     }
 }

src/BirdsiteLive/Services/FederationService.cs

@@ -6,6 +6,7 @@ using BirdsiteLive.DAL;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Moderation;
 using BirdsiteLive.Pipeline;
+using BirdsiteLive.Tools;
 using Microsoft.Extensions.Hosting;
 
 namespace BirdsiteLive.Services
@@ -32,6 +33,7 @@ namespace BirdsiteLive.Services
             try
             {
                 await _databaseInitializer.InitAndMigrateDbAsync();
+                InitStateSynchronization.IsDbInitialized = true;
                 await _moderationPipeline.ApplyModerationSettingsAsync();
                 await _statusPublicationPipeline.ExecuteAsync(stoppingToken);
             }

src/BirdsiteLive/Services/TweetCleanUpService.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline;
+using BirdsiteLive.Tools;
+using Microsoft.Extensions.Hosting;
+
+namespace BirdsiteLive.Services
+{
+    public class TweetCleanUpService : BackgroundService
+    {
+        private readonly ITweetCleanUpPipeline _cleanUpPipeline;
+        private readonly IHostApplicationLifetime _applicationLifetime;
+
+        #region Ctor
+        public TweetCleanUpService(IHostApplicationLifetime applicationLifetime, ITweetCleanUpPipeline cleanUpPipeline)
+        {
+            _applicationLifetime = applicationLifetime;
+            _cleanUpPipeline = cleanUpPipeline;
+        }
+        #endregion
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            try
+            {
+                // Wait for initialization
+                while (!InitStateSynchronization.IsDbInitialized)
+                {
+                    if (stoppingToken.IsCancellationRequested) return;
+                    await Task.Delay(250, stoppingToken);
+                }
+
+                await _cleanUpPipeline.ExecuteAsync(stoppingToken);
+            }
+            catch (Exception)
+            {
+                await Task.Delay(1000 * 30);
+                _applicationLifetime.StopApplication();
+            }
+        }
+    }
+}

src/BirdsiteLive/Tools/InitStateSynchronization.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.Tools
+{
+    public static class InitStateSynchronization
+    {
+        public static bool IsDbInitialized { get; set; }
+    }
+}

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs

@@ -5,7 +5,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -6,7 +6,7 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -2,7 +2,7 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -7,7 +7,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;

src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs

@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;