bump nugets

.github/workflows/dotnet-core.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 3.1.101
+        dotnet-version: 6.0.x
     - name: Install dependencies
       run: dotnet restore
       working-directory: ${{env.working-directory}}

Dockerfile

@@ -1,11 +1,11 @@
 #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
 
-FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
+FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
-FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
+FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish
 COPY ./src/ ./src/
 RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
 RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish

VARIABLES.md

@@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
 * `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
 * `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
+* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
 
 # Docker Compose full example
 

src/BSLManager/BSLManager.csproj

@@ -2,15 +2,15 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.3" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+    <PackageReference Include="Lamar" Version="10.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
+    <PackageReference Include="Terminal.Gui" Version="1.9.0" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="System.Text.Json" Version="4.7.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="System.Text.Json" Version="7.0.1" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -18,5 +18,7 @@
         public int UserCacheCapacity { get; set; }
         public string IpWhiteListing { get; set; }
         public bool EnableXRealIpHeader { get; set; }
+
+        public int MaxTweetRetention { get; set; }
     }
 }

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -9,6 +9,7 @@ using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Models;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 
@@ -22,6 +23,7 @@ namespace BirdsiteLive.Domain
         Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
             string targetInbox);
         Task DeleteUserAsync(string username, string targetHost, string targetInbox);
+        Task DeleteNoteAsync(SyncTweet tweet);
     }
 
     public class WebFinger
@@ -108,6 +110,30 @@ namespace BirdsiteLive.Domain
             }
         }
 
+        public async Task DeleteNoteAsync(SyncTweet tweet)
+        {
+            var acct = tweet.Acct.ToLowerInvariant().Trim();
+
+            var actor = $"https://{_instanceSettings.Domain}/users/{acct}";
+            var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}";
+
+            var delete = new ActivityDelete
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{noteId}#delete",
+                type = "Delete",
+                actor = actor,
+                to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
+                apObject = new Tombstone
+                {
+                    id = noteId,
+                    atomUrl = noteId
+                }
+            };
+
+            await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox);
+        }
+
         public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
         {
             try

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Folder Include="Processors\TweetsCleanUp\Base\" />
     <Folder Include="Tools\" />
   </ItemGroup>
 

src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRefreshTwitterUserStatusProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveFollowersProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTweetsProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTwitterUsersProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs

@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISaveProgressionProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs

@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISendTweetsToFollowersProcessor
     {

src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface IDeleteTweetsProcessor
+    {
+        Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs

@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface IRetrieveTweetsToDeleteProcessor
+    {
+        Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface ISaveDeletedTweetStatusProcessor
+    {
+        Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs

@@ -0,0 +1,10 @@
+using BirdsiteLive.DAL.Models;
+
+namespace BirdsiteLive.Pipeline.Models
+{
+    public class TweetToDelete
+    {
+        public SyncTweet Tweet { get; set; }
+        public bool DeleteSuccessful { get; set; }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs

@@ -6,12 +6,12 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
     {

src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs

@@ -2,10 +2,10 @@
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor
     {

src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs

@@ -5,14 +5,14 @@ using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
@@ -64,7 +64,7 @@ namespace BirdsiteLive.Pipeline.Processors
         private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
         {
             var tweets = new ExtractedTweet[0];
-            
+
             try
             {
                 if (user.LastTweetPostedId == -1)

src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs

@@ -7,18 +7,18 @@ using BirdsiteLive.Common.Extensions;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
-        
+
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
@@ -42,7 +42,7 @@ namespace BirdsiteLive.Pipeline.Processors
                     var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
 
                     var userCount = users.Any() ? users.Length : 1;
-                    var splitNumber = (int) Math.Ceiling(userCount / 15d);
+                    var splitNumber = (int)Math.Ceiling(userCount / 15d);
                     var splitUsers = users.Split(splitNumber).ToList();
 
                     foreach (var u in splitUsers)

src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs

@@ -5,11 +5,11 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using Microsoft.Extensions.Logging;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class SaveProgressionProcessor : ISaveProgressionProcessor
     {
@@ -36,13 +36,13 @@ namespace BirdsiteLive.Pipeline.Processors
                     await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
                     return;
                 }
-                if(userWithTweetsToSync.Followers.Length == 0)
+                if (userWithTweetsToSync.Followers.Length == 0)
                 {
                     _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
                     await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
                     return;
                 }
-            
+
                 var userId = userWithTweetsToSync.User.Id;
                 var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
                 var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();

src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs

@@ -10,7 +10,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter;
@@ -18,7 +18,7 @@ using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor
     {
@@ -83,7 +83,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 }
             }
         }
-        
+
         private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
         {
             foreach (var follower in followerWtInbox)
@@ -114,7 +114,7 @@ namespace BirdsiteLive.Pipeline.Processors
         {
             follower.PostingErrorCount++;
 
-            if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold 
+            if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold
                 && _instanceSettings.FailingFollowerCleanUpThreshold > 0
                 || follower.PostingErrorCount > 2147483600)
             {

src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+
+namespace BirdsiteLive.Pipeline.Processors.SubTasks
+{
+    public class SendTweetsTaskBase
+    {
+        private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
+
+        #region Ctor
+        protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal)
+        {
+            _syncTweetsPostgresDal = syncTweetsPostgresDal;
+        }
+        #endregion
+
+        protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox)
+        {
+            var tweet = new SyncTweet
+            {
+                Acct = acct,
+                TweetId = tweetId,
+                PublishedAt = DateTime.UtcNow,
+                Inbox = inbox,
+                Host = host
+            };
+            await _syncTweetsPostgresDal.SaveTweetAsync(tweet);
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToInboxTask.cs

@@ -17,17 +17,16 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user);
     }
 
-    public class SendTweetsToInboxTask : ISendTweetsToInboxTask
+    public class SendTweetsToInboxTask : SendTweetsTaskBase, ISendTweetsToInboxTask
     {
         private readonly IActivityPubService _activityPubService;
         private readonly IStatusService _statusService;
         private readonly IFollowersDal _followersDal;
         private readonly InstanceSettings _settings;
         private readonly ILogger<SendTweetsToInboxTask> _logger;
-
-
+        
         #region Ctor
-        public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
+        public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal)
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
@@ -61,6 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                         {
                             var note = _statusService.GetStatus(user.Acct, tweet);
                             await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+                            await SaveSyncTweetAsync(user.Acct, tweet.Id, follower.Host, inbox);
                         }
                     }
                     catch (ArgumentException e)

src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -2,6 +2,7 @@
 using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
+using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
@@ -16,7 +17,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance);
     }
 
-    public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
+    public class SendTweetsToSharedInboxTask : SendTweetsTaskBase, ISendTweetsToSharedInboxTask
     {
         private readonly IStatusService _statusService;
         private readonly IActivityPubService _activityPubService;
@@ -25,7 +26,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
 
         #region Ctor
-        public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
+        public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal)
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
@@ -61,6 +62,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                         {
                             var note = _statusService.GetStatus(user.Acct, tweet);
                             await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
+                            await SaveSyncTweetAsync(user.Acct, tweet.Id, host, inbox);
                         }
                     }
                     catch (ArgumentException e)

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs

@@ -0,0 +1,16 @@
+using BirdsiteLive.Common.Settings;
+using System;
+
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base
+{
+    public class RetentionBase
+    {
+        protected int GetRetentionTime(InstanceSettings settings)
+        {
+            var retentionTime = Math.Abs(settings.MaxTweetRetention);
+            if (retentionTime < 1) retentionTime = 1;
+            if (retentionTime > 90) retentionTime = 90;
+            return retentionTime;
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Domain;
+using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
+using BirdsiteLive.Pipeline.Models;
+using Microsoft.Extensions.Logging;
+
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
+{
+    public class DeleteTweetsProcessor : IDeleteTweetsProcessor
+    {
+        private readonly IActivityPubService _activityPubService;
+        private readonly ILogger<DeleteTweetsProcessor> _logger;
+
+        #region Ctor
+        public DeleteTweetsProcessor(IActivityPubService activityPubService, ILogger<DeleteTweetsProcessor> logger)
+        {
+            _activityPubService = activityPubService;
+            _logger = logger;
+        }
+        #endregion
+
+        public async Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct)
+        {
+            try
+            {
+                await _activityPubService.DeleteNoteAsync(tweet.Tweet);
+                tweet.DeleteSuccessful = true;
+            }
+            catch (HttpRequestException e)
+            {
+                var code = e.StatusCode;
+                if (code is HttpStatusCode.Gone or HttpStatusCode.NotFound)
+                {
+                    _logger.LogInformation("Tweet already deleted");
+                    tweet.DeleteSuccessful = true;
+                }
+                else
+                {
+                    _logger.LogError(e.Message, e);
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e.Message, e);
+            }
+
+            return tweet;
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
+
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
+{
+    public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor
+    {
+        private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
+        {
+            _syncTweetsPostgresDal = syncTweetsPostgresDal;
+            _instanceSettings = instanceSettings;
+        }
+        #endregion
+
+        public async Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct)
+        {
+            var batchSize = 100; 
+
+            for (;;)
+            {
+                ct.ThrowIfCancellationRequested();
+
+                var now = DateTime.UtcNow;
+
+               
+                var from = now.AddDays(-GetRetentionTime(_instanceSettings));
+                var dbBrowsingEnded = false;
+                var lastId = -1L;
+
+                do
+                {
+                    var tweets = await _syncTweetsPostgresDal.GetTweetsOlderThanAsync(from, lastId, batchSize);
+
+                    foreach (var syncTweet in tweets)
+                    {
+                        var tweet = new TweetToDelete
+                        {
+                            Tweet = syncTweet
+                        };
+                        await tweetsBufferBlock.SendAsync(tweet, ct);
+                    }
+
+                    if (tweets.Any()) lastId = tweets.Last().Id;
+                    if (tweets.Count < batchSize) dbBrowsingEnded = true;
+
+                } while (!dbBrowsingEnded);
+
+                await Task.Delay(TimeSpan.FromHours(3), ct);
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
+
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
+{
+    public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor
+    {
+        private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
+        {
+            _syncTweetsPostgresDal = syncTweetsPostgresDal;
+            _instanceSettings = instanceSettings;
+        }
+        #endregion
+
+        public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
+        {
+            var retentionTime = GetRetentionTime(_instanceSettings);
+            retentionTime += 20; // Delay until last retry
+            var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime);
+            if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
+            {
+                await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -4,7 +4,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using Microsoft.Extensions.Logging;
 

src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs

@@ -0,0 +1,76 @@
+using System.Threading.Tasks;
+using System.Threading;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
+using BirdsiteLive.Pipeline.Models;
+using Microsoft.Extensions.Logging;
+
+namespace BirdsiteLive.Pipeline
+{
+    public interface ITweetCleanUpPipeline
+    {
+        Task ExecuteAsync(CancellationToken ct);
+    }
+
+    public class TweetCleanUpPipeline : ITweetCleanUpPipeline
+    {
+        private readonly IRetrieveTweetsToDeleteProcessor _retrieveTweetsToDeleteProcessor;
+        private readonly IDeleteTweetsProcessor _deleteTweetsProcessor;
+        private readonly ISaveDeletedTweetStatusProcessor _saveDeletedTweetStatusProcessor;
+        private readonly ILogger<TweetCleanUpPipeline> _logger;
+
+        #region Ctor
+        public TweetCleanUpPipeline(IRetrieveTweetsToDeleteProcessor retrieveTweetsToDeleteProcessor, IDeleteTweetsProcessor deleteTweetsProcessor, ISaveDeletedTweetStatusProcessor saveDeletedTweetStatusProcessor, ILogger<TweetCleanUpPipeline> logger)
+        {
+            _retrieveTweetsToDeleteProcessor = retrieveTweetsToDeleteProcessor;
+            _deleteTweetsProcessor = deleteTweetsProcessor;
+            _saveDeletedTweetStatusProcessor = saveDeletedTweetStatusProcessor;
+            _logger = logger;
+        }
+        #endregion
+
+        public async Task ExecuteAsync(CancellationToken ct)
+        {
+            // Create blocks 
+            var tweetsToDeleteBufferBlock = new BufferBlock<TweetToDelete>(new DataflowBlockOptions
+            {
+                BoundedCapacity = 200,
+                CancellationToken = ct
+            });
+            var deleteTweetsBlock = new TransformBlock<TweetToDelete, TweetToDelete>(
+                async x => await _deleteTweetsProcessor.ProcessAsync(x, ct),
+                new ExecutionDataflowBlockOptions()
+                {
+                    MaxDegreeOfParallelism = 5, 
+                    CancellationToken = ct
+                });
+            var deletedTweetsBufferBlock = new BufferBlock<TweetToDelete>(new DataflowBlockOptions
+            {
+                BoundedCapacity = 200,
+                CancellationToken = ct
+            });
+            var saveProgressionBlock = new ActionBlock<TweetToDelete>(
+                async x => await _saveDeletedTweetStatusProcessor.ProcessAsync(x, ct),
+                new ExecutionDataflowBlockOptions
+                {
+                    MaxDegreeOfParallelism = 5, 
+                    CancellationToken = ct
+                });
+
+            // Link pipeline
+            var dataflowLinkOptions = new DataflowLinkOptions { PropagateCompletion = true };
+            tweetsToDeleteBufferBlock.LinkTo(deleteTweetsBlock, dataflowLinkOptions);
+            deleteTweetsBlock.LinkTo(deletedTweetsBufferBlock, dataflowLinkOptions);
+            deletedTweetsBufferBlock.LinkTo(saveProgressionBlock, dataflowLinkOptions);
+
+            // Launch tweet retriever 
+            var retrieveTweetsToDeleteTask = _retrieveTweetsToDeleteProcessor.GetTweetsAsync(tweetsToDeleteBufferBlock, ct);
+
+            // Wait 
+            await Task.WhenAny(new[] { retrieveTweetsToDeleteTask, saveProgressionBlock.Completion });
+
+            var ex = retrieveTweetsToDeleteTask.IsFaulted ? retrieveTweetsToDeleteTask.Exception : saveProgressionBlock.Completion.Exception;
+            _logger.LogCritical(ex, "An error occurred, pipeline stopped");
+        }
+    }
+}

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
     <PackageReference Include="TweetinviAPI" Version="4.0.3" />
   </ItemGroup>
 

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
     <Version>0.22.0</Version>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
-    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />
+    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="10.0.1" />
+    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.17.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.11" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive/Controllers/MigrationController.cs

@@ -3,7 +3,6 @@ using Microsoft.AspNetCore.Mvc;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
-using Npgsql.TypeHandlers;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Domain.Enum;
 using BirdsiteLive.DAL.Contracts;

src/BirdsiteLive/Controllers/UsersController.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Mime;
-using System.Runtime.InteropServices.WindowsRuntime;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;

src/BirdsiteLive/Program.cs

@@ -29,6 +29,7 @@ namespace BirdsiteLive
                 .ConfigureServices(services =>
                 {
                     services.AddHostedService<FederationService>();
+                    services.AddHostedService<TweetCleanUpService>();
                 });
     }
 }

src/BirdsiteLive/Services/FederationService.cs

@@ -6,6 +6,7 @@ using BirdsiteLive.DAL;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Moderation;
 using BirdsiteLive.Pipeline;
+using BirdsiteLive.Tools;
 using Microsoft.Extensions.Hosting;
 
 namespace BirdsiteLive.Services
@@ -32,6 +33,7 @@ namespace BirdsiteLive.Services
             try
             {
                 await _databaseInitializer.InitAndMigrateDbAsync();
+                InitStateSynchronization.IsDbInitialized = true;
                 await _moderationPipeline.ApplyModerationSettingsAsync();
                 await _statusPublicationPipeline.ExecuteAsync(stoppingToken);
             }

src/BirdsiteLive/Services/TweetCleanUpService.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline;
+using BirdsiteLive.Tools;
+using Microsoft.Extensions.Hosting;
+
+namespace BirdsiteLive.Services
+{
+    public class TweetCleanUpService : BackgroundService
+    {
+        private readonly ITweetCleanUpPipeline _cleanUpPipeline;
+        private readonly IHostApplicationLifetime _applicationLifetime;
+
+        #region Ctor
+        public TweetCleanUpService(IHostApplicationLifetime applicationLifetime, ITweetCleanUpPipeline cleanUpPipeline)
+        {
+            _applicationLifetime = applicationLifetime;
+            _cleanUpPipeline = cleanUpPipeline;
+        }
+        #endregion
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            try
+            {
+                // Wait for initialization
+                while (!InitStateSynchronization.IsDbInitialized)
+                {
+                    if (stoppingToken.IsCancellationRequested) return;
+                    await Task.Delay(250, stoppingToken);
+                }
+
+                await _cleanUpPipeline.ExecuteAsync(stoppingToken);
+            }
+            catch (Exception)
+            {
+                await Task.Delay(1000 * 30);
+                _applicationLifetime.StopApplication();
+            }
+        }
+    }
+}

src/BirdsiteLive/Tools/InitStateSynchronization.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.Tools
+{
+    public static class InitStateSynchronization
+    {
+        public static bool IsDbInitialized { get; set; }
+    }
+}

src/BirdsiteLive/appsettings.json

@@ -25,7 +25,8 @@
     "SensitiveTwitterAccounts": null,
     "FailingTwitterUserCleanUpThreshold": 700,
     "FailingFollowerCleanUpThreshold": 30000,
-    "UserCacheCapacity": 10000 
+    "UserCacheCapacity": 10000,
+    "MaxTweetRetention": 20
   },
   "Db": {
     "Type": "postgres",

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj

@@ -5,8 +5,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dapper" Version="2.0.35" />
-    <PackageReference Include="Npgsql" Version="4.1.3.1" />
+    <PackageReference Include="Dapper" Version="2.0.123" />
+    <PackageReference Include="Npgsql" Version="7.0.1" />
   </ItemGroup>
 
   <ItemGroup>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs

@@ -195,6 +195,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                     acct VARCHAR(50) NOT NULL,
                     tweetId BIGINT NOT NULL,
                     inbox VARCHAR(2048) NOT NULL,
+                    host VARCHAR(253) NOT NULL,
                     publishedAt TIMESTAMP (2) WITHOUT TIME ZONE NOT NULL,
 
                     UNIQUE (acct, tweetId, inbox)

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs

@@ -22,21 +22,26 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
         {
             if (tweet.PublishedAt == default) throw new ArgumentException("publishedAt");
             if (tweet.TweetId == default) throw new ArgumentException("tweetId");
+            if (string.IsNullOrWhiteSpace(tweet.Acct)) throw new ArgumentException("acct");
+            if (string.IsNullOrWhiteSpace(tweet.Inbox)) throw new ArgumentException("inbox");
+            if (string.IsNullOrWhiteSpace(tweet.Host)) throw new ArgumentException("host");
 
             var acct = tweet.Acct.ToLowerInvariant().Trim();
             var inbox = tweet.Inbox.ToLowerInvariant().Trim();
+            var host = tweet.Host.ToLowerInvariant().Trim();
 
             using (var dbConnection = Connection)
             {
                 dbConnection.Open();
 
                 return (await dbConnection.QueryAsync<long>(
-                    $"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,publishedAt) VALUES(@acct,@tweetId,@inbox,@publishedAt) RETURNING id",
+                    $"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,host,publishedAt) VALUES(@acct,@tweetId,@inbox,@host,@publishedAt) RETURNING id",
                     new
                     {
                         acct,
                         tweetId = tweet.TweetId,
                         inbox,
+                        host,
                         publishedAt = tweet.PublishedAt.ToUniversalTime()
                     })).First();
             }

src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs

@@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Models
         public long Id { get; set; }
         
         public string Acct { get; set; }
+        public string Host { get; set; } //TODO
         public string Inbox { get; set; }
         public long TweetId { get; set; }
         public DateTime PublishedAt { get; set; }

src/Tests/BSLManager.Tests/BSLManager.Tests.csproj

@@ -1,16 +1,19 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj

@@ -1,16 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj

@@ -1,16 +1,19 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj

@@ -1,16 +1,19 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj

@@ -1,16 +1,19 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base;
@@ -33,7 +34,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             {
                 Acct = "test",
                 PublishedAt = DateTime.UtcNow,
-                Inbox = "https://instance.ext/inbox",
+                Inbox = "/inbox",
+                Host = "instance.ext",
                 TweetId = 4567889
             };
 
@@ -47,6 +49,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsTrue(result.Id > 0);
             Assert.AreEqual(tweet.Acct, result.Acct);
             Assert.AreEqual(tweet.Inbox, result.Inbox);
+            Assert.AreEqual(tweet.Host, result.Host);
             Assert.AreEqual(tweet.TweetId, result.TweetId);
             Assert.IsTrue(Math.Abs((tweet.PublishedAt - result.PublishedAt).Seconds) < 5);
         }
@@ -58,7 +61,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             {
                 Acct = "test",
                 PublishedAt = DateTime.UtcNow,
-                Inbox = "https://instance.ext/inbox",
+                Inbox = "/inbox",
+                Host = "instance.ext",
                 TweetId = 4567889
             };
 
@@ -77,15 +81,19 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var now = DateTime.UtcNow;
             var dal = new SyncTweetsPostgresDal(_settings);
 
+            var allData = new List<SyncTweet>();
+
             for (var i = 0; i < 100; i++)
             {
                 var tweet = new SyncTweet
                 {
                     Acct = "test",
                     PublishedAt = now.AddDays(-10 - i),
-                    Inbox = "https://instance.ext/inbox",
+                    Inbox = "/inbox",
+                    Host = "instance.ext",
                     TweetId = 4567889 + i
                 };
+                allData.Add(tweet);
                 await dal.SaveTweetAsync(tweet);
             }
 
@@ -100,6 +108,13 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 Assert.IsTrue(res.PublishedAt < date);
                 Assert.IsTrue(res.Id > 10);
                 Assert.IsTrue(res.Id < 25);
+
+                var original = allData.First(x => x.Acct == res.Acct && x.TweetId == res.TweetId);
+                Assert.AreEqual(original.Acct, res.Acct);
+                Assert.AreEqual(original.Inbox, res.Inbox);
+                Assert.AreEqual(original.Host, res.Host);
+                Assert.AreEqual(original.TweetId, res.TweetId);
+                Assert.IsTrue(Math.Abs((original.PublishedAt - res.PublishedAt).Seconds) < 5);
             }
         }
 
@@ -115,7 +130,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {
                     Acct = "test",
                     PublishedAt = now.AddDays(-10 - i),
-                    Inbox = "https://instance.ext/inbox",
+                    Inbox = "/inbox",
+                    Host = "instance.ext",
                     TweetId = 4567889 + i
                 };
                 await dal.SaveTweetAsync(tweet);
@@ -140,7 +156,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {
                     Acct = "test",
                     PublishedAt = now.AddDays(-10 - i),
-                    Inbox = "https://instance.ext/inbox",
+                    Inbox = "/inbox",
+                    Host = "instance.ext",
                     TweetId = 4567889 + i
                 };
                 await dal.SaveTweetAsync(tweet);

src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj

@@ -1,17 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj

@@ -1,17 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj

@@ -1,17 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj

@@ -1,17 +1,19 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
-
+    <TargetFramework>net6.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs

@@ -7,8 +7,7 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs

@@ -5,7 +5,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs

@@ -5,7 +5,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -6,7 +6,7 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -2,7 +2,7 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -7,7 +7,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors;
+using BirdsiteLive.Pipeline.Processors.Federation;
 using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Net;
 using System.Net.Http;
+using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
@@ -87,15 +88,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId
+                                          && y.Inbox == inbox 
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -155,15 +167,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -237,15 +252,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId
+                                          && y.Inbox == inbox
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -319,15 +345,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId
+                                          && y.Inbox == inbox
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -402,15 +439,29 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            foreach (var tweetId in new[] { tweetId2, tweetId3 })
+            {
+                syncTweetDalMock
+                    .Setup(x => x.SaveTweetAsync(
+                        It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                              && y.TweetId == tweetId
+                                              && y.Inbox == inbox
+                                              && y.Host == host
+                                              && y.PublishedAt != default)))
+                    .ReturnsAsync(-1);
+            }
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -493,9 +544,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId2
+                                          && y.Inbox == inbox
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
+
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
 
             try
             {
@@ -507,6 +569,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 activityPubService.VerifyAll();
                 statusServiceMock.VerifyAll();
                 followersDalMock.VerifyAll();
+                syncTweetDalMock.VerifyAll();
                 #endregion
             }
         }
@@ -571,15 +634,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -634,9 +700,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
             #endregion
 
-            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
 
             try
             {
@@ -649,6 +717,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 activityPubService.VerifyAll();
                 statusServiceMock.VerifyAll();
                 followersDalMock.VerifyAll();
+                syncTweetDalMock.VerifyAll();
                 #endregion
             }
         }

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs

@@ -109,15 +109,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId
+                                          && y.Inbox == inbox
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -199,15 +210,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -302,15 +316,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId
+                                          && y.Inbox == inbox
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -405,15 +430,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                .Setup(x => x.SaveTweetAsync(
+                    It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                          && y.TweetId == tweetId
+                                          && y.Inbox == inbox
+                                          && y.Host == host
+                                          && y.PublishedAt != default)))
+                .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -509,15 +545,29 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            foreach (var tweetId in new[] { tweetId2, tweetId3 })
+            {
+                syncTweetDalMock
+                    .Setup(x => x.SaveTweetAsync(
+                        It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                              && y.TweetId == tweetId
+                                              && y.Inbox == inbox
+                                              && y.Host == host
+                                              && y.PublishedAt != default)))
+                    .ReturnsAsync(-1);
+            }
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -621,9 +671,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
+            syncTweetDalMock
+                    .Setup(x => x.SaveTweetAsync(
+                        It.Is<SyncTweet>(y => y.Acct == twitterHandle
+                                              && y.TweetId == tweetId2
+                                              && y.Inbox == inbox
+                                              && y.Host == host
+                                              && y.PublishedAt != default)))
+                    .ReturnsAsync(-1);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
 
             try
             {
@@ -635,6 +695,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 activityPubService.VerifyAll();
                 statusServiceMock.VerifyAll();
                 followersDalMock.VerifyAll();
+                syncTweetDalMock.VerifyAll();
                 #endregion
             }
         }
@@ -720,15 +781,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             }
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
             await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
 
             #region Validations
             activityPubService.VerifyAll();
             statusServiceMock.VerifyAll();
             followersDalMock.VerifyAll();
+            syncTweetDalMock.VerifyAll();
             #endregion
         }
 
@@ -800,9 +864,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
+
+            var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
             #endregion
 
-            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
+            var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
 
             try
             {
@@ -815,6 +881,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 activityPubService.VerifyAll();
                 statusServiceMock.VerifyAll();
                 followersDalMock.VerifyAll();
+                syncTweetDalMock.VerifyAll();
                 #endregion
             }
         }

src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs

@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;