diff --git a/VARIABLES.md b/VARIABLES.md index f43bddc..bb8bf61 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. * `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided). * `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy). +* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted # Docker Compose full example diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index a67977c..0ece251 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -18,5 +18,7 @@ public int UserCacheCapacity { get; set; } public string IpWhiteListing { get; set; } public bool EnableXRealIpHeader { get; set; } + + public int MaxTweetRetention { get; set; } } } diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index f3d930b..b72c3cf 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs new file mode 100644 index 0000000..58942ac --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs @@ -0,0 +1,16 @@ +using BirdsiteLive.Common.Settings; +using System; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base +{ + public class RetentionBase + { + protected int GetRetentionTime(InstanceSettings settings) + { + var retentionTime = Math.Abs(settings.MaxTweetRetention); + if (retentionTime < 1) retentionTime = 1; + if (retentionTime > 90) retentionTime = 90; + return retentionTime; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs index 82c6ef7..daf4729 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs @@ -3,20 +3,24 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base; namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class RetrieveTweetsToDeleteProcessor : IRetrieveTweetsToDeleteProcessor + public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor { private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + private readonly InstanceSettings _instanceSettings; #region Ctor - public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal) + public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings) { _syncTweetsPostgresDal = syncTweetsPostgresDal; + _instanceSettings = instanceSettings; } #endregion @@ -29,7 +33,9 @@ namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp ct.ThrowIfCancellationRequested(); var now = DateTime.UtcNow; - var from = now.AddDays(-20); + + + var from = now.AddDays(-GetRetentionTime(_instanceSettings)); var dbBrowsingEnded = false; var lastId = -1L; diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs index 1af1288..817d0be 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs @@ -1,26 +1,32 @@ using System; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base; namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class SaveDeletedTweetStatusProcessor : ISaveDeletedTweetStatusProcessor + public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor { private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + private readonly InstanceSettings _instanceSettings; #region Ctor - public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal) + public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings) { _syncTweetsPostgresDal = syncTweetsPostgresDal; + _instanceSettings = instanceSettings; } #endregion public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct) { - var highLimitDate = DateTime.UtcNow.AddDays(-40); //TODO get settings value + var retentionTime = GetRetentionTime(_instanceSettings); + retentionTime += 20; // Delay until last retry + var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime); if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate) { await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id); diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 4cee8ba..9628532 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -25,7 +25,8 @@ "SensitiveTwitterAccounts": null, "FailingTwitterUserCleanUpThreshold": 700, "FailingFollowerCleanUpThreshold": 30000, - "UserCacheCapacity": 10000 + "UserCacheCapacity": 10000, + "MaxTweetRetention": 20 }, "Db": { "Type": "postgres",