set tweet retention delay in settings
This commit is contained in:
parent
80ac1363e5
commit
405087360c
|
@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
|
|||
* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
|
||||
* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
|
||||
* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
|
||||
* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
|
||||
|
||||
# Docker Compose full example
|
||||
|
||||
|
|
|
@ -18,5 +18,7 @@
|
|||
public int UserCacheCapacity { get; set; }
|
||||
public string IpWhiteListing { get; set; }
|
||||
public bool EnableXRealIpHeader { get; set; }
|
||||
|
||||
public int MaxTweetRetention { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Processors\TweetsCleanUp\" />
|
||||
<Folder Include="Processors\TweetsCleanUp\Base\" />
|
||||
<Folder Include="Tools\" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using System;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base
|
||||
{
|
||||
public class RetentionBase
|
||||
{
|
||||
protected int GetRetentionTime(InstanceSettings settings)
|
||||
{
|
||||
var retentionTime = Math.Abs(settings.MaxTweetRetention);
|
||||
if (retentionTime < 1) retentionTime = 1;
|
||||
if (retentionTime > 90) retentionTime = 90;
|
||||
return retentionTime;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -3,20 +3,24 @@ using System.Linq;
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class RetrieveTweetsToDeleteProcessor : IRetrieveTweetsToDeleteProcessor
|
||||
public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal)
|
||||
public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
@ -29,7 +33,9 @@ namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
|||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var from = now.AddDays(-20);
|
||||
|
||||
|
||||
var from = now.AddDays(-GetRetentionTime(_instanceSettings));
|
||||
var dbBrowsingEnded = false;
|
||||
var lastId = -1L;
|
||||
|
||||
|
|
|
@ -1,26 +1,32 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class SaveDeletedTweetStatusProcessor : ISaveDeletedTweetStatusProcessor
|
||||
public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal)
|
||||
public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
|
||||
{
|
||||
var highLimitDate = DateTime.UtcNow.AddDays(-40); //TODO get settings value
|
||||
var retentionTime = GetRetentionTime(_instanceSettings);
|
||||
retentionTime += 20; // Delay until last retry
|
||||
var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime);
|
||||
if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
|
||||
{
|
||||
await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);
|
||||
|
|
|
@ -25,7 +25,8 @@
|
|||
"SensitiveTwitterAccounts": null,
|
||||
"FailingTwitterUserCleanUpThreshold": 700,
|
||||
"FailingFollowerCleanUpThreshold": 30000,
|
||||
"UserCacheCapacity": 10000
|
||||
"UserCacheCapacity": 10000,
|
||||
"MaxTweetRetention": 20
|
||||
},
|
||||
"Db": {
|
||||
"Type": "postgres",
|
||||
|
|
Loading…
Reference in New Issue