set tweet retention delay in settings
This commit is contained in:
parent
80ac1363e5
commit
405087360c
|
@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
|
||||||
* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
|
* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
|
||||||
* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
|
* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
|
||||||
* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
|
* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
|
||||||
|
* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
|
||||||
|
|
||||||
# Docker Compose full example
|
# Docker Compose full example
|
||||||
|
|
||||||
|
|
|
@ -18,5 +18,7 @@
|
||||||
public int UserCacheCapacity { get; set; }
|
public int UserCacheCapacity { get; set; }
|
||||||
public string IpWhiteListing { get; set; }
|
public string IpWhiteListing { get; set; }
|
||||||
public bool EnableXRealIpHeader { get; set; }
|
public bool EnableXRealIpHeader { get; set; }
|
||||||
|
|
||||||
|
public int MaxTweetRetention { get; set; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,7 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Folder Include="Processors\TweetsCleanUp\" />
|
<Folder Include="Processors\TweetsCleanUp\Base\" />
|
||||||
<Folder Include="Tools\" />
|
<Folder Include="Tools\" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
using BirdsiteLive.Common.Settings;
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base
|
||||||
|
{
|
||||||
|
public class RetentionBase
|
||||||
|
{
|
||||||
|
protected int GetRetentionTime(InstanceSettings settings)
|
||||||
|
{
|
||||||
|
var retentionTime = Math.Abs(settings.MaxTweetRetention);
|
||||||
|
if (retentionTime < 1) retentionTime = 1;
|
||||||
|
if (retentionTime > 90) retentionTime = 90;
|
||||||
|
return retentionTime;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -3,20 +3,24 @@ using System.Linq;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using System.Threading.Tasks.Dataflow;
|
using System.Threading.Tasks.Dataflow;
|
||||||
|
using BirdsiteLive.Common.Settings;
|
||||||
using BirdsiteLive.DAL.Contracts;
|
using BirdsiteLive.DAL.Contracts;
|
||||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||||
using BirdsiteLive.Pipeline.Models;
|
using BirdsiteLive.Pipeline.Models;
|
||||||
|
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||||
|
|
||||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||||
{
|
{
|
||||||
public class RetrieveTweetsToDeleteProcessor : IRetrieveTweetsToDeleteProcessor
|
public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor
|
||||||
{
|
{
|
||||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||||
|
private readonly InstanceSettings _instanceSettings;
|
||||||
|
|
||||||
#region Ctor
|
#region Ctor
|
||||||
public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal)
|
public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||||
{
|
{
|
||||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||||
|
_instanceSettings = instanceSettings;
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
@ -29,7 +33,9 @@ namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||||
ct.ThrowIfCancellationRequested();
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
var now = DateTime.UtcNow;
|
var now = DateTime.UtcNow;
|
||||||
var from = now.AddDays(-20);
|
|
||||||
|
|
||||||
|
var from = now.AddDays(-GetRetentionTime(_instanceSettings));
|
||||||
var dbBrowsingEnded = false;
|
var dbBrowsingEnded = false;
|
||||||
var lastId = -1L;
|
var lastId = -1L;
|
||||||
|
|
||||||
|
|
|
@ -1,26 +1,32 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
using BirdsiteLive.Common.Settings;
|
||||||
using BirdsiteLive.DAL.Contracts;
|
using BirdsiteLive.DAL.Contracts;
|
||||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||||
using BirdsiteLive.Pipeline.Models;
|
using BirdsiteLive.Pipeline.Models;
|
||||||
|
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||||
|
|
||||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||||
{
|
{
|
||||||
public class SaveDeletedTweetStatusProcessor : ISaveDeletedTweetStatusProcessor
|
public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor
|
||||||
{
|
{
|
||||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||||
|
private readonly InstanceSettings _instanceSettings;
|
||||||
|
|
||||||
#region Ctor
|
#region Ctor
|
||||||
public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal)
|
public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||||
{
|
{
|
||||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||||
|
_instanceSettings = instanceSettings;
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
|
public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
|
||||||
{
|
{
|
||||||
var highLimitDate = DateTime.UtcNow.AddDays(-40); //TODO get settings value
|
var retentionTime = GetRetentionTime(_instanceSettings);
|
||||||
|
retentionTime += 20; // Delay until last retry
|
||||||
|
var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime);
|
||||||
if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
|
if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
|
||||||
{
|
{
|
||||||
await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);
|
await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);
|
||||||
|
|
|
@ -25,7 +25,8 @@
|
||||||
"SensitiveTwitterAccounts": null,
|
"SensitiveTwitterAccounts": null,
|
||||||
"FailingTwitterUserCleanUpThreshold": 700,
|
"FailingTwitterUserCleanUpThreshold": 700,
|
||||||
"FailingFollowerCleanUpThreshold": 30000,
|
"FailingFollowerCleanUpThreshold": 30000,
|
||||||
"UserCacheCapacity": 10000
|
"UserCacheCapacity": 10000,
|
||||||
|
"MaxTweetRetention": 20
|
||||||
},
|
},
|
||||||
"Db": {
|
"Db": {
|
||||||
"Type": "postgres",
|
"Type": "postgres",
|
||||||
|
|
Loading…
Reference in New Issue