From 79ceab82b693ebd5c0aed4a95e31809293ad3f18 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 30 Dec 2022 22:45:34 -0500 Subject: [PATCH 01/27] added ip whitelisting --- .../Settings/InstanceSettings.cs | 1 + .../Middlewares/IpWhitelistingMiddleware.cs | 68 +++++++++++++++++++ src/BirdsiteLive/Startup.cs | 5 ++ 3 files changed, 74 insertions(+) create mode 100644 src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 0f701ff..4526c1b 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -16,5 +16,6 @@ public int FailingFollowerCleanUpThreshold { get; set; } = -1; public int UserCacheCapacity { get; set; } + public string IpWhiteListing { get; set; } } } diff --git a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs new file mode 100644 index 0000000..1846d8f --- /dev/null +++ b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs @@ -0,0 +1,68 @@ +using BirdsiteLive.Common.Settings; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; + +namespace BirdsiteLive.Middlewares +{ + public class IpWhitelistingMiddleware + { + private readonly RequestDelegate _next; + private readonly ILogger _logger; + private readonly byte[][] _safelist; + private readonly bool _ipWhitelistingSet; + + public IpWhitelistingMiddleware( + RequestDelegate next, + ILogger logger, + InstanceSettings instanceSettings) + { + if (!string.IsNullOrWhiteSpace(instanceSettings.IpWhiteListing)) + { + var ips = instanceSettings.IpWhiteListing.Split(';'); + _safelist = new byte[ips.Length][]; + for (var i = 0; i < ips.Length; i++) + { + _safelist[i] = IPAddress.Parse(ips[i]).GetAddressBytes(); + } + _ipWhitelistingSet = true; + } + + _next = next; + _logger = logger; + } + + public async Task Invoke(HttpContext context) + { + //if (context.Request.Method != HttpMethod.Get.Method) + if (_ipWhitelistingSet) + { + var remoteIp = context.Connection.RemoteIpAddress; + _logger.LogDebug("Request from Remote IP address: {RemoteIp}", remoteIp); + + var bytes = remoteIp.GetAddressBytes(); + var badIp = true; + foreach (var address in _safelist) + { + if (address.SequenceEqual(bytes)) + { + badIp = false; + break; + } + } + + if (badIp) + { + _logger.LogWarning("Forbidden Request from Remote IP address: {RemoteIp}", remoteIp); + context.Response.StatusCode = (int)HttpStatusCode.NotFound; + return; + } + } + + await _next.Invoke(context); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index 16c7ee4..e088a56 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -8,6 +8,7 @@ using BirdsiteLive.Common.Structs; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Settings; +using BirdsiteLive.Middlewares; using BirdsiteLive.Models; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Tools; @@ -18,6 +19,7 @@ using Microsoft.AspNetCore.HttpsPolicy; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace BirdsiteLive { @@ -132,6 +134,9 @@ namespace BirdsiteLive app.UseAuthorization(); + var instanceSettings = Configuration.GetSection("Instance").Get(); + app.UseMiddleware(instanceSettings); + app.UseEndpoints(endpoints => { endpoints.MapControllerRoute( From 0c2acc3d7afba6110063f62326ffbd241ec0fa2f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 30 Dec 2022 22:49:19 -0500 Subject: [PATCH 02/27] better ip parsing --- src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs index 1846d8f..e09ad27 100644 --- a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs +++ b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs @@ -1,4 +1,5 @@ using BirdsiteLive.Common.Settings; +using BirdsiteLive.Domain.Tools; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using System.Linq; @@ -22,7 +23,7 @@ namespace BirdsiteLive.Middlewares { if (!string.IsNullOrWhiteSpace(instanceSettings.IpWhiteListing)) { - var ips = instanceSettings.IpWhiteListing.Split(';'); + var ips = PatternsParser.Parse(instanceSettings.IpWhiteListing); _safelist = new byte[ips.Length][]; for (var i = 0; i < ips.Length; i++) { @@ -37,7 +38,6 @@ namespace BirdsiteLive.Middlewares public async Task Invoke(HttpContext context) { - //if (context.Request.Method != HttpMethod.Get.Method) if (_ipWhitelistingSet) { var remoteIp = context.Connection.RemoteIpAddress; From 8bbbe037c412193bbd06e5f93f34cb241cc4af38 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 30 Dec 2022 22:52:11 -0500 Subject: [PATCH 03/27] added IP whitelisting --- VARIABLES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/VARIABLES.md b/VARIABLES.md index 2ef0cdf..990e045 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -51,6 +51,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins) * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc) * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. +* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided). # Docker Compose full example From 8f420325124e0709d6ad23b35517a11a04f067f6 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 31 Dec 2022 00:05:12 -0500 Subject: [PATCH 04/27] added redirected remote ip detection --- src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs index e09ad27..b8490ae 100644 --- a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs +++ b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs @@ -41,6 +41,14 @@ namespace BirdsiteLive.Middlewares if (_ipWhitelistingSet) { var remoteIp = context.Connection.RemoteIpAddress; + + var forwardedIp = context.Request.Headers.FirstOrDefault(x => x.Key == "X-Real-IP").Value.ToString(); + if (!string.IsNullOrWhiteSpace(forwardedIp)) + { + _logger.LogDebug("Redirected IP address detected"); + remoteIp = IPAddress.Parse(forwardedIp); + } + _logger.LogDebug("Request from Remote IP address: {RemoteIp}", remoteIp); var bytes = remoteIp.GetAddressBytes(); From f8ab522505edc8054beb85e62e2798d10aa1341f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 31 Dec 2022 00:30:00 -0500 Subject: [PATCH 05/27] updated doc for ip whitelisting --- INSTALLATION.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/INSTALLATION.md b/INSTALLATION.md index 74d9131..f39fd42 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -186,6 +186,32 @@ services: + command: --interval 300 ``` +## IP Whitelisting + +If you want to use the IP Whitelisting functionality (see related [variable](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)) and you are using the nginx reverse proxy set as before, please add the following: + +``` +sudo nano /etc/nginx/sites-enabled/{your-domain-name.com} +``` + +``` diff +server { + listen 80; + server_name {your-domain-name.com}; + location / { + proxy_pass http://localhost:5000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; ++ proxy_set_header X-Real-IP $remote_addr; + } +} +``` + ## More options You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) From 432484eaaa8314b750f1f96712ebc6f561f5d22c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 31 Dec 2022 01:29:55 -0500 Subject: [PATCH 06/27] testing deletion --- .../Models/Tombstone.cs | 9 ++++++ .../Controllers/DebugingController.cs | 31 +++++++++++++++++++ src/BirdsiteLive/Views/Debuging/Index.cshtml | 5 +++ 3 files changed, 45 insertions(+) create mode 100644 src/BirdsiteLive.ActivityPub/Models/Tombstone.cs diff --git a/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs b/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs new file mode 100644 index 0000000..49dcabe --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.ActivityPub.Models +{ + public class Tombstone + { + public string id { get; set; } + public readonly string type = "Tombstone"; + public string atomUrl { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 8f37f22..f3e7d4e 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -56,6 +56,35 @@ namespace BirdsiteLive.Controllers return View("Index"); } + private static string _noteId; + + [HttpPost] + public async Task DeleteNote() + { + var username = "twitter"; + var actor = $"https://{_instanceSettings.Domain}/users/{username}"; + var targetHost = "ioc.exchange"; + var target = $"https://{targetHost}/users/test"; + var inbox = $"/inbox"; + + var delete = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{_noteId}#delete", + type = "Delete", + actor = actor, + to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = new Tombstone + { + id = _noteId, + atomUrl = _noteId + } + }; + + await _activityPubService.PostDataAsync(delete, targetHost, actor, inbox); + return View("Index"); + } + [HttpPost] public async Task PostNote() { @@ -70,6 +99,8 @@ namespace BirdsiteLive.Controllers var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}"; var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}"; + _noteId = noteId; + var to = $"{actor}/followers"; to = target; diff --git a/src/BirdsiteLive/Views/Debuging/Index.cshtml b/src/BirdsiteLive/Views/Debuging/Index.cshtml index e343cf2..886b88d 100644 --- a/src/BirdsiteLive/Views/Debuging/Index.cshtml +++ b/src/BirdsiteLive/Views/Debuging/Index.cshtml @@ -18,6 +18,11 @@ +
+ + + +
From 584266a0404832481c41e458c2c4c97892bdd325 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 31 Dec 2022 03:41:31 -0500 Subject: [PATCH 07/27] created syncTweets DAL --- .../DbInitializerPostgresDal.cs | 24 ++- .../DataAccessLayers/SyncTweetsPostgresDal.cs | 94 ++++++++++ .../Settings/PostgresSettings.cs | 1 + .../Contracts/ISyncTweetsPostgresDal.cs | 15 ++ .../BirdsiteLive.DAL/Models/SyncTweet.cs | 14 ++ .../Base/PostgresTestingBase.cs | 1 + .../SyncTweetsPostgresDalTests.cs | 166 ++++++++++++++++++ 7 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs create mode 100644 src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs create mode 100644 src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs create mode 100644 src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 55b38f6..f30dfa7 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 5); + private readonly Version _currentVersion = new Version(2, 6); private const string DbVersionType = "db-version"; #region Ctor @@ -136,7 +136,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers new Tuple(new Version(2,1), new Version(2,2)), new Tuple(new Version(2,2), new Version(2,3)), new Tuple(new Version(2,3), new Version(2,4)), - new Tuple(new Version(2,4), new Version(2,5)) + new Tuple(new Version(2,4), new Version(2,5)), + new Tuple(new Version(2,5), new Version(2,6)) }; } @@ -184,6 +185,22 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN"; await _tools.ExecuteRequestAsync(addDeletedToAcct); } + else if (from == new Version(2, 5) && to == new Version(2, 6)) + { + // Create Synchronized Tweets table + var createFollowers = $@"CREATE TABLE {_settings.SynchronizedTweetsTableName} + ( + id SERIAL PRIMARY KEY, + + acct VARCHAR(50) NOT NULL, + tweetId BIGINT NOT NULL, + inbox VARCHAR(2048) NOT NULL, + publishedAt TIMESTAMP (2) WITHOUT TIME ZONE NOT NULL, + + UNIQUE (acct, tweetId, inbox) + );"; + await _tools.ExecuteRequestAsync(createFollowers); + } else { throw new NotImplementedException(); @@ -212,7 +229,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers $@"DROP TABLE {_settings.DbVersionTableName};", $@"DROP TABLE {_settings.TwitterUserTableName};", $@"DROP TABLE {_settings.FollowersTableName};", - $@"DROP TABLE {_settings.CachedTweetsTableName};" + $@"DROP TABLE {_settings.CachedTweetsTableName};", + $@"DROP TABLE {_settings.SynchronizedTweetsTableName};" }; foreach (var r in dropsRequests) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs new file mode 100644 index 0000000..c38a141 --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.DAL.Postgres.DataAccessLayers.Base; +using BirdsiteLive.DAL.Postgres.Settings; +using Dapper; + +namespace BirdsiteLive.DAL.Postgres.DataAccessLayers +{ + public class SyncTweetsPostgresDal : PostgresBase, ISyncTweetsPostgresDal + { + #region Ctor + public SyncTweetsPostgresDal(PostgresSettings settings) : base(settings) + { + } + #endregion + + public async Task SaveTweetAsync(SyncTweet tweet) + { + if (tweet.PublishedAt == default) throw new ArgumentException("publishedAt"); + if (tweet.TweetId == default) throw new ArgumentException("tweetId"); + + var acct = tweet.Acct.ToLowerInvariant().Trim(); + var inbox = tweet.Inbox.ToLowerInvariant().Trim(); + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + return (await dbConnection.QueryAsync( + $"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,publishedAt) VALUES(@acct,@tweetId,@inbox,@publishedAt) RETURNING id", + new + { + acct, + tweetId = tweet.TweetId, + inbox, + publishedAt = tweet.PublishedAt.ToUniversalTime() + })).First(); + } + } + + public async Task GetTweetAsync(long id) + { + if (id == default) throw new ArgumentException("id"); + + var query = $"SELECT * FROM {_settings.SynchronizedTweetsTableName} WHERE id = @id"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = (await dbConnection.QueryAsync(query, new { id })).FirstOrDefault(); + return result; + } + } + + public async Task DeleteTweetAsync(long id) + { + if (id == default) throw new ArgumentException("id"); + + var query = $"DELETE FROM {_settings.SynchronizedTweetsTableName} WHERE id = @id"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + await dbConnection.QueryAsync(query, new { id }); + } + } + + public async Task> GetTweetsOlderThanAsync(DateTime date, long from = -1, int size = 100) + { + if (date == default) throw new ArgumentException("date"); + + var query = $"SELECT * FROM {_settings.SynchronizedTweetsTableName} WHERE id > @from AND publishedAt < @date ORDER BY id ASC LIMIT @size"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = await dbConnection.QueryAsync(query, new + { + from, + date, + size + }); + return result.ToList(); + } + } + } +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs index c7504ef..6be43da 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs @@ -8,5 +8,6 @@ public string TwitterUserTableName { get; set; } = "twitter_users"; public string FollowersTableName { get; set; } = "followers"; public string CachedTweetsTableName { get; set; } = "cached_tweets"; + public string SynchronizedTweetsTableName { get; set; } = "sync_tweets"; } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs new file mode 100644 index 0000000..42f843f --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.DAL.Contracts +{ + public interface ISyncTweetsPostgresDal + { + Task SaveTweetAsync(SyncTweet tweet); + Task GetTweetAsync(long id); + Task DeleteTweetAsync(long id); + Task> GetTweetsOlderThanAsync(DateTime date, long from = -1, int size = 100); + } +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs new file mode 100644 index 0000000..c8879a3 --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs @@ -0,0 +1,14 @@ +using System; + +namespace BirdsiteLive.DAL.Models +{ + public class SyncTweet + { + public long Id { get; set; } + + public string Acct { get; set; } + public string Inbox { get; set; } + public long TweetId { get; set; } + public DateTime PublishedAt { get; set; } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs index 72bf352..00f0f9c 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs @@ -19,6 +19,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4), FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4), TwitterUserTableName = "TwitterUserTableName" + RandomGenerator.GetString(4), + SynchronizedTweetsTableName = "SynchronizedTweetsTableName" + RandomGenerator.GetString(4), }; _tools = new PostgresTools(_settings); } diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs new file mode 100644 index 0000000..d252359 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs @@ -0,0 +1,166 @@ +using System; +using System.Linq; +using BirdsiteLive.DAL.Postgres.DataAccessLayers; +using BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers +{ + [TestClass] + public class SyncTweetsPostgresDalTests : PostgresTestingBase + { + [TestInitialize] + public async Task TestInit() + { + var dal = new DbInitializerPostgresDal(_settings, _tools); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); + } + + [TestCleanup] + public async Task CleanUp() + { + var dal = new DbInitializerPostgresDal(_settings, _tools); + await dal.DeleteAllAsync(); + } + + [TestMethod] + public async Task CreateAndGetTweets() + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = DateTime.UtcNow, + Inbox = "https://instance.ext/inbox", + TweetId = 4567889 + }; + + var dal = new SyncTweetsPostgresDal(_settings); + + var id = await dal.SaveTweetAsync(tweet); + var result = await dal.GetTweetAsync(id); + + Assert.IsNotNull(result); + + Assert.IsTrue(result.Id > 0); + Assert.AreEqual(tweet.Acct, result.Acct); + Assert.AreEqual(tweet.Inbox, result.Inbox); + Assert.AreEqual(tweet.TweetId, result.TweetId); + Assert.IsTrue(Math.Abs((tweet.PublishedAt - result.PublishedAt).Seconds) < 5); + } + + [TestMethod] + public async Task CreateDeleteAndGetTweets() + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = DateTime.UtcNow, + Inbox = "https://instance.ext/inbox", + TweetId = 4567889 + }; + + var dal = new SyncTweetsPostgresDal(_settings); + + var id = await dal.SaveTweetAsync(tweet); + await dal.DeleteTweetAsync(id); + var result = await dal.GetTweetAsync(id); + + Assert.IsNull(result); + } + + [TestMethod] + public async Task CreateAndGetTweetsByDate() + { + var now = DateTime.UtcNow; + var dal = new SyncTweetsPostgresDal(_settings); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = now.AddDays(-10 - i), + Inbox = "https://instance.ext/inbox", + TweetId = 4567889 + i + }; + await dal.SaveTweetAsync(tweet); + } + + var date = now.AddDays(-20); + var result = await dal.GetTweetsOlderThanAsync(date, -1, 10); + + Assert.IsNotNull(result); + Assert.IsTrue(result.Count == 10); + + foreach (var res in result) + { + Assert.IsTrue(res.PublishedAt < date); + Assert.IsTrue(res.Id > 10); + Assert.IsTrue(res.Id < 25); + } + } + + [TestMethod] + public async Task CreateAndGetTweetsByDate_Offset() + { + var now = DateTime.UtcNow; + var dal = new SyncTweetsPostgresDal(_settings); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = now.AddDays(-10 - i), + Inbox = "https://instance.ext/inbox", + TweetId = 4567889 + i + }; + await dal.SaveTweetAsync(tweet); + } + + var date = now.AddDays(-20); + var result = await dal.GetTweetsOlderThanAsync(date, 1000, 10); + + Assert.IsNotNull(result); + Assert.IsTrue(result.Count == 0); + } + + [TestMethod] + public async Task CreateAndGetTweetsByDate_Iteration() + { + var now = DateTime.UtcNow; + var dal = new SyncTweetsPostgresDal(_settings); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = now.AddDays(-10 - i), + Inbox = "https://instance.ext/inbox", + TweetId = 4567889 + i + }; + await dal.SaveTweetAsync(tweet); + } + + var date = now.AddDays(-20); + var result = await dal.GetTweetsOlderThanAsync(date, -1, 10); + var result2 = await dal.GetTweetsOlderThanAsync(date, result.Last().Id, 10); + + var global = result.ToList(); + global.AddRange(result2); + var d = global.GroupBy(x => x.Id).Count(); + Assert.AreEqual(20, d); + + foreach (var res in global) + { + Assert.IsTrue(res.PublishedAt < date); + Assert.IsTrue(res.Id > 10); + Assert.IsTrue(res.Id < 35); + } + } + } +} \ No newline at end of file From d80a00136db3600ab44eb38e00db31f108867302 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 31 Dec 2022 19:26:14 -0500 Subject: [PATCH 08/27] enable XRealIP Header manually --- INSTALLATION.md | 38 +++++++++++++++++++ VARIABLES.md | 1 + .../Settings/InstanceSettings.cs | 1 + .../Middlewares/IpWhitelistingMiddleware.cs | 14 +++++-- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/INSTALLATION.md b/INSTALLATION.md index f39fd42..569c2a1 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -212,6 +212,44 @@ server { } ``` +And edit the docker-compose file as follow: + +```diff +version: "3" + +networks: + birdsitelivenetwork: + external: false + +services: + server: + image: nicolasconstant/birdsitelive:latest + restart: always + container_name: birdsitelive + environment: + - Instance:Domain=domain.name + - Instance:AdminEmail=name@domain.ext ++ - Instance:IpWhiteListing=127.0.0.1;127.0.0.2 ++ - Instance:EnableXRealIpHeader=true + - Db:Type=postgres + - Db:Host=db + - Db:Name=birdsitelive + - Db:User=birdsitelive + - Db:Password=birdsitelive + - Twitter:ConsumerKey=twitter.api.key + - Twitter:ConsumerSecret=twitter.api.key + networks: + - birdsitelivenetwork + ports: + - "5000:80" + depends_on: + - db + + db: + image: postgres:9.6 + [...] +``` + ## More options You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) diff --git a/VARIABLES.md b/VARIABLES.md index 990e045..f43bddc 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -52,6 +52,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc) * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. * `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided). +* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy). # Docker Compose full example diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 4526c1b..a67977c 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -17,5 +17,6 @@ public int UserCacheCapacity { get; set; } public string IpWhiteListing { get; set; } + public bool EnableXRealIpHeader { get; set; } } } diff --git a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs index b8490ae..da4c275 100644 --- a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs +++ b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs @@ -13,6 +13,7 @@ namespace BirdsiteLive.Middlewares { private readonly RequestDelegate _next; private readonly ILogger _logger; + private readonly InstanceSettings _instanceSettings; private readonly byte[][] _safelist; private readonly bool _ipWhitelistingSet; @@ -34,6 +35,7 @@ namespace BirdsiteLive.Middlewares _next = next; _logger = logger; + _instanceSettings = instanceSettings; } public async Task Invoke(HttpContext context) @@ -42,11 +44,15 @@ namespace BirdsiteLive.Middlewares { var remoteIp = context.Connection.RemoteIpAddress; - var forwardedIp = context.Request.Headers.FirstOrDefault(x => x.Key == "X-Real-IP").Value.ToString(); - if (!string.IsNullOrWhiteSpace(forwardedIp)) + if (_instanceSettings.EnableXRealIpHeader) { - _logger.LogDebug("Redirected IP address detected"); - remoteIp = IPAddress.Parse(forwardedIp); + var forwardedIp = context.Request.Headers.FirstOrDefault(x => x.Key == "X-Real-IP").Value + .ToString(); + if (!string.IsNullOrWhiteSpace(forwardedIp)) + { + _logger.LogDebug("Redirected IP address detected"); + remoteIp = IPAddress.Parse(forwardedIp); + } } _logger.LogDebug("Request from Remote IP address: {RemoteIp}", remoteIp); From 3460c728951aafb8c865be00b7f2551daa2ae459 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 5 Jan 2023 23:55:50 -0500 Subject: [PATCH 09/27] saving synchronized tweets --- .../Processors/SubTasks/SendTweetsTaskBase.cs | 31 +++++++ .../SubTasks/SendTweetsToInboxTask.cs | 8 +- .../SubTasks/SendTweetsToSharedInboxTask.cs | 6 +- .../SubTasks/SendTweetsToInboxTaskTests.cs | 80 +++++++++++++++++-- .../SubTasks/SendTweetsToSharedInboxTests.cs | 78 ++++++++++++++++-- 5 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs new file mode 100644 index 0000000..79fd5ef --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Pipeline.Processors.SubTasks; + +public class SendTweetsTaskBase +{ + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + + #region Ctor + protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + } + #endregion + + protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox) + { + var inboxUrl = $"https://{host}/{inbox.Trim('/')}"; + var tweet = new SyncTweet + { + Acct = acct, + TweetId = tweetId, + PublishedAt = DateTime.UtcNow, + Inbox = inboxUrl + }; + await _syncTweetsPostgresDal.SaveTweetAsync(tweet); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index a6f6982..0c91750 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -17,17 +17,16 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks Task ExecuteAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user); } - public class SendTweetsToInboxTask : ISendTweetsToInboxTask + public class SendTweetsToInboxTask : SendTweetsTaskBase, ISendTweetsToInboxTask { private readonly IActivityPubService _activityPubService; private readonly IStatusService _statusService; private readonly IFollowersDal _followersDal; private readonly InstanceSettings _settings; private readonly ILogger _logger; - - + #region Ctor - public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) + public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal) { _activityPubService = activityPubService; _statusService = statusService; @@ -61,6 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { var note = _statusService.GetStatus(user.Acct, tweet); await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + await SaveSyncTweetAsync(user.Acct, tweet.Id, follower.Host, inbox); } } catch (ArgumentException e) diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index 1abe183..95f32fd 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; @@ -16,7 +17,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance); } - public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask + public class SendTweetsToSharedInboxTask : SendTweetsTaskBase, ISendTweetsToSharedInboxTask { private readonly IStatusService _statusService; private readonly IActivityPubService _activityPubService; @@ -25,7 +26,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks private readonly ILogger _logger; #region Ctor - public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) + public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal) { _activityPubService = activityPubService; _statusService = statusService; @@ -61,6 +62,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { var note = _statusService.GetStatus(user.Acct, tweet); await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + await SaveSyncTweetAsync(user.Acct, tweet.Id, host, inbox); } } catch (ArgumentException e) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs index 367a642..81361ed 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; @@ -87,15 +88,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -155,15 +166,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -237,15 +251,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -319,15 +343,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -402,15 +436,28 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); + } #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -493,9 +540,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId2 + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); + #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -507,6 +564,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } @@ -571,15 +629,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -634,9 +695,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -649,6 +712,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs index 7ab06a2..5e61379 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs @@ -109,15 +109,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -199,15 +209,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -302,15 +315,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -405,15 +428,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -509,15 +542,28 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); + } #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -621,9 +667,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId2 + && y.Inbox == $"https://{host}{inbox}" + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -635,6 +690,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } @@ -720,15 +776,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -800,9 +859,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -815,6 +876,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } From 5b32a9021f71c5418e583e2ee75ab034d6966c61 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 5 Jan 2023 23:58:58 -0500 Subject: [PATCH 10/27] fix CICD --- .../Processors/SubTasks/SendTweetsTaskBase.cs | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs index 79fd5ef..58ae052 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs @@ -3,29 +3,30 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -namespace BirdsiteLive.Pipeline.Processors.SubTasks; - -public class SendTweetsTaskBase +namespace BirdsiteLive.Pipeline.Processors.SubTasks { - private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; - - #region Ctor - protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal) + public class SendTweetsTaskBase { - _syncTweetsPostgresDal = syncTweetsPostgresDal; - } - #endregion + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; - protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox) - { - var inboxUrl = $"https://{host}/{inbox.Trim('/')}"; - var tweet = new SyncTweet + #region Ctor + protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal) { - Acct = acct, - TweetId = tweetId, - PublishedAt = DateTime.UtcNow, - Inbox = inboxUrl - }; - await _syncTweetsPostgresDal.SaveTweetAsync(tweet); + _syncTweetsPostgresDal = syncTweetsPostgresDal; + } + #endregion + + protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox) + { + var inboxUrl = $"https://{host}/{inbox.Trim('/')}"; + var tweet = new SyncTweet + { + Acct = acct, + TweetId = tweetId, + PublishedAt = DateTime.UtcNow, + Inbox = inboxUrl + }; + await _syncTweetsPostgresDal.SaveTweetAsync(tweet); + } } } \ No newline at end of file From b223bb02162272aa7e19eafc273a589f9894cc2c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 00:28:45 -0500 Subject: [PATCH 11/27] refactoring + init workerservice to delete tweets --- .../BirdsiteLive.Pipeline.csproj | 1 + .../IRefreshTwitterUserStatusProcessor.cs | 2 +- .../IRetrieveFollowersProcessor.cs | 2 +- .../IRetrieveTweetsProcessor.cs | 2 +- .../IRetrieveTwitterUsersProcessor.cs | 2 +- .../ISaveProgressionProcessor.cs | 2 +- .../ISendTweetsToFollowersProcessor.cs | 2 +- .../RefreshTwitterUserStatusProcessor.cs | 4 +- .../RetrieveFollowersProcessor.cs | 4 +- .../RetrieveTweetsProcessor.cs | 6 +-- .../RetrieveTwitterUsersProcessor.cs | 8 ++-- .../SaveProgressionProcessor.cs | 8 ++-- .../SendTweetsToFollowersProcessor.cs | 8 ++-- .../SubTasks/SendTweetsTaskBase.cs | 0 .../SubTasks/SendTweetsToInboxTask.cs | 0 .../SubTasks/SendTweetsToSharedInboxTask.cs | 0 .../TweetsCleanUp/DeleteTweetsProcessor.cs | 7 +++ .../RetrieveTweetsToDeleteProcessor.cs | 7 +++ .../SaveDeletedTweetStatusProcessor.cs | 7 +++ .../StatusPublicationPipeline.cs | 2 +- .../TweetCleanUpPipeline.cs | 18 ++++++++ src/BirdsiteLive/Program.cs | 1 + .../Services/FederationService.cs | 2 + .../Services/TweetCleanUpService.cs | 43 +++++++++++++++++++ .../Tools/InitStateSynchronization.cs | 7 +++ .../RetrieveTweetsProcessorTests.cs | 2 +- .../RetrieveTwitterUsersProcessorTests.cs | 2 +- .../SaveProgressionProcessorTests.cs | 2 +- .../SendTweetsToFollowersProcessorTests.cs | 2 +- .../StatusPublicationPipelineTests.cs | 2 +- 30 files changed, 124 insertions(+), 31 deletions(-) rename src/BirdsiteLive.Pipeline/Contracts/{ => Federation}/IRefreshTwitterUserStatusProcessor.cs (84%) rename src/BirdsiteLive.Pipeline/Contracts/{ => Federation}/IRetrieveFollowersProcessor.cs (89%) rename src/BirdsiteLive.Pipeline/Contracts/{ => Federation}/IRetrieveTweetsProcessor.cs (84%) rename src/BirdsiteLive.Pipeline/Contracts/{ => Federation}/IRetrieveTwitterUsersProcessor.cs (85%) rename src/BirdsiteLive.Pipeline/Contracts/{ => Federation}/ISaveProgressionProcessor.cs (81%) rename src/BirdsiteLive.Pipeline/Contracts/{ => Federation}/ISendTweetsToFollowersProcessor.cs (83%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/RefreshTwitterUserStatusProcessor.cs (97%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/RetrieveFollowersProcessor.cs (89%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/RetrieveTweetsProcessor.cs (96%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/RetrieveTwitterUsersProcessor.cs (93%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/SaveProgressionProcessor.cs (94%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/SendTweetsToFollowersProcessor.cs (97%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/SubTasks/SendTweetsTaskBase.cs (100%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/SubTasks/SendTweetsToInboxTask.cs (100%) rename src/BirdsiteLive.Pipeline/Processors/{ => Federation}/SubTasks/SendTweetsToSharedInboxTask.cs (100%) create mode 100644 src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs create mode 100644 src/BirdsiteLive/Services/TweetCleanUpService.cs create mode 100644 src/BirdsiteLive/Tools/InitStateSynchronization.cs diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 8601b19..a190af6 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -19,6 +19,7 @@ + diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs similarity index 84% rename from src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs index 9f20e59..bf74efd 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRefreshTwitterUserStatusProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs similarity index 89% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs index a9ef35c..567d41b 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs @@ -3,7 +3,7 @@ using System.Threading; using System.Threading.Tasks; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRetrieveFollowersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs similarity index 84% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs index 49712c2..0cbf2cb 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRetrieveTweetsProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs similarity index 85% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs index b71ae93..c6d08d0 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRetrieveTwitterUsersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs similarity index 81% rename from src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs index 6b1c9ba..f0320cb 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface ISaveProgressionProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs similarity index 83% rename from src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs index 33db423..49d47f0 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface ISendTweetsToFollowersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs similarity index 97% rename from src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs index 739d50b..a3e7ab2 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs @@ -6,12 +6,12 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs similarity index 89% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs index 57e3e49..7f696f7 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs @@ -2,10 +2,10 @@ using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs similarity index 96% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs index 321fbf0..d38feb2 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs @@ -5,14 +5,14 @@ using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; using Tweetinvi.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor { @@ -64,7 +64,7 @@ namespace BirdsiteLive.Pipeline.Processors private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user) { var tweets = new ExtractedTweet[0]; - + try { if (user.LastTweetPostedId == -1) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs similarity index 93% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs index d9d0ffb..7d16ac5 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs @@ -7,18 +7,18 @@ using BirdsiteLive.Common.Extensions; using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; private readonly IMaxUsersNumberProvider _maxUsersNumberProvider; private readonly ILogger _logger; - + public int WaitFactor = 1000 * 60; //1 min #region Ctor @@ -42,7 +42,7 @@ namespace BirdsiteLive.Pipeline.Processors var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false); var userCount = users.Any() ? users.Length : 1; - var splitNumber = (int) Math.Ceiling(userCount / 15d); + var splitNumber = (int)Math.Ceiling(userCount / 15d); var splitUsers = users.Split(splitNumber).ToList(); foreach (var u in splitUsers) diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs similarity index 94% rename from src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs index 1f94871..0b29769 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs @@ -5,11 +5,11 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using Microsoft.Extensions.Logging; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class SaveProgressionProcessor : ISaveProgressionProcessor { @@ -36,13 +36,13 @@ namespace BirdsiteLive.Pipeline.Processors await UpdateUserSyncDateAsync(userWithTweetsToSync.User); return; } - if(userWithTweetsToSync.Followers.Length == 0) + if (userWithTweetsToSync.Followers.Length == 0) { _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct); await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User); return; } - + var userId = userWithTweetsToSync.User.Id; var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList(); var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs similarity index 97% rename from src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs index e210f39..3a52544 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs @@ -10,7 +10,7 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter; @@ -18,7 +18,7 @@ using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; using Tweetinvi.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor { @@ -83,7 +83,7 @@ namespace BirdsiteLive.Pipeline.Processors } } } - + private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List followerWtInbox, SyncTwitterUser user) { foreach (var follower in followerWtInbox) @@ -114,7 +114,7 @@ namespace BirdsiteLive.Pipeline.Processors { follower.PostingErrorCount++; - if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold + if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold && _instanceSettings.FailingFollowerCleanUpThreshold > 0 || follower.PostingErrorCount > 2147483600) { diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs similarity index 100% rename from src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsTaskBase.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToInboxTask.cs similarity index 100% rename from src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToInboxTask.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToSharedInboxTask.cs similarity index 100% rename from src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToSharedInboxTask.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs new file mode 100644 index 0000000..9dbb765 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +{ + public class DeleteTweetsProcessor + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs new file mode 100644 index 0000000..dd2c565 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +{ + public class RetrieveTweetsToDeleteProcessor + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs new file mode 100644 index 0000000..ccf2181 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +{ + public class SaveDeletedTweetStatusProcessor + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index c6917e7..f0a81e6 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using Microsoft.Extensions.Logging; diff --git a/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs b/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs new file mode 100644 index 0000000..9dfc451 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs @@ -0,0 +1,18 @@ +using System.Threading.Tasks; +using System.Threading; + +namespace BirdsiteLive.Pipeline +{ + public interface ITweetCleanUpPipeline + { + Task ExecuteAsync(CancellationToken ct); + } + + public class TweetCleanUpPipeline : ITweetCleanUpPipeline + { + public async Task ExecuteAsync(CancellationToken ct) + { + throw new System.NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Program.cs b/src/BirdsiteLive/Program.cs index d238b02..7f29e76 100644 --- a/src/BirdsiteLive/Program.cs +++ b/src/BirdsiteLive/Program.cs @@ -29,6 +29,7 @@ namespace BirdsiteLive .ConfigureServices(services => { services.AddHostedService(); + services.AddHostedService(); }); } } diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 0b0faed..32e8b5e 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -6,6 +6,7 @@ using BirdsiteLive.DAL; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Moderation; using BirdsiteLive.Pipeline; +using BirdsiteLive.Tools; using Microsoft.Extensions.Hosting; namespace BirdsiteLive.Services @@ -32,6 +33,7 @@ namespace BirdsiteLive.Services try { await _databaseInitializer.InitAndMigrateDbAsync(); + InitStateSynchronization.IsDbInitialized = true; await _moderationPipeline.ApplyModerationSettingsAsync(); await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } diff --git a/src/BirdsiteLive/Services/TweetCleanUpService.cs b/src/BirdsiteLive/Services/TweetCleanUpService.cs new file mode 100644 index 0000000..c2d81bd --- /dev/null +++ b/src/BirdsiteLive/Services/TweetCleanUpService.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline; +using BirdsiteLive.Tools; +using Microsoft.Extensions.Hosting; + +namespace BirdsiteLive.Services +{ + public class TweetCleanUpService : BackgroundService + { + private readonly ITweetCleanUpPipeline _cleanUpPipeline; + private readonly IHostApplicationLifetime _applicationLifetime; + + #region Ctor + public TweetCleanUpService(IHostApplicationLifetime applicationLifetime, ITweetCleanUpPipeline cleanUpPipeline) + { + _applicationLifetime = applicationLifetime; + _cleanUpPipeline = cleanUpPipeline; + } + #endregion + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + // Wait for initialization + while (!InitStateSynchronization.IsDbInitialized) + { + if (stoppingToken.IsCancellationRequested) return; + await Task.Delay(250, stoppingToken); + } + + await _cleanUpPipeline.ExecuteAsync(stoppingToken); + } + catch (Exception) + { + await Task.Delay(1000 * 30); + _applicationLifetime.StopApplication(); + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Tools/InitStateSynchronization.cs b/src/BirdsiteLive/Tools/InitStateSynchronization.cs new file mode 100644 index 0000000..c2a2e42 --- /dev/null +++ b/src/BirdsiteLive/Tools/InitStateSynchronization.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Tools +{ + public static class InitStateSynchronization + { + public static bool IsDbInitialized { get; set; } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index f95ad82..1f69775 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index daf0bfa..8a5a5ba 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index d245713..f38d806 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -2,7 +2,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs index 8a78038..bd03747 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs @@ -7,7 +7,7 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs index 81eeb59..d21d296 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; From e579e1b11c1f1448055142cc512cc76371abdb49 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 01:50:45 -0500 Subject: [PATCH 12/27] first implementation of tweet clean up's pipeline --- src/BirdsiteLive.Domain/ActivityPubService.cs | 26 ++++++++ .../TweetsCleanUp/IDeleteTweetsProcessor.cs | 11 ++++ .../IRetrieveTweetsToDeleteProcessor.cs | 12 ++++ .../ISaveDeletedTweetStatusProcessor.cs | 11 ++++ .../Models/TweetToDelete.cs | 10 ++++ .../TweetsCleanUp/DeleteTweetsProcessor.cs | 43 ++++++++++++- .../RetrieveTweetsToDeleteProcessor.cs | 57 +++++++++++++++++- .../SaveDeletedTweetStatusProcessor.cs | 29 ++++++++- .../TweetCleanUpPipeline.cs | 60 ++++++++++++++++++- .../BirdsiteLive.DAL/Models/SyncTweet.cs | 1 + 10 files changed, 250 insertions(+), 10 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index cab0c58..9a893c7 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -9,6 +9,7 @@ using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Converters; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -22,6 +23,7 @@ namespace BirdsiteLive.Domain Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); Task DeleteUserAsync(string username, string targetHost, string targetInbox); + Task DeleteNoteAsync(SyncTweet tweet); } public class WebFinger @@ -108,6 +110,30 @@ namespace BirdsiteLive.Domain } } + public async Task DeleteNoteAsync(SyncTweet tweet) + { + var acct = tweet.Acct.ToLowerInvariant().Trim(); + + var actor = $"https://{_instanceSettings.Domain}/users/{acct}"; + var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}"; + + var delete = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{noteId}#delete", + type = "Delete", + actor = actor, + to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = new Tombstone + { + id = noteId, + atomUrl = noteId + } + }; + + await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox); + } + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { try diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs new file mode 100644 index 0000000..c74659d --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp +{ + public interface IDeleteTweetsProcessor + { + Task ProcessAsync(TweetToDelete tweet, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs new file mode 100644 index 0000000..45f9442 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp +{ + public interface IRetrieveTweetsToDeleteProcessor + { + Task GetTweetsAsync(BufferBlock tweetsBufferBlock, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs new file mode 100644 index 0000000..0486fb9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp +{ + public interface ISaveDeletedTweetStatusProcessor + { + Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs b/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs new file mode 100644 index 0000000..dd411f9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs @@ -0,0 +1,10 @@ +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Pipeline.Models +{ + public class TweetToDelete + { + public SyncTweet Tweet { get; set; } + public bool DeleteSuccessful { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs index 9dbb765..04d844c 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs @@ -1,7 +1,44 @@ -namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class DeleteTweetsProcessor + public class DeleteTweetsProcessor : IDeleteTweetsProcessor { - + private readonly IActivityPubService _activityPubService; + private readonly ILogger _logger; + + #region Ctor + public DeleteTweetsProcessor(IActivityPubService activityPubService, ILogger logger) + { + _activityPubService = activityPubService; + _logger = logger; + } + #endregion + + public async Task ProcessAsync(TweetToDelete tweet, CancellationToken ct) + { + try + { + await _activityPubService.DeleteNoteAsync(tweet.Tweet); + tweet.DeleteSuccessful = true; + } + catch (HttpRequestException e) + { + //TODO check code under .NET 5 + } + catch (Exception e) + { + _logger.LogError(e.Message, e); + } + + return tweet; + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs index dd2c565..82c6ef7 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs @@ -1,7 +1,58 @@ -namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class RetrieveTweetsToDeleteProcessor + public class RetrieveTweetsToDeleteProcessor : IRetrieveTweetsToDeleteProcessor { - + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + + #region Ctor + public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + } + #endregion + + public async Task GetTweetsAsync(BufferBlock tweetsBufferBlock, CancellationToken ct) + { + var batchSize = 100; + + for (;;) + { + ct.ThrowIfCancellationRequested(); + + var now = DateTime.UtcNow; + var from = now.AddDays(-20); + var dbBrowsingEnded = false; + var lastId = -1L; + + do + { + var tweets = await _syncTweetsPostgresDal.GetTweetsOlderThanAsync(from, lastId, batchSize); + + foreach (var syncTweet in tweets) + { + var tweet = new TweetToDelete + { + Tweet = syncTweet + }; + await tweetsBufferBlock.SendAsync(tweet, ct); + } + + if (tweets.Any()) lastId = tweets.Last().Id; + if (tweets.Count < batchSize) dbBrowsingEnded = true; + + } while (!dbBrowsingEnded); + + await Task.Delay(TimeSpan.FromHours(3), ct); + } + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs index ccf2181..1af1288 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs @@ -1,7 +1,30 @@ -namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class SaveDeletedTweetStatusProcessor + public class SaveDeletedTweetStatusProcessor : ISaveDeletedTweetStatusProcessor { - + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + + #region Ctor + public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + } + #endregion + + public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct) + { + var highLimitDate = DateTime.UtcNow.AddDays(-40); //TODO get settings value + if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate) + { + await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id); + } + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs b/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs index 9dfc451..10f69c0 100644 --- a/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs +++ b/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs @@ -1,5 +1,9 @@ using System.Threading.Tasks; using System.Threading; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline { @@ -10,9 +14,63 @@ namespace BirdsiteLive.Pipeline public class TweetCleanUpPipeline : ITweetCleanUpPipeline { + private readonly IRetrieveTweetsToDeleteProcessor _retrieveTweetsToDeleteProcessor; + private readonly IDeleteTweetsProcessor _deleteTweetsProcessor; + private readonly ISaveDeletedTweetStatusProcessor _saveDeletedTweetStatusProcessor; + private readonly ILogger _logger; + + #region Ctor + public TweetCleanUpPipeline(IRetrieveTweetsToDeleteProcessor retrieveTweetsToDeleteProcessor, IDeleteTweetsProcessor deleteTweetsProcessor, ISaveDeletedTweetStatusProcessor saveDeletedTweetStatusProcessor, ILogger logger) + { + _retrieveTweetsToDeleteProcessor = retrieveTweetsToDeleteProcessor; + _deleteTweetsProcessor = deleteTweetsProcessor; + _saveDeletedTweetStatusProcessor = saveDeletedTweetStatusProcessor; + _logger = logger; + } + #endregion + public async Task ExecuteAsync(CancellationToken ct) { - throw new System.NotImplementedException(); + // Create blocks + var tweetsToDeleteBufferBlock = new BufferBlock(new DataflowBlockOptions + { + BoundedCapacity = 200, + CancellationToken = ct + }); + var deleteTweetsBlock = new TransformBlock( + async x => await _deleteTweetsProcessor.ProcessAsync(x, ct), + new ExecutionDataflowBlockOptions() + { + MaxDegreeOfParallelism = 5, + CancellationToken = ct + }); + var deletedTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions + { + BoundedCapacity = 200, + CancellationToken = ct + }); + var saveProgressionBlock = new ActionBlock( + async x => await _saveDeletedTweetStatusProcessor.ProcessAsync(x, ct), + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = 5, + CancellationToken = ct + }); + + // Link pipeline + var dataflowLinkOptions = new DataflowLinkOptions { PropagateCompletion = true }; + tweetsToDeleteBufferBlock.LinkTo(deleteTweetsBlock, dataflowLinkOptions); + deleteTweetsBlock.LinkTo(deletedTweetsBufferBlock, dataflowLinkOptions); + deletedTweetsBufferBlock.LinkTo(saveProgressionBlock, dataflowLinkOptions); + + // Launch tweet retriever + var retrieveTweetsToDeleteTask = _retrieveTweetsToDeleteProcessor.GetTweetsAsync(tweetsToDeleteBufferBlock, ct); + + // Wait + await Task.WhenAny(new[] { retrieveTweetsToDeleteTask, saveProgressionBlock.Completion }); + + var ex = retrieveTweetsToDeleteTask.IsFaulted ? retrieveTweetsToDeleteTask.Exception : saveProgressionBlock.Completion.Exception; + _logger.LogCritical(ex, "An error occurred, pipeline stopped"); } } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs index c8879a3..b727a1c 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs @@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Models public long Id { get; set; } public string Acct { get; set; } + public string Host { get; set; } //TODO public string Inbox { get; set; } public long TweetId { get; set; } public DateTime PublishedAt { get; set; } From bb4ef071c2ac45af9bf499e9aa62b11f5f75d78a Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 01:51:50 -0500 Subject: [PATCH 13/27] fix namespaces --- .../Processors/RefreshTwitterUserStatusProcessorTests.cs | 3 +-- .../Processors/RetrieveFollowersProcessorTests.cs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs index cd2d116..9e224f7 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs @@ -7,8 +7,7 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs index 4679259..99b3e29 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; From bc5577808943b764156fe636e6259b8d8df19464 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:05:26 -0500 Subject: [PATCH 14/27] added host to synctweet --- .../DbInitializerPostgresDal.cs | 1 + .../DataAccessLayers/SyncTweetsPostgresDal.cs | 7 ++++- .../SyncTweetsPostgresDalTests.cs | 27 +++++++++++++++---- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index f30dfa7..24ccf6e 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -195,6 +195,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers acct VARCHAR(50) NOT NULL, tweetId BIGINT NOT NULL, inbox VARCHAR(2048) NOT NULL, + host VARCHAR(253) NOT NULL, publishedAt TIMESTAMP (2) WITHOUT TIME ZONE NOT NULL, UNIQUE (acct, tweetId, inbox) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs index c38a141..ff05db4 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs @@ -22,21 +22,26 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers { if (tweet.PublishedAt == default) throw new ArgumentException("publishedAt"); if (tweet.TweetId == default) throw new ArgumentException("tweetId"); + if (string.IsNullOrWhiteSpace(tweet.Acct)) throw new ArgumentException("acct"); + if (string.IsNullOrWhiteSpace(tweet.Inbox)) throw new ArgumentException("inbox"); + if (string.IsNullOrWhiteSpace(tweet.Host)) throw new ArgumentException("host"); var acct = tweet.Acct.ToLowerInvariant().Trim(); var inbox = tweet.Inbox.ToLowerInvariant().Trim(); + var host = tweet.Host.ToLowerInvariant().Trim(); using (var dbConnection = Connection) { dbConnection.Open(); return (await dbConnection.QueryAsync( - $"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,publishedAt) VALUES(@acct,@tweetId,@inbox,@publishedAt) RETURNING id", + $"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,host,publishedAt) VALUES(@acct,@tweetId,@inbox,@host,@publishedAt) RETURNING id", new { acct, tweetId = tweet.TweetId, inbox, + host, publishedAt = tweet.PublishedAt.ToUniversalTime() })).First(); } diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs index d252359..c65cf24 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base; @@ -33,7 +34,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { Acct = "test", PublishedAt = DateTime.UtcNow, - Inbox = "https://instance.ext/inbox", + Inbox = "/inbox", + Host = "instance.ext", TweetId = 4567889 }; @@ -47,6 +49,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsTrue(result.Id > 0); Assert.AreEqual(tweet.Acct, result.Acct); Assert.AreEqual(tweet.Inbox, result.Inbox); + Assert.AreEqual(tweet.Host, result.Host); Assert.AreEqual(tweet.TweetId, result.TweetId); Assert.IsTrue(Math.Abs((tweet.PublishedAt - result.PublishedAt).Seconds) < 5); } @@ -58,7 +61,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { Acct = "test", PublishedAt = DateTime.UtcNow, - Inbox = "https://instance.ext/inbox", + Inbox = "/inbox", + Host = "instance.ext", TweetId = 4567889 }; @@ -77,15 +81,19 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var now = DateTime.UtcNow; var dal = new SyncTweetsPostgresDal(_settings); + var allData = new List(); + for (var i = 0; i < 100; i++) { var tweet = new SyncTweet { Acct = "test", PublishedAt = now.AddDays(-10 - i), - Inbox = "https://instance.ext/inbox", + Inbox = "/inbox", + Host = "instance.ext", TweetId = 4567889 + i }; + allData.Add(tweet); await dal.SaveTweetAsync(tweet); } @@ -100,6 +108,13 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsTrue(res.PublishedAt < date); Assert.IsTrue(res.Id > 10); Assert.IsTrue(res.Id < 25); + + var original = allData.First(x => x.Acct == res.Acct && x.TweetId == res.TweetId); + Assert.AreEqual(original.Acct, res.Acct); + Assert.AreEqual(original.Inbox, res.Inbox); + Assert.AreEqual(original.Host, res.Host); + Assert.AreEqual(original.TweetId, res.TweetId); + Assert.IsTrue(Math.Abs((original.PublishedAt - res.PublishedAt).Seconds) < 5); } } @@ -115,7 +130,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { Acct = "test", PublishedAt = now.AddDays(-10 - i), - Inbox = "https://instance.ext/inbox", + Inbox = "/inbox", + Host = "instance.ext", TweetId = 4567889 + i }; await dal.SaveTweetAsync(tweet); @@ -140,7 +156,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { Acct = "test", PublishedAt = now.AddDays(-10 - i), - Inbox = "https://instance.ext/inbox", + Inbox = "/inbox", + Host = "instance.ext", TweetId = 4567889 + i }; await dal.SaveTweetAsync(tweet); From f1a49d1dd1f6ca224dc703f08d9e9f7a1f92d5f5 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:08:57 -0500 Subject: [PATCH 15/27] updated saving tweets + tests --- .../Federation/SubTasks/SendTweetsTaskBase.cs | 4 ++-- .../SubTasks/SendTweetsToInboxTaskTests.cs | 15 ++++++++++----- .../SubTasks/SendTweetsToSharedInboxTests.cs | 15 ++++++++++----- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs index 58ae052..1cb9be9 100644 --- a/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs @@ -18,13 +18,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox) { - var inboxUrl = $"https://{host}/{inbox.Trim('/')}"; var tweet = new SyncTweet { Acct = acct, TweetId = tweetId, PublishedAt = DateTime.UtcNow, - Inbox = inboxUrl + Inbox = inbox, + Host = host }; await _syncTweetsPostgresDal.SaveTweetAsync(tweet); } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs index 81361ed..76b7a7f 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -94,7 +94,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion @@ -257,7 +258,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion @@ -349,7 +351,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion @@ -444,7 +447,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); } @@ -546,7 +550,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId2 - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs index 5e61379..26db9f6 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs @@ -115,7 +115,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion @@ -321,7 +322,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion @@ -434,7 +436,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion @@ -550,7 +553,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); } @@ -673,7 +677,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks .Setup(x => x.SaveTweetAsync( It.Is(y => y.Acct == twitterHandle && y.TweetId == tweetId2 - && y.Inbox == $"https://{host}{inbox}" + && y.Inbox == inbox + && y.Host == host && y.PublishedAt != default))) .ReturnsAsync(-1); #endregion From be13b6f7c14e993dc3a4930b95c295c0d20daaae Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:20:28 -0500 Subject: [PATCH 16/27] .NET 5 migration for HttpRequestException StatusCode --- .../BirdsiteLive.Pipeline.csproj | 2 +- .../TweetsCleanUp/DeleteTweetsProcessor.cs | 12 +++++++++++- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- src/BirdsiteLive/Controllers/UsersController.cs | 1 - .../BirdsiteLive.Pipeline.Tests.csproj | 3 +-- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index a190af6..6a56215 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -1,7 +1,7 @@  - netstandard2.0 + net5.0 latest diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs index 04d844c..a9841a9 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs @@ -1,4 +1,5 @@ using System; +using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -31,7 +32,16 @@ namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp } catch (HttpRequestException e) { - //TODO check code under .NET 5 + var code = e.StatusCode; + if (code is HttpStatusCode.Gone or HttpStatusCode.NotFound) + { + _logger.LogInformation("Tweet already deleted"); + tweet.DeleteSuccessful = true; + } + else + { + _logger.LogError(e.Message, e); + } } catch (Exception e) { diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 122ff88..45ec28f 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net5.0 d21486de-a812-47eb-a419-05682bb68856 Linux 0.22.0 diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index e66dac5..c636c0e 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Mime; -using System.Runtime.InteropServices.WindowsRuntime; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index d1cfd06..304de46 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -1,8 +1,7 @@ - netcoreapp3.1 - + net5.0 false From a2010c7e3f340e0af514466811b17642f1e0420b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:30:42 -0500 Subject: [PATCH 17/27] migratrion to .NET 6 + update docker --- Dockerfile | 4 ++-- src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj | 2 +- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- .../BirdsiteLive.Pipeline.Tests.csproj | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11a4422..a59f7f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. -FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base +FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base WORKDIR /app EXPOSE 80 EXPOSE 443 -FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish +FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish COPY ./src/ ./src/ RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 6a56215..f3d930b 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -1,7 +1,7 @@  - net5.0 + net6.0 latest diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 45ec28f..eeef98f 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -1,7 +1,7 @@  - net5.0 + net6.0 d21486de-a812-47eb-a419-05682bb68856 Linux 0.22.0 diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index 304de46..a87bc83 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -1,7 +1,7 @@ - + - net5.0 + net6.0 false From 8589c48c9f181282430715bbb5f9f56720729c6f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:33:31 -0500 Subject: [PATCH 18/27] .NET 6 migration --- .github/workflows/dotnet-core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index d28cb08..358ee45 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -16,7 +16,7 @@ jobs: - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: - dotnet-version: 3.1.101 + dotnet-version: 6 - name: Install dependencies run: dotnet restore working-directory: ${{env.working-directory}} From 61730269f2983dd1f613ed9e2f084643a5e7dcc2 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:34:22 -0500 Subject: [PATCH 19/27] Update dotnet-core.yml --- .github/workflows/dotnet-core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index 358ee45..ae430cc 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -16,7 +16,7 @@ jobs: - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: - dotnet-version: 6 + dotnet-version: 6.0.x - name: Install dependencies run: dotnet restore working-directory: ${{env.working-directory}} From c75507926c7c568172af3072a3c0d4dd03fdadd4 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:37:34 -0500 Subject: [PATCH 20/27] Test and Manager .NET 6 migration --- src/BSLManager/BSLManager.csproj | 2 +- src/Tests/BSLManager.Tests/BSLManager.Tests.csproj | 4 ++-- .../BirdsiteLive.ActivityPub.Tests.csproj | 2 +- .../BirdsiteLive.Common.Tests.csproj | 2 +- .../BirdsiteLive.Cryptography.Tests.csproj | 4 ++-- .../BirdsiteLive.DAL.Postgres.Tests.csproj | 4 ++-- .../BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj | 2 +- .../BirdsiteLive.Domain.Tests.csproj | 4 ++-- .../BirdsiteLive.Moderation.Tests.csproj | 4 ++-- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj index 52e5cde..f3e1da6 100644 --- a/src/BSLManager/BSLManager.csproj +++ b/src/BSLManager/BSLManager.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj index 033cfe1..b29432c 100644 --- a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj +++ b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj @@ -1,7 +1,7 @@ - + - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj index 611d29e..85a8350 100644 --- a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj +++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj index 0a52603..cb48684 100644 --- a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj +++ b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj index 0c9c0a6..6419e7c 100644 --- a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj +++ b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj @@ -1,7 +1,7 @@ - + - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj index da05ef2..abed7ed 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj @@ -1,7 +1,7 @@ - + - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj index 0992b02..026af77 100644 --- a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj index 626c941..c199dab 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj +++ b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj @@ -1,7 +1,7 @@ - + - netcoreapp3.1 + net6.0 false diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj index e85b592..3d11047 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj +++ b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj @@ -1,7 +1,7 @@ - + - netcoreapp3.1 + net6.0 false From 405087360c97f24a233a1c5c7ad047262b67f2c0 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 02:52:23 -0500 Subject: [PATCH 21/27] set tweet retention delay in settings --- VARIABLES.md | 1 + .../Settings/InstanceSettings.cs | 2 ++ .../BirdsiteLive.Pipeline.csproj | 2 +- .../TweetsCleanUp/Base/RetentionBase.cs | 16 ++++++++++++++++ .../RetrieveTweetsToDeleteProcessor.cs | 12 +++++++++--- .../SaveDeletedTweetStatusProcessor.cs | 12 +++++++++--- src/BirdsiteLive/appsettings.json | 3 ++- 7 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs diff --git a/VARIABLES.md b/VARIABLES.md index f43bddc..bb8bf61 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. * `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided). * `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy). +* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted # Docker Compose full example diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index a67977c..0ece251 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -18,5 +18,7 @@ public int UserCacheCapacity { get; set; } public string IpWhiteListing { get; set; } public bool EnableXRealIpHeader { get; set; } + + public int MaxTweetRetention { get; set; } } } diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index f3d930b..b72c3cf 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs new file mode 100644 index 0000000..58942ac --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs @@ -0,0 +1,16 @@ +using BirdsiteLive.Common.Settings; +using System; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base +{ + public class RetentionBase + { + protected int GetRetentionTime(InstanceSettings settings) + { + var retentionTime = Math.Abs(settings.MaxTweetRetention); + if (retentionTime < 1) retentionTime = 1; + if (retentionTime > 90) retentionTime = 90; + return retentionTime; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs index 82c6ef7..daf4729 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs @@ -3,20 +3,24 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base; namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class RetrieveTweetsToDeleteProcessor : IRetrieveTweetsToDeleteProcessor + public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor { private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + private readonly InstanceSettings _instanceSettings; #region Ctor - public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal) + public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings) { _syncTweetsPostgresDal = syncTweetsPostgresDal; + _instanceSettings = instanceSettings; } #endregion @@ -29,7 +33,9 @@ namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp ct.ThrowIfCancellationRequested(); var now = DateTime.UtcNow; - var from = now.AddDays(-20); + + + var from = now.AddDays(-GetRetentionTime(_instanceSettings)); var dbBrowsingEnded = false; var lastId = -1L; diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs index 1af1288..817d0be 100644 --- a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs @@ -1,26 +1,32 @@ using System; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base; namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp { - public class SaveDeletedTweetStatusProcessor : ISaveDeletedTweetStatusProcessor + public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor { private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + private readonly InstanceSettings _instanceSettings; #region Ctor - public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal) + public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings) { _syncTweetsPostgresDal = syncTweetsPostgresDal; + _instanceSettings = instanceSettings; } #endregion public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct) { - var highLimitDate = DateTime.UtcNow.AddDays(-40); //TODO get settings value + var retentionTime = GetRetentionTime(_instanceSettings); + retentionTime += 20; // Delay until last retry + var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime); if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate) { await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id); diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 4cee8ba..9628532 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -25,7 +25,8 @@ "SensitiveTwitterAccounts": null, "FailingTwitterUserCleanUpThreshold": 700, "FailingFollowerCleanUpThreshold": 30000, - "UserCacheCapacity": 10000 + "UserCacheCapacity": 10000, + "MaxTweetRetention": 20 }, "Db": { "Type": "postgres", From e63c7950c9cb510beea6198d4a669bf04e6231db Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 03:07:23 -0500 Subject: [PATCH 22/27] bump nugets --- src/BSLManager/BSLManager.csproj | 8 ++++---- .../BirdsiteLive.ActivityPub.csproj | 4 ++-- .../BirdsiteLive.Cryptography.csproj | 2 +- .../BirdsiteLive.Domain.csproj | 2 +- .../BirdsiteLive.Pipeline.csproj | 6 +++--- .../BirdsiteLive.Twitter.csproj | 2 +- src/BirdsiteLive/BirdsiteLive.csproj | 8 ++++---- .../BirdsiteLive.DAL.Postgres.csproj | 2 +- .../BSLManager.Tests/BSLManager.Tests.csproj | 11 +++++++---- .../BirdsiteLive.ActivityPub.Tests.csproj | 11 +++++++---- .../BirdsiteLive.Common.Tests.csproj | 13 ++++++++----- .../BirdsiteLive.Cryptography.Tests.csproj | 11 +++++++---- .../BirdsiteLive.DAL.Postgres.Tests.csproj | 11 +++++++---- .../BirdsiteLive.DAL.Tests.csproj | 15 +++++++++------ .../BirdsiteLive.Domain.Tests.csproj | 13 ++++++++----- .../BirdsiteLive.Moderation.Tests.csproj | 13 ++++++++----- .../BirdsiteLive.Pipeline.Tests.csproj | 13 ++++++++----- 17 files changed, 86 insertions(+), 59 deletions(-) diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj index f3e1da6..ee4f323 100644 --- a/src/BSLManager/BSLManager.csproj +++ b/src/BSLManager/BSLManager.csproj @@ -6,10 +6,10 @@ - - - - + + + + diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj index a690b63..f914c67 100644 --- a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj +++ b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj index f0d9f4f..53ee6d9 100644 --- a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj +++ b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index 7bc9873..57219d8 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index b72c3cf..bf20e96 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -6,9 +6,9 @@ - - - + + + diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj index 438b4e1..49ddda9 100644 --- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj +++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index eeef98f..c0aca59 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -8,10 +8,10 @@ - - - - + + + + diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj index 690c878..a164405 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj index b29432c..7f9a210 100644 --- a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj +++ b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj @@ -7,10 +7,13 @@ - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj index 85a8350..05bd857 100644 --- a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj +++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj @@ -7,10 +7,13 @@ - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj index cb48684..59ee199 100644 --- a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj +++ b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -7,10 +7,13 @@ - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj index 6419e7c..ed1ed50 100644 --- a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj +++ b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj @@ -7,10 +7,13 @@ - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj index abed7ed..ae70505 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj @@ -7,10 +7,13 @@ - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj index 026af77..8bc759d 100644 --- a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -7,11 +7,14 @@ - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj index c199dab..852e4b5 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj +++ b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj @@ -7,11 +7,14 @@ - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj index 3d11047..9b86065 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj +++ b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj @@ -7,11 +7,14 @@ - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index a87bc83..e3aae19 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -6,11 +6,14 @@ - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + From ef9ee3230c1075c7ce35a3bc8d1afa6355faddf6 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 6 Jan 2023 03:11:28 -0500 Subject: [PATCH 23/27] bump nugets --- src/BSLManager/BSLManager.csproj | 2 +- src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj | 2 +- src/BirdsiteLive/Controllers/MigrationController.cs | 1 - .../BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj index ee4f323..1a725e9 100644 --- a/src/BSLManager/BSLManager.csproj +++ b/src/BSLManager/BSLManager.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj index 53ee6d9..4fc9f61 100644 --- a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj +++ b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index f2cde09..8d4b816 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -3,7 +3,6 @@ using Microsoft.AspNetCore.Mvc; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; -using Npgsql.TypeHandlers; using BirdsiteLive.Domain; using BirdsiteLive.Domain.Enum; using BirdsiteLive.DAL.Contracts; diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj index a164405..b72a179 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj @@ -6,7 +6,7 @@ - + From 57a08b67b1078eaa801c794ebd904976387a2d60 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 11 Jan 2023 01:13:23 -0500 Subject: [PATCH 24/27] better Delete Actor logic --- .../BusinessUseCases/ProcessDeleteUser.cs | 10 ++++++++++ src/BirdsiteLive.Domain/UserService.cs | 20 ++++++++++++------- .../Controllers/InboxController.cs | 5 +++-- .../DataAccessLayers/FollowersPostgresDal.cs | 15 ++++++++++++++ .../Contracts/IFollowersDal.cs | 1 + 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs index a35b6c8..a36c963 100644 --- a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs @@ -9,6 +9,7 @@ namespace BirdsiteLive.Domain.BusinessUseCases { Task ExecuteAsync(Follower follower); Task ExecuteAsync(string followerUsername, string followerDomain); + Task ExecuteAsync(string actorId); } public class ProcessDeleteUser : IProcessDeleteUser @@ -33,6 +34,15 @@ namespace BirdsiteLive.Domain.BusinessUseCases await ExecuteAsync(follower); } + public async Task ExecuteAsync(string actorId) + { + // Get Follower and Twitter Users + var follower = await _followersDal.GetFollowerAsync(actorId); + if (follower == null) return; + + await ExecuteAsync(follower); + } + public async Task ExecuteAsync(Follower follower) { // Remove twitter users if no more followers diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 6f88543..ed31c62 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -280,15 +280,21 @@ namespace BirdsiteLive.Domain public async Task DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityDelete activity, string body) { - // Validate - var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); - if (!sigValidation.SignatureIsValidated) return false; + if (activity.apObject is string apObject) + { + if (!string.Equals(activity.actor.Trim(), apObject.Trim(), StringComparison.InvariantCultureIgnoreCase)) return true; - // Remove user and followings - var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation); - var followerHost = SigValidationResultExtractor.GetHost(sigValidation); + try + { + // Validate + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); + if (!sigValidation.SignatureIsValidated) return false; + } + catch (FollowerIsGoneException){} - await _processDeleteUser.ExecuteAsync(followerUserName, followerHost); + // Remove user and followings + await _processDeleteUser.ExecuteAsync(activity.actor.Trim()); + } return true; } diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index f92a0a6..57825af 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -49,14 +49,15 @@ namespace BirdsiteLive.Controllers case "Delete": { var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityDelete, body); if (succeeded) return Accepted(); else return Unauthorized(); } } } } - catch (FollowerIsGoneException) { } //TODO: check if user in DB + catch (FollowerIsGoneException) { } return Accepted(); } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs index db2f9f7..c93ad5e 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs @@ -82,6 +82,21 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } + public async Task GetFollowerAsync(string actorId) + { + var query = $"SELECT * FROM {_settings.FollowersTableName} WHERE actorid = @actorid"; + + actorId = actorId.ToLowerInvariant().Trim(); + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = (await dbConnection.QueryAsync(query, new { actorId })).FirstOrDefault(); + return Convert(result); + } + } + public async Task GetFollowersAsync(int followedUserId) { if (followedUserId == default) throw new ArgumentException("followedUserId"); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs index fe87b28..cb97b8e 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs @@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Contracts public interface IFollowersDal { Task GetFollowerAsync(string acct, string host); + Task GetFollowerAsync(string actorId); Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null, Dictionary followingSyncStatus = null); Task GetFollowersAsync(int followedUserId); From 8a99f5ecceb14d8d52804ff5070f7f9f3283d08b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 12 Mar 2023 03:45:55 -0400 Subject: [PATCH 25/27] adding tests for new pipeline --- .../BirdsiteLive.Pipeline.Tests.csproj | 1 + .../RefreshTwitterUserStatusProcessorTests.cs | 22 +-- .../RetrieveFollowersProcessorTests.cs | 2 +- .../RetrieveTweetsProcessorTests.cs | 0 .../RetrieveTwitterUsersProcessorTests.cs | 14 +- .../SaveProgressionProcessorTests.cs | 0 .../SendTweetsToFollowersProcessorTests.cs | 0 .../SubTasks/SendTweetsToInboxTaskTests.cs | 0 .../SubTasks/SendTweetsToSharedInboxTests.cs | 0 .../DeleteTweetsProcessorTests.cs | 160 ++++++++++++++++++ .../RetrieveTweetsToDeleteProcessorTests.cs | 76 +++++++++ .../SaveDeletedTweetStatusProcessorTests.cs | 6 + .../TweetCleanUpPipelineTests.cs | 6 + 13 files changed, 268 insertions(+), 19 deletions(-) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/RefreshTwitterUserStatusProcessorTests.cs (99%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/RetrieveFollowersProcessorTests.cs (97%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/RetrieveTweetsProcessorTests.cs (100%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/RetrieveTwitterUsersProcessorTests.cs (98%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/SaveProgressionProcessorTests.cs (100%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/SendTweetsToFollowersProcessorTests.cs (100%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/SubTasks/SendTweetsToInboxTaskTests.cs (100%) rename src/Tests/BirdsiteLive.Pipeline.Tests/Processors/{ => Federation}/SubTasks/SendTweetsToSharedInboxTests.cs (100%) create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index e3aae19..7d8e345 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RefreshTwitterUserStatusProcessorTests.cs similarity index 99% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RefreshTwitterUserStatusProcessorTests.cs index 9e224f7..a1c9757 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RefreshTwitterUserStatusProcessorTests.cs @@ -13,7 +13,7 @@ using BirdsiteLive.Twitter.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -namespace BirdsiteLive.Pipeline.Tests.Processors +namespace BirdsiteLive.Pipeline.Tests.Processors.Federation { [TestClass] public class RefreshTwitterUserStatusProcessorTests @@ -60,7 +60,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); #region Validations - Assert.AreEqual(2 , result.Length); + Assert.AreEqual(2, result.Length); Assert.IsTrue(result.Any(x => x.User.Id == userId1)); Assert.IsTrue(result.Any(x => x.User.Id == userId2)); @@ -288,7 +288,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Throws(new Exception()); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -360,7 +360,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -432,7 +432,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -506,7 +506,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { Protected = true }); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -581,7 +581,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { Protected = true }); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -609,14 +609,14 @@ namespace BirdsiteLive.Pipeline.Tests.Processors removeTwitterAccountActionMock.VerifyAll(); #endregion } - + [TestMethod] public async Task ProcessAsync_Error_NotInit_Test() { #region Stubs var userId1 = 1; var acct1 = "user1"; - + var users = new List { new SyncTwitterUser @@ -640,7 +640,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct1))) .Returns((TwitterUser)null); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct1))) @@ -708,7 +708,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Throws(new RateLimitExceededException()); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveFollowersProcessorTests.cs similarity index 97% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveFollowersProcessorTests.cs index 99b3e29..d687541 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveFollowersProcessorTests.cs @@ -9,7 +9,7 @@ using BirdsiteLive.Pipeline.Processors.Federation; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -namespace BirdsiteLive.Pipeline.Tests.Processors +namespace BirdsiteLive.Pipeline.Tests.Processors.Federation { [TestClass] public class RetrieveFollowersProcessorTests diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTweetsProcessorTests.cs similarity index 100% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTweetsProcessorTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTwitterUsersProcessorTests.cs similarity index 98% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTwitterUsersProcessorTests.cs index 8a5a5ba..f8956d3 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTwitterUsersProcessorTests.cs @@ -12,7 +12,7 @@ using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -namespace BirdsiteLive.Pipeline.Tests.Processors +namespace BirdsiteLive.Pipeline.Tests.Processors.Federation { [TestClass] public class RetrieveTwitterUsersProcessorTests @@ -43,7 +43,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == maxUsers), It.Is(y => y == false))) .ReturnsAsync(users); - + var loggerMock = new Mock>(); #endregion @@ -100,7 +100,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(300)); - + #region Validations maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); @@ -139,7 +139,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); - + var loggerMock = new Mock>(); #endregion @@ -190,7 +190,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; - var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); @@ -200,7 +200,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors Assert.AreEqual(0, buffer.Count); #endregion } - + [TestMethod] public async Task GetTwitterUsersAsync_Exception_Test() { @@ -258,7 +258,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .ReturnsAsync(maxUsers); var twitterUserDalMock = new Mock(MockBehavior.Strict); - + var loggerMock = new Mock>(); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SaveProgressionProcessorTests.cs similarity index 100% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SaveProgressionProcessorTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SendTweetsToFollowersProcessorTests.cs similarity index 100% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SendTweetsToFollowersProcessorTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToInboxTaskTests.cs similarity index 100% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToInboxTaskTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToSharedInboxTests.cs similarity index 100% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToSharedInboxTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs new file mode 100644 index 0000000..d81c581 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs @@ -0,0 +1,160 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.Federation; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp +{ + [TestClass] + public class DeleteTweetsProcessorTests + { + [TestMethod] + public async Task Process_DeleteTweet_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsTrue(result.DeleteSuccessful); + #endregion + } + + [TestMethod] + public async Task Process_DeleteTweet_NotFound_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Throws(new HttpRequestException("not found", null, HttpStatusCode.NotFound)); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsTrue(result.DeleteSuccessful); + #endregion + } + + [TestMethod] + public async Task Process_DeleteTweet_HttpError_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Throws(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsFalse(result.DeleteSuccessful); + #endregion + } + + [TestMethod] + public async Task Process_DeleteTweet_GenericException_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Throws(new Exception()); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsFalse(result.DeleteSuccessful); + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs new file mode 100644 index 0000000..b9f66d5 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using System.Xml.Linq; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Org.BouncyCastle.Crypto; + +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp +{ + [TestClass] + public class RetrieveTweetsToDeleteProcessorTests + { + [TestMethod] + public async Task Process_Test() + { + #region Stubs + var settings = new InstanceSettings + { + + }; + var bufferBlock = new BufferBlock(); + + var tweetId1 = 42; + var tweetId2 = 43; + + var tweet1 = new SyncTweet + { + Id = tweetId1 + }; + var tweet2 = new SyncTweet + { + Id = tweetId2 + }; + var batch = new List + { + tweet1, + tweet2 + }; + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.GetTweetsOlderThanAsync(It.IsAny(), It.Is(y => y == -1), It.Is(y => y == 100))) + .ReturnsAsync(batch); + #endregion + + var processor = new RetrieveTweetsToDeleteProcessor(dalMock.Object, settings); + processor.GetTweetsAsync(bufferBlock, CancellationToken.None); + + await Task.Delay(TimeSpan.FromSeconds(1)); + + #region Validations + dalMock.VerifyAll(); + + Assert.AreEqual(batch.Count, bufferBlock.Count); + + bufferBlock.TryReceiveAll(out var all); + foreach (var tweet in all) + { + Assert.IsTrue(batch.Any(x => x.Id == tweet.Tweet.Id)); + } + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs new file mode 100644 index 0000000..10d21c7 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs @@ -0,0 +1,6 @@ +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp; + +public class SaveDeletedTweetStatusProcessorTests +{ + +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs new file mode 100644 index 0000000..eaaef54 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs @@ -0,0 +1,6 @@ +namespace BirdsiteLive.Pipeline.Tests; + +public class TweetCleanUpPipelineTests +{ + +} \ No newline at end of file From 5b93c2b7607e147a16c0bfb2d8cd838ee89c5152 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 13 Mar 2023 23:50:05 -0400 Subject: [PATCH 26/27] added tests --- .../RetrieveTweetsToDeleteProcessorTests.cs | 55 +++++++++++++ .../SaveDeletedTweetStatusProcessorTests.cs | 80 ++++++++++++++++++- .../TweetCleanUpPipelineTests.cs | 48 ++++++++++- 3 files changed, 177 insertions(+), 6 deletions(-) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs index b9f66d5..7d7f7d1 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs @@ -72,5 +72,60 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp } #endregion } + + [TestMethod] + public async Task Process_Pagination_Test() + { + #region Stubs + var settings = new InstanceSettings { }; + var bufferBlock = new BufferBlock(); + + + var batch = new List(); + var batch2 = new List(); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Id = i + }; + batch.Add(tweet); + } + + for (var i = 100; i < 120; i++) + { + var tweet = new SyncTweet + { + Id = i + }; + batch2.Add(tweet); + } + + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.GetTweetsOlderThanAsync(It.IsAny(), It.Is(y => y == -1), It.Is(y => y == 100))) + .ReturnsAsync(batch); + + dalMock + .Setup(x => x.GetTweetsOlderThanAsync(It.IsAny(), It.Is(y => y == 99), It.Is(y => y == 100))) + .ReturnsAsync(batch2); + #endregion + + var processor = new RetrieveTweetsToDeleteProcessor(dalMock.Object, settings); + processor.GetTweetsAsync(bufferBlock, CancellationToken.None); + + await Task.Delay(TimeSpan.FromSeconds(1)); + + #region Validations + dalMock.VerifyAll(); + + Assert.AreEqual(batch.Count + batch2.Count, bufferBlock.Count); + #endregion + } + } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs index 10d21c7..ea002cb 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs @@ -1,6 +1,80 @@ -namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp; +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; -public class SaveDeletedTweetStatusProcessorTests +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp { - + [TestClass] + public class SaveDeletedTweetStatusProcessorTests + { + [TestMethod] + public async Task Process_DeleteSuccessfull_Test() + { + #region Stubs + var settings = new InstanceSettings { }; + var tweetId = 42; + var tweetToDelete = new TweetToDelete + { + DeleteSuccessful = true, + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.DeleteTweetAsync(It.Is(y => y == tweetId))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveDeletedTweetStatusProcessor(dalMock.Object, settings); + await processor.ProcessAsync(tweetToDelete, CancellationToken.None); + + #region Validations + dalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task Process_Expired_Test() + { + #region Stubs + var settings = new InstanceSettings { }; + var tweetId = 42; + var tweetToDelete = new TweetToDelete + { + DeleteSuccessful = false, + Tweet = new SyncTweet + { + Id = tweetId, + PublishedAt = DateTime.UtcNow.AddDays(-30) + } + }; + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.DeleteTweetAsync(It.Is(y => y == tweetId))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveDeletedTweetStatusProcessor(dalMock.Object, settings); + await processor.ProcessAsync(tweetToDelete, CancellationToken.None); + + #region Validations + dalMock.VerifyAll(); + #endregion + } + } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs index eaaef54..a7f2751 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs @@ -1,6 +1,48 @@ -namespace BirdsiteLive.Pipeline.Tests; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Contracts.Federation; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; -public class TweetCleanUpPipelineTests +namespace BirdsiteLive.Pipeline.Tests { - + [TestClass] + public class TweetCleanUpPipelineTests + { + [TestMethod] + public async Task Process() + { + #region Stubs + var ct = new CancellationTokenSource(10); + #endregion + + #region Mocks + var retrieveTweetsToDeleteProcessor = new Mock(MockBehavior.Strict); + retrieveTweetsToDeleteProcessor + .Setup(x => x.GetTweetsAsync( + It.IsAny>(), + It.IsAny())) + .Returns(Task.Delay(0)); + + var deleteTweetsProcessor = new Mock(MockBehavior.Strict); + var saveDeletedTweetStatusProcessor = new Mock(MockBehavior.Strict); + var logger = new Mock>(); + + #endregion + + var pipeline = new TweetCleanUpPipeline(retrieveTweetsToDeleteProcessor.Object, deleteTweetsProcessor.Object, saveDeletedTweetStatusProcessor.Object, logger.Object); + await pipeline.ExecuteAsync(ct.Token); + + #region Validations + retrieveTweetsToDeleteProcessor.VerifyAll(); + deleteTweetsProcessor.VerifyAll(); + saveDeletedTweetStatusProcessor.VerifyAll(); + #endregion + } + } } \ No newline at end of file From 899a595e8c770d2a5b048479a12e66fef04b1ad9 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 13 Mar 2023 23:56:34 -0400 Subject: [PATCH 27/27] road to 0.23.0 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index c0aca59..acdd0be 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ net6.0 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.22.0 + 0.23.0