octo
/
BirdsiteLive
mirror of https://github.com/NicolasConstant/BirdsiteLive
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

don't retrieve deleted users

This commit is contained in:
Nicolas Constant 2022-12-24 18:44:41 -05:00
parent d219c59cfe
commit 1c3da007fd
No known key found for this signature in database
GPG Key ID: 1E9F677FB01A5688
8 changed files with 93 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/BirdsiteLive.Domain/MigrationService.cs
View File

@ -117,6 +117,7 @@ namespace BirdsiteLive.Domain
twitterAccount.MovedTo = validatedUser.ObjectId; twitterAccount.MovedTo = validatedUser.ObjectId;
twitterAccount.MovedToAcct = validatedUser.FediverseAcct; twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers // Notify Followers
@ -180,6 +181,7 @@ namespace BirdsiteLive.Domain
} }
twitterAccount.Deleted = true; twitterAccount.Deleted = true;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers // Notify Followers

2
src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs
View File

@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors
{ {
if (type == ModerationTypeEnum.None) return; if (type == ModerationTypeEnum.None) return;
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(); var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
foreach (var user in twitterUsers) foreach (var user in twitterUsers)
{ {

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View File

@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors
try try
{ {
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
var userCount = users.Any() ? users.Length : 1; var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d); var splitNumber = (int) Math.Ceiling(userCount / 15d);

14
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View File

@ -69,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task<int> GetTwitterUsersCountAsync() public async Task<int> GetTwitterUsersCountAsync()
{ {
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
using (var dbConnection = Connection) using (var dbConnection = Connection)
{ {
@ -82,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task<int> GetFailingTwitterUsersCountAsync() public async Task<int> GetFailingTwitterUsersCountAsync()
{ {
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE"; var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
using (var dbConnection = Connection) using (var dbConnection = Connection)
{ {
@ -93,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
} }
} }
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber) public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser)
{ {
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
using (var dbConnection = Connection) using (var dbConnection = Connection)
{ {
@ -106,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
} }
} }
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync() public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser)
{ {
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}";
using (var dbConnection = Connection) using (var dbConnection = Connection)
{ {

4
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View File

@ -10,8 +10,8 @@ namespace BirdsiteLive.DAL.Contracts
string movedToAcct = null); string movedToAcct = null);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct); Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser> GetTwitterUserAsync(int id); Task<SyncTwitterUser> GetTwitterUserAsync(int id);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber); Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(); Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted);
Task UpdateTwitterUserAsync(SyncTwitterUser user); Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(string acct);

71
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View File

@ -348,12 +348,71 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
} }
var result = await dal.GetAllTwitterUsersAsync(1100); for (int i = 0; i < 10; i++)
{
var acct = $"deleted-myid{i}";
var lastTweetId = 148L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var user = await dal.GetTwitterUserAsync(acct);
user.Deleted = true;
user.LastSync = DateTime.UtcNow;
await dal.UpdateTwitterUserAsync(user);
}
var result = await dal.GetAllTwitterUsersAsync(1100, false);
Assert.AreEqual(1000, result.Length); Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default); Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
foreach (var user in result)
{
Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo));
Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct));
Assert.IsFalse(user.Deleted);
}
}
[TestMethod]
public async Task GetAllTwitterUsers_Top_RetrieveDeleted()
{
var dal = new TwitterUserPostgresDal(_settings);
for (var i = 0; i < 1000; i++)
{
var acct = $"myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
for (int i = 0; i < 10; i++)
{
var acct = $"migrated-myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
for (int i = 0; i < 10; i++)
{
var acct = $"deleted-myid{i}";
var lastTweetId = 148L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var user = await dal.GetTwitterUserAsync(acct);
user.Deleted = true;
user.LastSync = DateTime.UtcNow;
await dal.UpdateTwitterUserAsync(user);
}
var result = await dal.GetAllTwitterUsersAsync(1100, true);
Assert.AreEqual(1020, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
} }
[TestMethod] [TestMethod]
@ -371,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
// Update accounts // Update accounts
var now = DateTime.UtcNow; var now = DateTime.UtcNow;
var allUsers = await dal.GetAllTwitterUsersAsync(); var allUsers = await dal.GetAllTwitterUsersAsync(false);
foreach (var acc in allUsers) foreach (var acc in allUsers)
{ {
var lastSync = now.AddDays(acc.LastTweetPostedId); var lastSync = now.AddDays(acc.LastTweetPostedId);
@ -382,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
// Create a not init account // Create a not init account
await dal.CreateTwitterUserAsync("not_init", -1); await dal.CreateTwitterUserAsync("not_init", -1);
var result = await dal.GetAllTwitterUsersAsync(10); var result = await dal.GetAllTwitterUsersAsync(10, false);
Assert.IsTrue(result.Any(x => x.Acct == "myid0")); Assert.IsTrue(result.Any(x => x.Acct == "myid0"));
Assert.IsTrue(result.Any(x => x.Acct == "myid8")); Assert.IsTrue(result.Any(x => x.Acct == "myid8"));
@ -405,7 +464,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId); await dal.CreateTwitterUserAsync(acct, lastTweetId);
} }
var allUsers = await dal.GetAllTwitterUsersAsync(100); var allUsers = await dal.GetAllTwitterUsersAsync(100, false);
for (var i = 0; i < 20; i++) for (var i = 0; i < 20; i++)
{ {
var user = allUsers[i]; var user = allUsers[i];
@ -413,7 +472,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false); await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false);
} }
var result = await dal.GetAllTwitterUsersAsync(10); var result = await dal.GetAllTwitterUsersAsync(10, false);
Assert.AreEqual(10, result.Length); Assert.AreEqual(10, result.Length);
Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].Acct == default);
@ -444,7 +503,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
} }
var result = await dal.GetAllTwitterUsersAsync(); var result = await dal.GetAllTwitterUsersAsync(false);
Assert.AreEqual(1000, result.Length); Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].Acct == default);

8
src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs
View File

@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray()); .ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict); var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray()); .ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict); var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray()); .ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict); var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray()); .ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict); var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);

15
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View File

@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync( .Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users); .ReturnsAsync(users);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync( .SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users.ToArray()) .ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0])
@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync( .SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users.ToArray()) .ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0])
@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync( .Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(new SyncTwitterUser[0]); .ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync( .Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception())); .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();