Compare commits
32 Commits
8b1a61c197
...
cf0797aed8
Author | SHA1 | Date |
---|---|---|
Nicolas Constant | cf0797aed8 | |
Nicolas Constant | 899a595e8c | |
Nicolas Constant | 03a6c53e23 | |
Nicolas Constant | 5b93c2b760 | |
Nicolas Constant | 8a99f5ecce | |
Nicolas Constant | 57a08b67b1 | |
Nicolas Constant | ef9ee3230c | |
Nicolas Constant | e63c7950c9 | |
Nicolas Constant | 405087360c | |
Nicolas Constant | 80ac1363e5 | |
Nicolas Constant | c75507926c | |
Nicolas Constant | 61730269f2 | |
Nicolas Constant | 8589c48c9f | |
Nicolas Constant | a2010c7e3f | |
Nicolas Constant | be13b6f7c1 | |
Nicolas Constant | f1a49d1dd1 | |
Nicolas Constant | bc55778089 | |
Nicolas Constant | bb4ef071c2 | |
Nicolas Constant | e579e1b11c | |
Nicolas Constant | b223bb0216 | |
Nicolas Constant | 5b32a9021f | |
Nicolas Constant | 3460c72895 | |
Nicolas Constant | 99f4c65707 | |
Nicolas Constant | d80a00136d | |
Nicolas Constant | 584266a040 | |
Nicolas Constant | 432484eaaa | |
Nicolas Constant | f8ab522505 | |
Nicolas Constant | 8f42032512 | |
Nicolas Constant | 2542258ce4 | |
Nicolas Constant | 8bbbe037c4 | |
Nicolas Constant | 0c2acc3d7a | |
Nicolas Constant | 79ceab82b6 |
|
@ -16,7 +16,7 @@ jobs:
|
|||
- name: Setup .NET Core
|
||||
uses: actions/setup-dotnet@v1
|
||||
with:
|
||||
dotnet-version: 3.1.101
|
||||
dotnet-version: 6.0.x
|
||||
- name: Install dependencies
|
||||
run: dotnet restore
|
||||
working-directory: ${{env.working-directory}}
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base
|
||||
WORKDIR /app
|
||||
EXPOSE 80
|
||||
EXPOSE 443
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
|
||||
FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish
|
||||
COPY ./src/ ./src/
|
||||
RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
|
||||
RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
|
||||
|
|
|
@ -186,6 +186,70 @@ services:
|
|||
+ command: --interval 300
|
||||
```
|
||||
|
||||
## IP Whitelisting
|
||||
|
||||
If you want to use the IP Whitelisting functionality (see related [variable](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)) and you are using the nginx reverse proxy set as before, please add the following:
|
||||
|
||||
```
|
||||
sudo nano /etc/nginx/sites-enabled/{your-domain-name.com}
|
||||
```
|
||||
|
||||
``` diff
|
||||
server {
|
||||
listen 80;
|
||||
server_name {your-domain-name.com};
|
||||
location / {
|
||||
proxy_pass http://localhost:5000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection keep-alive;
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
+ proxy_set_header X-Real-IP $remote_addr;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And edit the docker-compose file as follow:
|
||||
|
||||
```diff
|
||||
version: "3"
|
||||
|
||||
networks:
|
||||
birdsitelivenetwork:
|
||||
external: false
|
||||
|
||||
services:
|
||||
server:
|
||||
image: nicolasconstant/birdsitelive:latest
|
||||
restart: always
|
||||
container_name: birdsitelive
|
||||
environment:
|
||||
- Instance:Domain=domain.name
|
||||
- Instance:AdminEmail=name@domain.ext
|
||||
+ - Instance:IpWhiteListing=127.0.0.1;127.0.0.2
|
||||
+ - Instance:EnableXRealIpHeader=true
|
||||
- Db:Type=postgres
|
||||
- Db:Host=db
|
||||
- Db:Name=birdsitelive
|
||||
- Db:User=birdsitelive
|
||||
- Db:Password=birdsitelive
|
||||
- Twitter:ConsumerKey=twitter.api.key
|
||||
- Twitter:ConsumerSecret=twitter.api.key
|
||||
networks:
|
||||
- birdsitelivenetwork
|
||||
ports:
|
||||
- "5000:80"
|
||||
depends_on:
|
||||
- db
|
||||
|
||||
db:
|
||||
image: postgres:9.6
|
||||
[...]
|
||||
```
|
||||
|
||||
## More options
|
||||
|
||||
You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)
|
||||
|
|
|
@ -51,6 +51,9 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
|
|||
* `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
|
||||
* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
|
||||
* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
|
||||
* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
|
||||
* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
|
||||
* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
|
||||
|
||||
# Docker Compose full example
|
||||
|
||||
|
|
|
@ -2,15 +2,15 @@
|
|||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Lamar" Version="5.0.3" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
|
||||
<PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
|
||||
<PackageReference Include="Lamar" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
|
||||
<PackageReference Include="Terminal.Gui" Version="1.9.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="System.Text.Json" Version="4.7.2" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
|
||||
<PackageReference Include="System.Text.Json" Version="7.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
namespace BirdsiteLive.ActivityPub.Models
|
||||
{
|
||||
public class Tombstone
|
||||
{
|
||||
public string id { get; set; }
|
||||
public readonly string type = "Tombstone";
|
||||
public string atomUrl { get; set; }
|
||||
}
|
||||
}
|
|
@ -16,5 +16,9 @@
|
|||
public int FailingFollowerCleanUpThreshold { get; set; } = -1;
|
||||
|
||||
public int UserCacheCapacity { get; set; }
|
||||
public string IpWhiteListing { get; set; }
|
||||
public bool EnableXRealIpHeader { get; set; }
|
||||
|
||||
public int MaxTweetRetention { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Asn1" Version="1.0.9" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
|
||||
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -9,6 +9,7 @@ using BirdsiteLive.ActivityPub;
|
|||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
|
@ -22,6 +23,7 @@ namespace BirdsiteLive.Domain
|
|||
Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
|
||||
string targetInbox);
|
||||
Task DeleteUserAsync(string username, string targetHost, string targetInbox);
|
||||
Task DeleteNoteAsync(SyncTweet tweet);
|
||||
}
|
||||
|
||||
public class WebFinger
|
||||
|
@ -108,6 +110,30 @@ namespace BirdsiteLive.Domain
|
|||
}
|
||||
}
|
||||
|
||||
public async Task DeleteNoteAsync(SyncTweet tweet)
|
||||
{
|
||||
var acct = tweet.Acct.ToLowerInvariant().Trim();
|
||||
|
||||
var actor = $"https://{_instanceSettings.Domain}/users/{acct}";
|
||||
var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}";
|
||||
|
||||
var delete = new ActivityDelete
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{noteId}#delete",
|
||||
type = "Delete",
|
||||
actor = actor,
|
||||
to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
apObject = new Tombstone
|
||||
{
|
||||
id = noteId,
|
||||
atomUrl = noteId
|
||||
}
|
||||
};
|
||||
|
||||
await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox);
|
||||
}
|
||||
|
||||
public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
|
||||
{
|
||||
try
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -9,6 +9,7 @@ namespace BirdsiteLive.Domain.BusinessUseCases
|
|||
{
|
||||
Task ExecuteAsync(Follower follower);
|
||||
Task ExecuteAsync(string followerUsername, string followerDomain);
|
||||
Task ExecuteAsync(string actorId);
|
||||
}
|
||||
|
||||
public class ProcessDeleteUser : IProcessDeleteUser
|
||||
|
@ -33,6 +34,15 @@ namespace BirdsiteLive.Domain.BusinessUseCases
|
|||
await ExecuteAsync(follower);
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(string actorId)
|
||||
{
|
||||
// Get Follower and Twitter Users
|
||||
var follower = await _followersDal.GetFollowerAsync(actorId);
|
||||
if (follower == null) return;
|
||||
|
||||
await ExecuteAsync(follower);
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Follower follower)
|
||||
{
|
||||
// Remove twitter users if no more followers
|
||||
|
|
|
@ -280,15 +280,21 @@ namespace BirdsiteLive.Domain
|
|||
public async Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders,
|
||||
ActivityDelete activity, string body)
|
||||
{
|
||||
// Validate
|
||||
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
|
||||
if (!sigValidation.SignatureIsValidated) return false;
|
||||
if (activity.apObject is string apObject)
|
||||
{
|
||||
if (!string.Equals(activity.actor.Trim(), apObject.Trim(), StringComparison.InvariantCultureIgnoreCase)) return true;
|
||||
|
||||
// Remove user and followings
|
||||
var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
|
||||
var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
|
||||
try
|
||||
{
|
||||
// Validate
|
||||
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
|
||||
if (!sigValidation.SignatureIsValidated) return false;
|
||||
}
|
||||
catch (FollowerIsGoneException){}
|
||||
|
||||
await _processDeleteUser.ExecuteAsync(followerUserName, followerHost);
|
||||
// Remove user and followings
|
||||
await _processDeleteUser.ExecuteAsync(activity.actor.Trim());
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
|
@ -1,14 +1,14 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<LangVersion>latest</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
|
||||
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
|
||||
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
@ -19,6 +19,7 @@
|
|||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Processors\TweetsCleanUp\Base\" />
|
||||
<Folder Include="Tools\" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@ using System.Threading.Tasks;
|
|||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRefreshTwitterUserStatusProcessor
|
||||
{
|
|
@ -3,7 +3,7 @@ using System.Threading;
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRetrieveFollowersProcessor
|
||||
{
|
|
@ -3,7 +3,7 @@ using System.Threading.Tasks;
|
|||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRetrieveTweetsProcessor
|
||||
{
|
|
@ -3,7 +3,7 @@ using System.Threading.Tasks;
|
|||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRetrieveTwitterUsersProcessor
|
||||
{
|
|
@ -2,7 +2,7 @@
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface ISaveProgressionProcessor
|
||||
{
|
|
@ -2,7 +2,7 @@
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface ISendTweetsToFollowersProcessor
|
||||
{
|
|
@ -0,0 +1,11 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
|
||||
{
|
||||
public interface IDeleteTweetsProcessor
|
||||
{
|
||||
Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
|
||||
{
|
||||
public interface IRetrieveTweetsToDeleteProcessor
|
||||
{
|
||||
Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
|
||||
{
|
||||
public interface ISaveDeletedTweetStatusProcessor
|
||||
{
|
||||
Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Models
|
||||
{
|
||||
public class TweetToDelete
|
||||
{
|
||||
public SyncTweet Tweet { get; set; }
|
||||
public bool DeleteSuccessful { get; set; }
|
||||
}
|
||||
}
|
|
@ -6,12 +6,12 @@ using BirdsiteLive.Common.Settings;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
|
||||
{
|
|
@ -2,10 +2,10 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor
|
||||
{
|
|
@ -5,14 +5,14 @@ using System.Threading;
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Tweetinvi.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
|
||||
{
|
|
@ -7,11 +7,11 @@ using BirdsiteLive.Common.Extensions;
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Tools;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
|
||||
{
|
||||
|
@ -42,7 +42,7 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
|
||||
|
||||
var userCount = users.Any() ? users.Length : 1;
|
||||
var splitNumber = (int) Math.Ceiling(userCount / 15d);
|
||||
var splitNumber = (int)Math.Ceiling(userCount / 15d);
|
||||
var splitUsers = users.Split(splitNumber).ToList();
|
||||
|
||||
foreach (var u in splitUsers)
|
|
@ -5,11 +5,11 @@ using System.Threading.Tasks;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class SaveProgressionProcessor : ISaveProgressionProcessor
|
||||
{
|
||||
|
@ -36,7 +36,7 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
|
||||
return;
|
||||
}
|
||||
if(userWithTweetsToSync.Followers.Length == 0)
|
||||
if (userWithTweetsToSync.Followers.Length == 0)
|
||||
{
|
||||
_logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
|
||||
await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
|
|
@ -10,7 +10,7 @@ using BirdsiteLive.DAL.Contracts;
|
|||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.SubTasks;
|
||||
using BirdsiteLive.Twitter;
|
||||
|
@ -18,7 +18,7 @@ using BirdsiteLive.Twitter.Models;
|
|||
using Microsoft.Extensions.Logging;
|
||||
using Tweetinvi.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor
|
||||
{
|
|
@ -0,0 +1,32 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
||||
{
|
||||
public class SendTweetsTaskBase
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
|
||||
#region Ctor
|
||||
protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = acct,
|
||||
TweetId = tweetId,
|
||||
PublishedAt = DateTime.UtcNow,
|
||||
Inbox = inbox,
|
||||
Host = host
|
||||
};
|
||||
await _syncTweetsPostgresDal.SaveTweetAsync(tweet);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -17,7 +17,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user);
|
||||
}
|
||||
|
||||
public class SendTweetsToInboxTask : ISendTweetsToInboxTask
|
||||
public class SendTweetsToInboxTask : SendTweetsTaskBase, ISendTweetsToInboxTask
|
||||
{
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
private readonly IStatusService _statusService;
|
||||
|
@ -25,9 +25,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
private readonly InstanceSettings _settings;
|
||||
private readonly ILogger<SendTweetsToInboxTask> _logger;
|
||||
|
||||
|
||||
#region Ctor
|
||||
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
|
||||
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal)
|
||||
{
|
||||
_activityPubService = activityPubService;
|
||||
_statusService = statusService;
|
||||
|
@ -61,6 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
{
|
||||
var note = _statusService.GetStatus(user.Acct, tweet);
|
||||
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
|
||||
await SaveSyncTweetAsync(user.Acct, tweet.Id, follower.Host, inbox);
|
||||
}
|
||||
}
|
||||
catch (ArgumentException e)
|
|
@ -2,6 +2,7 @@
|
|||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
@ -16,7 +17,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance);
|
||||
}
|
||||
|
||||
public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
|
||||
public class SendTweetsToSharedInboxTask : SendTweetsTaskBase, ISendTweetsToSharedInboxTask
|
||||
{
|
||||
private readonly IStatusService _statusService;
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
|
@ -25,7 +26,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
|
||||
|
||||
#region Ctor
|
||||
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
|
||||
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal)
|
||||
{
|
||||
_activityPubService = activityPubService;
|
||||
_statusService = statusService;
|
||||
|
@ -61,6 +62,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
{
|
||||
var note = _statusService.GetStatus(user.Acct, tweet);
|
||||
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
|
||||
await SaveSyncTweetAsync(user.Acct, tweet.Id, host, inbox);
|
||||
}
|
||||
}
|
||||
catch (ArgumentException e)
|
|
@ -0,0 +1,16 @@
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using System;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base
|
||||
{
|
||||
public class RetentionBase
|
||||
{
|
||||
protected int GetRetentionTime(InstanceSettings settings)
|
||||
{
|
||||
var retentionTime = Math.Abs(settings.MaxTweetRetention);
|
||||
if (retentionTime < 1) retentionTime = 1;
|
||||
if (retentionTime > 90) retentionTime = 90;
|
||||
return retentionTime;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,54 @@
|
|||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class DeleteTweetsProcessor : IDeleteTweetsProcessor
|
||||
{
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
private readonly ILogger<DeleteTweetsProcessor> _logger;
|
||||
|
||||
#region Ctor
|
||||
public DeleteTweetsProcessor(IActivityPubService activityPubService, ILogger<DeleteTweetsProcessor> logger)
|
||||
{
|
||||
_activityPubService = activityPubService;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _activityPubService.DeleteNoteAsync(tweet.Tweet);
|
||||
tweet.DeleteSuccessful = true;
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
var code = e.StatusCode;
|
||||
if (code is HttpStatusCode.Gone or HttpStatusCode.NotFound)
|
||||
{
|
||||
_logger.LogInformation("Tweet already deleted");
|
||||
tweet.DeleteSuccessful = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(e.Message, e);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e.Message, e);
|
||||
}
|
||||
|
||||
return tweet;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,64 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct)
|
||||
{
|
||||
var batchSize = 100;
|
||||
|
||||
for (;;)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
|
||||
var from = now.AddDays(-GetRetentionTime(_instanceSettings));
|
||||
var dbBrowsingEnded = false;
|
||||
var lastId = -1L;
|
||||
|
||||
do
|
||||
{
|
||||
var tweets = await _syncTweetsPostgresDal.GetTweetsOlderThanAsync(from, lastId, batchSize);
|
||||
|
||||
foreach (var syncTweet in tweets)
|
||||
{
|
||||
var tweet = new TweetToDelete
|
||||
{
|
||||
Tweet = syncTweet
|
||||
};
|
||||
await tweetsBufferBlock.SendAsync(tweet, ct);
|
||||
}
|
||||
|
||||
if (tweets.Any()) lastId = tweets.Last().Id;
|
||||
if (tweets.Count < batchSize) dbBrowsingEnded = true;
|
||||
|
||||
} while (!dbBrowsingEnded);
|
||||
|
||||
await Task.Delay(TimeSpan.FromHours(3), ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,36 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
|
||||
{
|
||||
var retentionTime = GetRetentionTime(_instanceSettings);
|
||||
retentionTime += 20; // Delay until last retry
|
||||
var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime);
|
||||
if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
|
||||
{
|
||||
await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -4,7 +4,7 @@ using System.Threading;
|
|||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
|
|
|
@ -0,0 +1,76 @@
|
|||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline
|
||||
{
|
||||
public interface ITweetCleanUpPipeline
|
||||
{
|
||||
Task ExecuteAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
public class TweetCleanUpPipeline : ITweetCleanUpPipeline
|
||||
{
|
||||
private readonly IRetrieveTweetsToDeleteProcessor _retrieveTweetsToDeleteProcessor;
|
||||
private readonly IDeleteTweetsProcessor _deleteTweetsProcessor;
|
||||
private readonly ISaveDeletedTweetStatusProcessor _saveDeletedTweetStatusProcessor;
|
||||
private readonly ILogger<TweetCleanUpPipeline> _logger;
|
||||
|
||||
#region Ctor
|
||||
public TweetCleanUpPipeline(IRetrieveTweetsToDeleteProcessor retrieveTweetsToDeleteProcessor, IDeleteTweetsProcessor deleteTweetsProcessor, ISaveDeletedTweetStatusProcessor saveDeletedTweetStatusProcessor, ILogger<TweetCleanUpPipeline> logger)
|
||||
{
|
||||
_retrieveTweetsToDeleteProcessor = retrieveTweetsToDeleteProcessor;
|
||||
_deleteTweetsProcessor = deleteTweetsProcessor;
|
||||
_saveDeletedTweetStatusProcessor = saveDeletedTweetStatusProcessor;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
// Create blocks
|
||||
var tweetsToDeleteBufferBlock = new BufferBlock<TweetToDelete>(new DataflowBlockOptions
|
||||
{
|
||||
BoundedCapacity = 200,
|
||||
CancellationToken = ct
|
||||
});
|
||||
var deleteTweetsBlock = new TransformBlock<TweetToDelete, TweetToDelete>(
|
||||
async x => await _deleteTweetsProcessor.ProcessAsync(x, ct),
|
||||
new ExecutionDataflowBlockOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = 5,
|
||||
CancellationToken = ct
|
||||
});
|
||||
var deletedTweetsBufferBlock = new BufferBlock<TweetToDelete>(new DataflowBlockOptions
|
||||
{
|
||||
BoundedCapacity = 200,
|
||||
CancellationToken = ct
|
||||
});
|
||||
var saveProgressionBlock = new ActionBlock<TweetToDelete>(
|
||||
async x => await _saveDeletedTweetStatusProcessor.ProcessAsync(x, ct),
|
||||
new ExecutionDataflowBlockOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = 5,
|
||||
CancellationToken = ct
|
||||
});
|
||||
|
||||
// Link pipeline
|
||||
var dataflowLinkOptions = new DataflowLinkOptions { PropagateCompletion = true };
|
||||
tweetsToDeleteBufferBlock.LinkTo(deleteTweetsBlock, dataflowLinkOptions);
|
||||
deleteTweetsBlock.LinkTo(deletedTweetsBufferBlock, dataflowLinkOptions);
|
||||
deletedTweetsBufferBlock.LinkTo(saveProgressionBlock, dataflowLinkOptions);
|
||||
|
||||
// Launch tweet retriever
|
||||
var retrieveTweetsToDeleteTask = _retrieveTweetsToDeleteProcessor.GetTweetsAsync(tweetsToDeleteBufferBlock, ct);
|
||||
|
||||
// Wait
|
||||
await Task.WhenAny(new[] { retrieveTweetsToDeleteTask, saveProgressionBlock.Completion });
|
||||
|
||||
var ex = retrieveTweetsToDeleteTask.IsFaulted ? retrieveTweetsToDeleteTask.Exception : saveProgressionBlock.Completion.Exception;
|
||||
_logger.LogCritical(ex, "An error occurred, pipeline stopped");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -5,7 +5,7 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
|
||||
<PackageReference Include="TweetinviAPI" Version="4.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -1,17 +1,17 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
|
||||
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
|
||||
<Version>0.22.0</Version>
|
||||
<Version>0.23.0</Version>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />
|
||||
<PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.17.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.11" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -56,6 +56,35 @@ namespace BirdsiteLive.Controllers
|
|||
return View("Index");
|
||||
}
|
||||
|
||||
private static string _noteId;
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> DeleteNote()
|
||||
{
|
||||
var username = "twitter";
|
||||
var actor = $"https://{_instanceSettings.Domain}/users/{username}";
|
||||
var targetHost = "ioc.exchange";
|
||||
var target = $"https://{targetHost}/users/test";
|
||||
var inbox = $"/inbox";
|
||||
|
||||
var delete = new ActivityDelete
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{_noteId}#delete",
|
||||
type = "Delete",
|
||||
actor = actor,
|
||||
to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
apObject = new Tombstone
|
||||
{
|
||||
id = _noteId,
|
||||
atomUrl = _noteId
|
||||
}
|
||||
};
|
||||
|
||||
await _activityPubService.PostDataAsync(delete, targetHost, actor, inbox);
|
||||
return View("Index");
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> PostNote()
|
||||
{
|
||||
|
@ -70,6 +99,8 @@ namespace BirdsiteLive.Controllers
|
|||
var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}";
|
||||
var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}";
|
||||
|
||||
_noteId = noteId;
|
||||
|
||||
var to = $"{actor}/followers";
|
||||
to = target;
|
||||
|
||||
|
|
|
@ -49,14 +49,15 @@ namespace BirdsiteLive.Controllers
|
|||
case "Delete":
|
||||
{
|
||||
var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path,
|
||||
r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body);
|
||||
r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers),
|
||||
activity as ActivityDelete, body);
|
||||
if (succeeded) return Accepted();
|
||||
else return Unauthorized();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (FollowerIsGoneException) { } //TODO: check if user in DB
|
||||
catch (FollowerIsGoneException) { }
|
||||
|
||||
return Accepted();
|
||||
}
|
||||
|
|
|
@ -3,7 +3,6 @@ using Microsoft.AspNetCore.Mvc;
|
|||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Npgsql.TypeHandlers;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Domain.Enum;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
|
|
|
@ -3,7 +3,6 @@ using System.Collections.Generic;
|
|||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Mime;
|
||||
using System.Runtime.InteropServices.WindowsRuntime;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
|
|
@ -0,0 +1,82 @@
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Domain.Tools;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BirdsiteLive.Middlewares
|
||||
{
|
||||
public class IpWhitelistingMiddleware
|
||||
{
|
||||
private readonly RequestDelegate _next;
|
||||
private readonly ILogger<IpWhitelistingMiddleware> _logger;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly byte[][] _safelist;
|
||||
private readonly bool _ipWhitelistingSet;
|
||||
|
||||
public IpWhitelistingMiddleware(
|
||||
RequestDelegate next,
|
||||
ILogger<IpWhitelistingMiddleware> logger,
|
||||
InstanceSettings instanceSettings)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(instanceSettings.IpWhiteListing))
|
||||
{
|
||||
var ips = PatternsParser.Parse(instanceSettings.IpWhiteListing);
|
||||
_safelist = new byte[ips.Length][];
|
||||
for (var i = 0; i < ips.Length; i++)
|
||||
{
|
||||
_safelist[i] = IPAddress.Parse(ips[i]).GetAddressBytes();
|
||||
}
|
||||
_ipWhitelistingSet = true;
|
||||
}
|
||||
|
||||
_next = next;
|
||||
_logger = logger;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
|
||||
public async Task Invoke(HttpContext context)
|
||||
{
|
||||
if (_ipWhitelistingSet)
|
||||
{
|
||||
var remoteIp = context.Connection.RemoteIpAddress;
|
||||
|
||||
if (_instanceSettings.EnableXRealIpHeader)
|
||||
{
|
||||
var forwardedIp = context.Request.Headers.FirstOrDefault(x => x.Key == "X-Real-IP").Value
|
||||
.ToString();
|
||||
if (!string.IsNullOrWhiteSpace(forwardedIp))
|
||||
{
|
||||
_logger.LogDebug("Redirected IP address detected");
|
||||
remoteIp = IPAddress.Parse(forwardedIp);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Request from Remote IP address: {RemoteIp}", remoteIp);
|
||||
|
||||
var bytes = remoteIp.GetAddressBytes();
|
||||
var badIp = true;
|
||||
foreach (var address in _safelist)
|
||||
{
|
||||
if (address.SequenceEqual(bytes))
|
||||
{
|
||||
badIp = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (badIp)
|
||||
{
|
||||
_logger.LogWarning("Forbidden Request from Remote IP address: {RemoteIp}", remoteIp);
|
||||
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await _next.Invoke(context);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -29,6 +29,7 @@ namespace BirdsiteLive
|
|||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddHostedService<FederationService>();
|
||||
services.AddHostedService<TweetCleanUpService>();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ using BirdsiteLive.DAL;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Moderation;
|
||||
using BirdsiteLive.Pipeline;
|
||||
using BirdsiteLive.Tools;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace BirdsiteLive.Services
|
||||
|
@ -32,6 +33,7 @@ namespace BirdsiteLive.Services
|
|||
try
|
||||
{
|
||||
await _databaseInitializer.InitAndMigrateDbAsync();
|
||||
InitStateSynchronization.IsDbInitialized = true;
|
||||
await _moderationPipeline.ApplyModerationSettingsAsync();
|
||||
await _statusPublicationPipeline.ExecuteAsync(stoppingToken);
|
||||
}
|
||||
|
|
|
@ -0,0 +1,43 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline;
|
||||
using BirdsiteLive.Tools;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace BirdsiteLive.Services
|
||||
{
|
||||
public class TweetCleanUpService : BackgroundService
|
||||
{
|
||||
private readonly ITweetCleanUpPipeline _cleanUpPipeline;
|
||||
private readonly IHostApplicationLifetime _applicationLifetime;
|
||||
|
||||
#region Ctor
|
||||
public TweetCleanUpService(IHostApplicationLifetime applicationLifetime, ITweetCleanUpPipeline cleanUpPipeline)
|
||||
{
|
||||
_applicationLifetime = applicationLifetime;
|
||||
_cleanUpPipeline = cleanUpPipeline;
|
||||
}
|
||||
#endregion
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Wait for initialization
|
||||
while (!InitStateSynchronization.IsDbInitialized)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) return;
|
||||
await Task.Delay(250, stoppingToken);
|
||||
}
|
||||
|
||||
await _cleanUpPipeline.ExecuteAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
await Task.Delay(1000 * 30);
|
||||
_applicationLifetime.StopApplication();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -8,6 +8,7 @@ using BirdsiteLive.Common.Structs;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
|
||||
using BirdsiteLive.DAL.Postgres.Settings;
|
||||
using BirdsiteLive.Middlewares;
|
||||
using BirdsiteLive.Models;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Tools;
|
||||
|
@ -18,6 +19,7 @@ using Microsoft.AspNetCore.HttpsPolicy;
|
|||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive
|
||||
{
|
||||
|
@ -132,6 +134,9 @@ namespace BirdsiteLive
|
|||
|
||||
app.UseAuthorization();
|
||||
|
||||
var instanceSettings = Configuration.GetSection("Instance").Get<InstanceSettings>();
|
||||
app.UseMiddleware<IpWhitelistingMiddleware>(instanceSettings);
|
||||
|
||||
app.UseEndpoints(endpoints =>
|
||||
{
|
||||
endpoints.MapControllerRoute(
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
namespace BirdsiteLive.Tools
|
||||
{
|
||||
public static class InitStateSynchronization
|
||||
{
|
||||
public static bool IsDbInitialized { get; set; }
|
||||
}
|
||||
}
|
|
@ -18,6 +18,11 @@
|
|||
<button type="submit" value="Submit">Post Note</button>
|
||||
</form>
|
||||
|
||||
<form asp-controller="Debuging" asp-action="DeleteNote" method="post">
|
||||
<!-- Input and Submit elements -->
|
||||
|
||||
<button type="submit" value="Submit">Delete Note</button>
|
||||
</form>
|
||||
|
||||
<form asp-controller="Debuging" asp-action="PostRejectFollow" method="post">
|
||||
<!-- Input and Submit elements -->
|
||||
|
|
|
@ -25,7 +25,8 @@
|
|||
"SensitiveTwitterAccounts": null,
|
||||
"FailingTwitterUserCleanUpThreshold": 700,
|
||||
"FailingFollowerCleanUpThreshold": 30000,
|
||||
"UserCacheCapacity": 10000
|
||||
"UserCacheCapacity": 10000,
|
||||
"MaxTweetRetention": 20
|
||||
},
|
||||
"Db": {
|
||||
"Type": "postgres",
|
||||
|
|
|
@ -5,8 +5,8 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Dapper" Version="2.0.35" />
|
||||
<PackageReference Include="Npgsql" Version="4.1.3.1" />
|
||||
<PackageReference Include="Dapper" Version="2.0.123" />
|
||||
<PackageReference Include="Npgsql" Version="7.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
|
|||
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
|
||||
{
|
||||
private readonly PostgresTools _tools;
|
||||
private readonly Version _currentVersion = new Version(2, 5);
|
||||
private readonly Version _currentVersion = new Version(2, 6);
|
||||
private const string DbVersionType = "db-version";
|
||||
|
||||
#region Ctor
|
||||
|
@ -136,7 +136,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
|
|||
new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
|
||||
new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
|
||||
new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
|
||||
new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
|
||||
new Tuple<Version, Version>(new Version(2,4), new Version(2,5)),
|
||||
new Tuple<Version, Version>(new Version(2,5), new Version(2,6))
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -184,6 +185,23 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
|
|||
var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN";
|
||||
await _tools.ExecuteRequestAsync(addDeletedToAcct);
|
||||
}
|
||||
else if (from == new Version(2, 5) && to == new Version(2, 6))
|
||||
{
|
||||
// Create Synchronized Tweets table
|
||||
var createFollowers = $@"CREATE TABLE {_settings.SynchronizedTweetsTableName}
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
|
||||
acct VARCHAR(50) NOT NULL,
|
||||
tweetId BIGINT NOT NULL,
|
||||
inbox VARCHAR(2048) NOT NULL,
|
||||
host VARCHAR(253) NOT NULL,
|
||||
publishedAt TIMESTAMP (2) WITHOUT TIME ZONE NOT NULL,
|
||||
|
||||
UNIQUE (acct, tweetId, inbox)
|
||||
);";
|
||||
await _tools.ExecuteRequestAsync(createFollowers);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
|
@ -212,7 +230,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
|
|||
$@"DROP TABLE {_settings.DbVersionTableName};",
|
||||
$@"DROP TABLE {_settings.TwitterUserTableName};",
|
||||
$@"DROP TABLE {_settings.FollowersTableName};",
|
||||
$@"DROP TABLE {_settings.CachedTweetsTableName};"
|
||||
$@"DROP TABLE {_settings.CachedTweetsTableName};",
|
||||
$@"DROP TABLE {_settings.SynchronizedTweetsTableName};"
|
||||
};
|
||||
|
||||
foreach (var r in dropsRequests)
|
||||
|
|
|
@ -82,6 +82,21 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
|
|||
}
|
||||
}
|
||||
|
||||
public async Task<Follower> GetFollowerAsync(string actorId)
|
||||
{
|
||||
var query = $"SELECT * FROM {_settings.FollowersTableName} WHERE actorid = @actorid";
|
||||
|
||||
actorId = actorId.ToLowerInvariant().Trim();
|
||||
|
||||
using (var dbConnection = Connection)
|
||||
{
|
||||
dbConnection.Open();
|
||||
|
||||
var result = (await dbConnection.QueryAsync<SerializedFollower>(query, new { actorId })).FirstOrDefault();
|
||||
return Convert(result);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<Follower[]> GetFollowersAsync(int followedUserId)
|
||||
{
|
||||
if (followedUserId == default) throw new ArgumentException("followedUserId");
|
||||
|
|
|
@ -0,0 +1,99 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.DAL.Postgres.DataAccessLayers.Base;
|
||||
using BirdsiteLive.DAL.Postgres.Settings;
|
||||
using Dapper;
|
||||
|
||||
namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
|
||||
{
|
||||
public class SyncTweetsPostgresDal : PostgresBase, ISyncTweetsPostgresDal
|
||||
{
|
||||
#region Ctor
|
||||
public SyncTweetsPostgresDal(PostgresSettings settings) : base(settings)
|
||||
{
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<long> SaveTweetAsync(SyncTweet tweet)
|
||||
{
|
||||
if (tweet.PublishedAt == default) throw new ArgumentException("publishedAt");
|
||||
if (tweet.TweetId == default) throw new ArgumentException("tweetId");
|
||||
if (string.IsNullOrWhiteSpace(tweet.Acct)) throw new ArgumentException("acct");
|
||||
if (string.IsNullOrWhiteSpace(tweet.Inbox)) throw new ArgumentException("inbox");
|
||||
if (string.IsNullOrWhiteSpace(tweet.Host)) throw new ArgumentException("host");
|
||||
|
||||
var acct = tweet.Acct.ToLowerInvariant().Trim();
|
||||
var inbox = tweet.Inbox.ToLowerInvariant().Trim();
|
||||
var host = tweet.Host.ToLowerInvariant().Trim();
|
||||
|
||||
using (var dbConnection = Connection)
|
||||
{
|
||||
dbConnection.Open();
|
||||
|
||||
return (await dbConnection.QueryAsync<long>(
|
||||
$"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,host,publishedAt) VALUES(@acct,@tweetId,@inbox,@host,@publishedAt) RETURNING id",
|
||||
new
|
||||
{
|
||||
acct,
|
||||
tweetId = tweet.TweetId,
|
||||
inbox,
|
||||
host,
|
||||
publishedAt = tweet.PublishedAt.ToUniversalTime()
|
||||
})).First();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<SyncTweet> GetTweetAsync(long id)
|
||||
{
|
||||
if (id == default) throw new ArgumentException("id");
|
||||
|
||||
var query = $"SELECT * FROM {_settings.SynchronizedTweetsTableName} WHERE id = @id";
|
||||
|
||||
using (var dbConnection = Connection)
|
||||
{
|
||||
dbConnection.Open();
|
||||
|
||||
var result = (await dbConnection.QueryAsync<SyncTweet>(query, new { id })).FirstOrDefault();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteTweetAsync(long id)
|
||||
{
|
||||
if (id == default) throw new ArgumentException("id");
|
||||
|
||||
var query = $"DELETE FROM {_settings.SynchronizedTweetsTableName} WHERE id = @id";
|
||||
|
||||
using (var dbConnection = Connection)
|
||||
{
|
||||
dbConnection.Open();
|
||||
|
||||
await dbConnection.QueryAsync(query, new { id });
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<SyncTweet>> GetTweetsOlderThanAsync(DateTime date, long from = -1, int size = 100)
|
||||
{
|
||||
if (date == default) throw new ArgumentException("date");
|
||||
|
||||
var query = $"SELECT * FROM {_settings.SynchronizedTweetsTableName} WHERE id > @from AND publishedAt < @date ORDER BY id ASC LIMIT @size";
|
||||
|
||||
using (var dbConnection = Connection)
|
||||
{
|
||||
dbConnection.Open();
|
||||
|
||||
var result = await dbConnection.QueryAsync<SyncTweet>(query, new
|
||||
{
|
||||
from,
|
||||
date,
|
||||
size
|
||||
});
|
||||
return result.ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -8,5 +8,6 @@
|
|||
public string TwitterUserTableName { get; set; } = "twitter_users";
|
||||
public string FollowersTableName { get; set; } = "followers";
|
||||
public string CachedTweetsTableName { get; set; } = "cached_tweets";
|
||||
public string SynchronizedTweetsTableName { get; set; } = "sync_tweets";
|
||||
}
|
||||
}
|
|
@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Contracts
|
|||
public interface IFollowersDal
|
||||
{
|
||||
Task<Follower> GetFollowerAsync(string acct, string host);
|
||||
Task<Follower> GetFollowerAsync(string actorId);
|
||||
Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null,
|
||||
Dictionary<int, long> followingSyncStatus = null);
|
||||
Task<Follower[]> GetFollowersAsync(int followedUserId);
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.DAL.Contracts
|
||||
{
|
||||
public interface ISyncTweetsPostgresDal
|
||||
{
|
||||
Task<long> SaveTweetAsync(SyncTweet tweet);
|
||||
Task<SyncTweet> GetTweetAsync(long id);
|
||||
Task DeleteTweetAsync(long id);
|
||||
Task<List<SyncTweet>> GetTweetsOlderThanAsync(DateTime date, long from = -1, int size = 100);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
using System;
|
||||
|
||||
namespace BirdsiteLive.DAL.Models
|
||||
{
|
||||
public class SyncTweet
|
||||
{
|
||||
public long Id { get; set; }
|
||||
|
||||
public string Acct { get; set; }
|
||||
public string Host { get; set; } //TODO
|
||||
public string Inbox { get; set; }
|
||||
public long TweetId { get; set; }
|
||||
public DateTime PublishedAt { get; set; }
|
||||
}
|
||||
}
|
|
@ -1,16 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.3.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,16 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,16 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.3.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,16 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,16 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -19,6 +19,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base
|
|||
CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4),
|
||||
FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4),
|
||||
TwitterUserTableName = "TwitterUserTableName" + RandomGenerator.GetString(4),
|
||||
SynchronizedTweetsTableName = "SynchronizedTweetsTableName" + RandomGenerator.GetString(4),
|
||||
};
|
||||
_tools = new PostgresTools(_settings);
|
||||
}
|
||||
|
|
|
@ -0,0 +1,183 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
|
||||
using BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
|
||||
{
|
||||
[TestClass]
|
||||
public class SyncTweetsPostgresDalTests : PostgresTestingBase
|
||||
{
|
||||
[TestInitialize]
|
||||
public async Task TestInit()
|
||||
{
|
||||
var dal = new DbInitializerPostgresDal(_settings, _tools);
|
||||
var init = new DatabaseInitializer(dal);
|
||||
await init.InitAndMigrateDbAsync();
|
||||
}
|
||||
|
||||
[TestCleanup]
|
||||
public async Task CleanUp()
|
||||
{
|
||||
var dal = new DbInitializerPostgresDal(_settings, _tools);
|
||||
await dal.DeleteAllAsync();
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CreateAndGetTweets()
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = "test",
|
||||
PublishedAt = DateTime.UtcNow,
|
||||
Inbox = "/inbox",
|
||||
Host = "instance.ext",
|
||||
TweetId = 4567889
|
||||
};
|
||||
|
||||
var dal = new SyncTweetsPostgresDal(_settings);
|
||||
|
||||
var id = await dal.SaveTweetAsync(tweet);
|
||||
var result = await dal.GetTweetAsync(id);
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
|
||||
Assert.IsTrue(result.Id > 0);
|
||||
Assert.AreEqual(tweet.Acct, result.Acct);
|
||||
Assert.AreEqual(tweet.Inbox, result.Inbox);
|
||||
Assert.AreEqual(tweet.Host, result.Host);
|
||||
Assert.AreEqual(tweet.TweetId, result.TweetId);
|
||||
Assert.IsTrue(Math.Abs((tweet.PublishedAt - result.PublishedAt).Seconds) < 5);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CreateDeleteAndGetTweets()
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = "test",
|
||||
PublishedAt = DateTime.UtcNow,
|
||||
Inbox = "/inbox",
|
||||
Host = "instance.ext",
|
||||
TweetId = 4567889
|
||||
};
|
||||
|
||||
var dal = new SyncTweetsPostgresDal(_settings);
|
||||
|
||||
var id = await dal.SaveTweetAsync(tweet);
|
||||
await dal.DeleteTweetAsync(id);
|
||||
var result = await dal.GetTweetAsync(id);
|
||||
|
||||
Assert.IsNull(result);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CreateAndGetTweetsByDate()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var dal = new SyncTweetsPostgresDal(_settings);
|
||||
|
||||
var allData = new List<SyncTweet>();
|
||||
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = "test",
|
||||
PublishedAt = now.AddDays(-10 - i),
|
||||
Inbox = "/inbox",
|
||||
Host = "instance.ext",
|
||||
TweetId = 4567889 + i
|
||||
};
|
||||
allData.Add(tweet);
|
||||
await dal.SaveTweetAsync(tweet);
|
||||
}
|
||||
|
||||
var date = now.AddDays(-20);
|
||||
var result = await dal.GetTweetsOlderThanAsync(date, -1, 10);
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
Assert.IsTrue(result.Count == 10);
|
||||
|
||||
foreach (var res in result)
|
||||
{
|
||||
Assert.IsTrue(res.PublishedAt < date);
|
||||
Assert.IsTrue(res.Id > 10);
|
||||
Assert.IsTrue(res.Id < 25);
|
||||
|
||||
var original = allData.First(x => x.Acct == res.Acct && x.TweetId == res.TweetId);
|
||||
Assert.AreEqual(original.Acct, res.Acct);
|
||||
Assert.AreEqual(original.Inbox, res.Inbox);
|
||||
Assert.AreEqual(original.Host, res.Host);
|
||||
Assert.AreEqual(original.TweetId, res.TweetId);
|
||||
Assert.IsTrue(Math.Abs((original.PublishedAt - res.PublishedAt).Seconds) < 5);
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CreateAndGetTweetsByDate_Offset()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var dal = new SyncTweetsPostgresDal(_settings);
|
||||
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = "test",
|
||||
PublishedAt = now.AddDays(-10 - i),
|
||||
Inbox = "/inbox",
|
||||
Host = "instance.ext",
|
||||
TweetId = 4567889 + i
|
||||
};
|
||||
await dal.SaveTweetAsync(tweet);
|
||||
}
|
||||
|
||||
var date = now.AddDays(-20);
|
||||
var result = await dal.GetTweetsOlderThanAsync(date, 1000, 10);
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
Assert.IsTrue(result.Count == 0);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CreateAndGetTweetsByDate_Iteration()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var dal = new SyncTweetsPostgresDal(_settings);
|
||||
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = "test",
|
||||
PublishedAt = now.AddDays(-10 - i),
|
||||
Inbox = "/inbox",
|
||||
Host = "instance.ext",
|
||||
TweetId = 4567889 + i
|
||||
};
|
||||
await dal.SaveTweetAsync(tweet);
|
||||
}
|
||||
|
||||
var date = now.AddDays(-20);
|
||||
var result = await dal.GetTweetsOlderThanAsync(date, -1, 10);
|
||||
var result2 = await dal.GetTweetsOlderThanAsync(date, result.Last().Id, 10);
|
||||
|
||||
var global = result.ToList();
|
||||
global.AddRange(result2);
|
||||
var d = global.GroupBy(x => x.Id).Count();
|
||||
Assert.AreEqual(20, d);
|
||||
|
||||
foreach (var res in global)
|
||||
{
|
||||
Assert.IsTrue(res.PublishedAt < date);
|
||||
Assert.IsTrue(res.Id > 10);
|
||||
Assert.IsTrue(res.Id < 35);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,17 +1,20 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
|
||||
<PackageReference Include="Moq" Version="4.14.5" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="Moq" Version="4.18.4" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,17 +1,20 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
|
||||
<PackageReference Include="Moq" Version="4.14.5" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="Moq" Version="4.18.4" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,17 +1,20 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
|
||||
<PackageReference Include="Moq" Version="4.14.5" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.3.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="Moq" Version="4.18.4" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -1,20 +1,23 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
|
||||
<PackageReference Include="Moq" Version="4.14.5" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
<PackageReference Include="Moq" Version="4.18.4" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.2.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Processors\TweetsCleanUp\" />
|
||||
<Folder Include="Tools\" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -7,14 +7,13 @@ using BirdsiteLive.Common.Settings;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors.Federation
|
||||
{
|
||||
[TestClass]
|
||||
public class RefreshTwitterUserStatusProcessorTests
|
||||
|
@ -61,7 +60,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
|
|||
var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
Assert.AreEqual(2 , result.Length);
|
||||
Assert.AreEqual(2, result.Length);
|
||||
Assert.IsTrue(result.Any(x => x.User.Id == userId1));
|
||||
Assert.IsTrue(result.Any(x => x.User.Id == userId2));
|
||||
|
|
@ -5,11 +5,11 @@ using System.Threading.Tasks;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors.Federation
|
||||
{
|
||||
[TestClass]
|
||||
public class RetrieveFollowersProcessorTests
|
|
@ -5,7 +5,7 @@ using System.Threading.Tasks;
|
|||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.Extensions.Logging;
|
|
@ -6,13 +6,13 @@ using System.Threading.Tasks.Dataflow;
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Processors;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using BirdsiteLive.Pipeline.Tools;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors.Federation
|
||||
{
|
||||
[TestClass]
|
||||
public class RetrieveTwitterUsersProcessorTests
|
||||
|
@ -190,7 +190,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
|
|||
|
||||
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
|
||||
processor.WaitFactor = 1;
|
||||
var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
|
||||
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
|
||||
|
||||
await Task.WhenAny(t, Task.Delay(50));
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
|
@ -7,7 +7,7 @@ using BirdsiteLive.DAL.Contracts;
|
|||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using BirdsiteLive.Pipeline.Processors.SubTasks;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.Extensions.Logging;
|
|
@ -2,6 +2,7 @@
|
|||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
|
@ -87,15 +88,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -155,15 +167,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -237,15 +252,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -319,15 +345,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -402,15 +439,29 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
foreach (var tweetId in new[] { tweetId2, tweetId3 })
|
||||
{
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
}
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -493,9 +544,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId2
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
|
||||
try
|
||||
{
|
||||
|
@ -507,6 +569,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
|
@ -571,15 +634,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -634,9 +700,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
|
||||
try
|
||||
{
|
||||
|
@ -649,6 +717,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
}
|
|
@ -109,15 +109,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -199,15 +210,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -302,15 +316,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -405,15 +430,26 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -509,15 +545,29 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
foreach (var tweetId in new[] { tweetId2, tweetId3 })
|
||||
{
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
}
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -621,9 +671,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
syncTweetDalMock
|
||||
.Setup(x => x.SaveTweetAsync(
|
||||
It.Is<SyncTweet>(y => y.Acct == twitterHandle
|
||||
&& y.TweetId == tweetId2
|
||||
&& y.Inbox == inbox
|
||||
&& y.Host == host
|
||||
&& y.PublishedAt != default)))
|
||||
.ReturnsAsync(-1);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
|
||||
try
|
||||
{
|
||||
|
@ -635,6 +695,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
|
@ -720,15 +781,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
}
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
|
||||
|
||||
#region Validations
|
||||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
@ -800,9 +864,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
|
||||
|
||||
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
|
||||
|
||||
var syncTweetDalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
#endregion
|
||||
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
|
||||
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object);
|
||||
|
||||
try
|
||||
{
|
||||
|
@ -815,6 +881,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
|
|||
activityPubService.VerifyAll();
|
||||
statusServiceMock.VerifyAll();
|
||||
followersDalMock.VerifyAll();
|
||||
syncTweetDalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
}
|
|
@ -0,0 +1,160 @@
|
|||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.Federation;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp
|
||||
{
|
||||
[TestClass]
|
||||
public class DeleteTweetsProcessorTests
|
||||
{
|
||||
[TestMethod]
|
||||
public async Task Process_DeleteTweet_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var tweetId = 42;
|
||||
var tweet = new TweetToDelete
|
||||
{
|
||||
Tweet = new SyncTweet
|
||||
{
|
||||
Id = tweetId
|
||||
}
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var serviceMock = new Mock<IActivityPubService>(MockBehavior.Strict);
|
||||
serviceMock
|
||||
.Setup(x => x.DeleteNoteAsync(It.Is<SyncTweet>(y => y.Id == tweetId)))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var loggerMock = new Mock<ILogger<DeleteTweetsProcessor>>();
|
||||
#endregion
|
||||
|
||||
var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object);
|
||||
var result = await processor.ProcessAsync(tweet, CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
serviceMock.VerifyAll();
|
||||
loggerMock.VerifyAll();
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
Assert.IsTrue(result.DeleteSuccessful);
|
||||
#endregion
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Process_DeleteTweet_NotFound_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var tweetId = 42;
|
||||
var tweet = new TweetToDelete
|
||||
{
|
||||
Tweet = new SyncTweet
|
||||
{
|
||||
Id = tweetId
|
||||
}
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var serviceMock = new Mock<IActivityPubService>(MockBehavior.Strict);
|
||||
serviceMock
|
||||
.Setup(x => x.DeleteNoteAsync(It.Is<SyncTweet>(y => y.Id == tweetId)))
|
||||
.Throws(new HttpRequestException("not found", null, HttpStatusCode.NotFound));
|
||||
|
||||
var loggerMock = new Mock<ILogger<DeleteTweetsProcessor>>();
|
||||
#endregion
|
||||
|
||||
var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object);
|
||||
var result = await processor.ProcessAsync(tweet, CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
serviceMock.VerifyAll();
|
||||
loggerMock.VerifyAll();
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
Assert.IsTrue(result.DeleteSuccessful);
|
||||
#endregion
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Process_DeleteTweet_HttpError_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var tweetId = 42;
|
||||
var tweet = new TweetToDelete
|
||||
{
|
||||
Tweet = new SyncTweet
|
||||
{
|
||||
Id = tweetId
|
||||
}
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var serviceMock = new Mock<IActivityPubService>(MockBehavior.Strict);
|
||||
serviceMock
|
||||
.Setup(x => x.DeleteNoteAsync(It.Is<SyncTweet>(y => y.Id == tweetId)))
|
||||
.Throws(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway));
|
||||
|
||||
var loggerMock = new Mock<ILogger<DeleteTweetsProcessor>>();
|
||||
#endregion
|
||||
|
||||
var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object);
|
||||
var result = await processor.ProcessAsync(tweet, CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
serviceMock.VerifyAll();
|
||||
loggerMock.VerifyAll();
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
Assert.IsFalse(result.DeleteSuccessful);
|
||||
#endregion
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Process_DeleteTweet_GenericException_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var tweetId = 42;
|
||||
var tweet = new TweetToDelete
|
||||
{
|
||||
Tweet = new SyncTweet
|
||||
{
|
||||
Id = tweetId
|
||||
}
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var serviceMock = new Mock<IActivityPubService>(MockBehavior.Strict);
|
||||
serviceMock
|
||||
.Setup(x => x.DeleteNoteAsync(It.Is<SyncTweet>(y => y.Id == tweetId)))
|
||||
.Throws(new Exception());
|
||||
|
||||
var loggerMock = new Mock<ILogger<DeleteTweetsProcessor>>();
|
||||
#endregion
|
||||
|
||||
var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object);
|
||||
var result = await processor.ProcessAsync(tweet, CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
serviceMock.VerifyAll();
|
||||
loggerMock.VerifyAll();
|
||||
|
||||
Assert.IsNotNull(result);
|
||||
Assert.IsFalse(result.DeleteSuccessful);
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,131 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using System.Xml.Linq;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
using Org.BouncyCastle.Crypto;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp
|
||||
{
|
||||
[TestClass]
|
||||
public class RetrieveTweetsToDeleteProcessorTests
|
||||
{
|
||||
[TestMethod]
|
||||
public async Task Process_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var settings = new InstanceSettings
|
||||
{
|
||||
|
||||
};
|
||||
var bufferBlock = new BufferBlock<TweetToDelete>();
|
||||
|
||||
var tweetId1 = 42;
|
||||
var tweetId2 = 43;
|
||||
|
||||
var tweet1 = new SyncTweet
|
||||
{
|
||||
Id = tweetId1
|
||||
};
|
||||
var tweet2 = new SyncTweet
|
||||
{
|
||||
Id = tweetId2
|
||||
};
|
||||
var batch = new List<SyncTweet>
|
||||
{
|
||||
tweet1,
|
||||
tweet2
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var dalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
dalMock
|
||||
.Setup(x => x.GetTweetsOlderThanAsync(It.IsAny<DateTime>(), It.Is<long>(y => y == -1), It.Is<int>(y => y == 100)))
|
||||
.ReturnsAsync(batch);
|
||||
#endregion
|
||||
|
||||
var processor = new RetrieveTweetsToDeleteProcessor(dalMock.Object, settings);
|
||||
processor.GetTweetsAsync(bufferBlock, CancellationToken.None);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(1));
|
||||
|
||||
#region Validations
|
||||
dalMock.VerifyAll();
|
||||
|
||||
Assert.AreEqual(batch.Count, bufferBlock.Count);
|
||||
|
||||
bufferBlock.TryReceiveAll(out var all);
|
||||
foreach (var tweet in all)
|
||||
{
|
||||
Assert.IsTrue(batch.Any(x => x.Id == tweet.Tweet.Id));
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Process_Pagination_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var settings = new InstanceSettings { };
|
||||
var bufferBlock = new BufferBlock<TweetToDelete>();
|
||||
|
||||
|
||||
var batch = new List<SyncTweet>();
|
||||
var batch2 = new List<SyncTweet>();
|
||||
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Id = i
|
||||
};
|
||||
batch.Add(tweet);
|
||||
}
|
||||
|
||||
for (var i = 100; i < 120; i++)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Id = i
|
||||
};
|
||||
batch2.Add(tweet);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var dalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
dalMock
|
||||
.Setup(x => x.GetTweetsOlderThanAsync(It.IsAny<DateTime>(), It.Is<long>(y => y == -1), It.Is<int>(y => y == 100)))
|
||||
.ReturnsAsync(batch);
|
||||
|
||||
dalMock
|
||||
.Setup(x => x.GetTweetsOlderThanAsync(It.IsAny<DateTime>(), It.Is<long>(y => y == 99), It.Is<int>(y => y == 100)))
|
||||
.ReturnsAsync(batch2);
|
||||
#endregion
|
||||
|
||||
var processor = new RetrieveTweetsToDeleteProcessor(dalMock.Object, settings);
|
||||
processor.GetTweetsAsync(bufferBlock, CancellationToken.None);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(1));
|
||||
|
||||
#region Validations
|
||||
dalMock.VerifyAll();
|
||||
|
||||
Assert.AreEqual(batch.Count + batch2.Count, bufferBlock.Count);
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,80 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp
|
||||
{
|
||||
[TestClass]
|
||||
public class SaveDeletedTweetStatusProcessorTests
|
||||
{
|
||||
[TestMethod]
|
||||
public async Task Process_DeleteSuccessfull_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var settings = new InstanceSettings { };
|
||||
var tweetId = 42;
|
||||
var tweetToDelete = new TweetToDelete
|
||||
{
|
||||
DeleteSuccessful = true,
|
||||
Tweet = new SyncTweet
|
||||
{
|
||||
Id = tweetId
|
||||
}
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var dalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
dalMock
|
||||
.Setup(x => x.DeleteTweetAsync(It.Is<long>(y => y == tweetId)))
|
||||
.Returns(Task.CompletedTask);
|
||||
#endregion
|
||||
|
||||
var processor = new SaveDeletedTweetStatusProcessor(dalMock.Object, settings);
|
||||
await processor.ProcessAsync(tweetToDelete, CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
dalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Process_Expired_Test()
|
||||
{
|
||||
#region Stubs
|
||||
var settings = new InstanceSettings { };
|
||||
var tweetId = 42;
|
||||
var tweetToDelete = new TweetToDelete
|
||||
{
|
||||
DeleteSuccessful = false,
|
||||
Tweet = new SyncTweet
|
||||
{
|
||||
Id = tweetId,
|
||||
PublishedAt = DateTime.UtcNow.AddDays(-30)
|
||||
}
|
||||
};
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var dalMock = new Mock<ISyncTweetsPostgresDal>(MockBehavior.Strict);
|
||||
dalMock
|
||||
.Setup(x => x.DeleteTweetAsync(It.Is<long>(y => y == tweetId)))
|
||||
.Returns(Task.CompletedTask);
|
||||
#endregion
|
||||
|
||||
var processor = new SaveDeletedTweetStatusProcessor(dalMock.Object, settings);
|
||||
await processor.ProcessAsync(tweetToDelete, CancellationToken.None);
|
||||
|
||||
#region Validations
|
||||
dalMock.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,7 +2,7 @@
|
|||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Tests
|
||||
{
|
||||
[TestClass]
|
||||
public class TweetCleanUpPipelineTests
|
||||
{
|
||||
[TestMethod]
|
||||
public async Task Process()
|
||||
{
|
||||
#region Stubs
|
||||
var ct = new CancellationTokenSource(10);
|
||||
#endregion
|
||||
|
||||
#region Mocks
|
||||
var retrieveTweetsToDeleteProcessor = new Mock<IRetrieveTweetsToDeleteProcessor>(MockBehavior.Strict);
|
||||
retrieveTweetsToDeleteProcessor
|
||||
.Setup(x => x.GetTweetsAsync(
|
||||
It.IsAny<BufferBlock<TweetToDelete>>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.Delay(0));
|
||||
|
||||
var deleteTweetsProcessor = new Mock<IDeleteTweetsProcessor>(MockBehavior.Strict);
|
||||
var saveDeletedTweetStatusProcessor = new Mock<ISaveDeletedTweetStatusProcessor>(MockBehavior.Strict);
|
||||
var logger = new Mock<ILogger<TweetCleanUpPipeline>>();
|
||||
|
||||
#endregion
|
||||
|
||||
var pipeline = new TweetCleanUpPipeline(retrieveTweetsToDeleteProcessor.Object, deleteTweetsProcessor.Object, saveDeletedTweetStatusProcessor.Object, logger.Object);
|
||||
await pipeline.ExecuteAsync(ct.Token);
|
||||
|
||||
#region Validations
|
||||
retrieveTweetsToDeleteProcessor.VerifyAll();
|
||||
deleteTweetsProcessor.VerifyAll();
|
||||
saveDeletedTweetStatusProcessor.VerifyAll();
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue