Compare commits
275 Commits
Author | SHA1 | Date |
---|---|---|
Nicolas Constant | cf0797aed8 | |
Nicolas Constant | 899a595e8c | |
Nicolas Constant | 03a6c53e23 | |
Nicolas Constant | 5b93c2b760 | |
Nicolas Constant | 8a99f5ecce | |
Nicolas Constant | 57a08b67b1 | |
Nicolas Constant | ef9ee3230c | |
Nicolas Constant | e63c7950c9 | |
Nicolas Constant | 405087360c | |
Nicolas Constant | 80ac1363e5 | |
Nicolas Constant | c75507926c | |
Nicolas Constant | 61730269f2 | |
Nicolas Constant | 8589c48c9f | |
Nicolas Constant | a2010c7e3f | |
Nicolas Constant | be13b6f7c1 | |
Nicolas Constant | f1a49d1dd1 | |
Nicolas Constant | bc55778089 | |
Nicolas Constant | bb4ef071c2 | |
Nicolas Constant | e579e1b11c | |
Nicolas Constant | b223bb0216 | |
Nicolas Constant | 5b32a9021f | |
Nicolas Constant | 3460c72895 | |
Nicolas Constant | 99f4c65707 | |
Nicolas Constant | d80a00136d | |
Nicolas Constant | 584266a040 | |
Nicolas Constant | 432484eaaa | |
Nicolas Constant | f8ab522505 | |
Nicolas Constant | 8f42032512 | |
Nicolas Constant | 2542258ce4 | |
Nicolas Constant | 8bbbe037c4 | |
Nicolas Constant | 0c2acc3d7a | |
Nicolas Constant | 79ceab82b6 | |
Nicolas Constant | 8b1a61c197 | |
Nicolas Constant | 8a3ca81731 | |
Nicolas Constant | f489e03a2b | |
Nicolas Constant | cc37ed32c2 | |
Nicolas Constant | 4461884975 | |
Nicolas Constant | 21ff67e3a8 | |
Nicolas Constant | e950801f56 | |
Nicolas Constant | eccd9bdd28 | |
Nicolas Constant | f45e9ed9f7 | |
Nicolas Constant | f7ca9fd86d | |
Nicolas Constant | d64063b273 | |
Nicolas Constant | 93110d9972 | |
Nicolas Constant | 8897b8838d | |
Nicolas Constant | 36d80be7cf | |
Nicolas Constant | 05cbddbf26 | |
Nicolas Constant | f8a354d90b | |
Nicolas Constant | 89289f2c3a | |
Nicolas Constant | e804b1929c | |
Nicolas Constant | 0dfe1f4f9f | |
Nicolas Constant | 52da17393b | |
Nicolas Constant | 7c267063f9 | |
Nicolas Constant | ae42b109e9 | |
Nicolas Constant | 27e735ca4d | |
Nicolas Constant | 0eb0aa3c5d | |
Nicolas Constant | 2dd1cc7381 | |
Nicolas Constant | c910edc6b3 | |
Nicolas Constant | d5a71bbaa6 | |
Nicolas Constant | ac297b815a | |
Nicolas Constant | 1c3da007fd | |
Nicolas Constant | d219c59cfe | |
Nicolas Constant | d543a1d4f9 | |
Nicolas Constant | 7658438741 | |
Nicolas Constant | 1a939b6147 | |
Nicolas Constant | 8840d1007c | |
Nicolas Constant | 9a6971c6bc | |
Nicolas Constant | 2e5bb28ff8 | |
Nicolas Constant | 4157f613ea | |
Nicolas Constant | 6f8a2c0373 | |
Nicolas Constant | 4d365e2043 | |
Elan Hasson | d08caf3684 | |
Nicolas Constant | 86c852b8a8 | |
Nicolas Constant | 1922b7dfc8 | |
Nicolas Constant | 4fb04c16b8 | |
Nicolas Constant | df68b9c370 | |
Nicolas Constant | 498134f215 | |
Nicolas Constant | 76b2e659ab | |
Nicolas Constant | 15f0ad55ae | |
Nicolas Constant | ec3234324c | |
Nicolas Constant | cc9985eb1d | |
Nicolas Constant | 2880de9dda | |
nemobis | 6df0529d0b | |
Nicolas Constant | 4c4fc95da3 | |
Nicolas Constant | 9415eb2e0c | |
Nicolas Constant | ed3faab924 | |
Nicolas Constant | 7007b6309a | |
Nicolas Constant | d0dd317723 | |
Nicolas Constant | 0e9938b712 | |
Nicolas Constant | e78bc262ed | |
Nicolas Constant | a7b4a4978a | |
Nicolas Constant | 4e9fec1a46 | |
Nicolas Constant | d59e89a901 | |
Nicolas Constant | b0e7601333 | |
Nicolas Constant | 9f9f88aab7 | |
Nicolas Constant | 420d8867e7 | |
Nicolas Constant | d1c5a59247 | |
Nicolas Constant | 662f97e53c | |
Nicolas Constant | 446b222881 | |
Nicolas Constant | b116f6a3ce | |
Nicolas Constant | c043e0b6a0 | |
Nicolas Constant | 1536880c73 | |
Nicolas Constant | 25ba19bc4f | |
Nicolas Constant | bf7baba789 | |
Nicolas Constant | c371218672 | |
Nicolas Constant | 18e0397efe | |
Nicolas Constant | 15d7e87466 | |
Nicolas Constant | 3a998b60ac | |
Nicolas Constant | 26cca6a306 | |
Nicolas Constant | 5c4641c6ae | |
Nicolas Constant | 04b8cfa0e4 | |
Nicolas Constant | a36171c163 | |
Nicolas Constant | 7205a09eaa | |
Nicolas Constant | 93b43ee4a0 | |
Nicolas Constant | e21eacd0f7 | |
Nicolas Constant | 5ef8af47eb | |
Nicolas Constant | 3a8a51979e | |
Nicolas Constant | f6b0c13ce8 | |
Nicolas Constant | a21b493910 | |
Nicolas Constant | 1855830703 | |
Nicolas Constant | 5014d7a396 | |
Nicolas Constant | 446c024822 | |
Nicolas Constant | 143d431f0f | |
Nicolas Constant | a94f524d17 | |
Nicolas Constant | c91be2556c | |
Nicolas Constant | 567453a0b8 | |
Nicolas Constant | 767b552929 | |
Nicolas Constant | 98e869f064 | |
Nicolas Constant | 9260869dfe | |
Nicolas Constant | 29728a4175 | |
Nicolas Constant | 18d2096dc3 | |
Nicolas Constant | 6e978f1cdd | |
Nicolas Constant | 806463c126 | |
Nicolas Constant | d3d330d74e | |
Nicolas Constant | 77e3caebe0 | |
Nicolas Constant | 713b0b0fd4 | |
Nicolas Constant | 2258c93e09 | |
Nicolas Constant | f594aefea8 | |
Nicolas Constant | 5121f6c7c2 | |
Nicolas Constant | 363481a997 | |
Nicolas Constant | c4ee6be8ce | |
Nicolas Constant | f7e00b4562 | |
Nicolas Constant | 12e4b36def | |
Nicolas Constant | b28532b5bd | |
Nicolas Constant | 71f6d3f3f4 | |
Nicolas Constant | 5b34819270 | |
Nicolas Constant | 05b5a05866 | |
Nicolas Constant | 66e1e84da2 | |
Nicolas Constant | 2a4136cc8d | |
Nicolas Constant | 22b0d6da84 | |
Nicolas Constant | 4eb2909d6c | |
Nicolas Constant | a93d4b8f31 | |
nytpu | 894f98b0f2 | |
Nicolas Constant | 07dc912624 | |
Nicolas Constant | e6bb9f192d | |
Nicolas Constant | 50cf8d799c | |
Nicolas Constant | 2730c40ae8 | |
Nicolas Constant | 9686d6187d | |
Nicolas Constant | a9a59eb433 | |
Nicolas Constant | 0896e8a2bf | |
Nicolas Constant | b3bb57157f | |
Nicolas Constant | 0e787bbca8 | |
Nicolas Constant | 7ed993e51d | |
Nicolas Constant | 2b09bc37f8 | |
Nicolas Constant | 40d02b3353 | |
Nicolas Constant | bc0e6e95d6 | |
Nicolas Constant | 77088c78a4 | |
Nicolas Constant | d95163f696 | |
Nicolas Constant | ed7ac5303e | |
Nicolas Constant | a2e547104f | |
Nicolas Constant | 77f4b49d9a | |
Nicolas Constant | cad6c2018e | |
Nicolas Constant | 32a4a6356b | |
Nicolas Constant | d0f817e1a8 | |
Nicolas Constant | 13026a56ad | |
Nicolas Constant | e64d584ca0 | |
Nicolas Constant | 2aa7e89e1a | |
Nicolas Constant | 809a6b605f | |
Nicolas Constant | 25221c33e0 | |
Nicolas Constant | f1a7146c67 | |
Nicolas Constant | 8d0a612238 | |
Nicolas Constant | 32ef34a094 | |
Nicolas Constant | 11adfa6c7d | |
Nicolas Constant | 1a578f3bbc | |
Nicolas Constant | 07d09ebdb3 | |
Nicolas Constant | c1f4aedada | |
Nicolas Constant | 08461e45c5 | |
Nicolas Constant | a4d62fac4f | |
Nicolas Constant | 84b0da5093 | |
Nicolas Constant | 3a505127eb | |
Nicolas Constant | fcdb73d391 | |
Nicolas Constant | 699f422e1b | |
Nicolas Constant | d421cd1163 | |
Nicolas Constant | 834c999d53 | |
Nicolas Constant | ff5b5e2b5a | |
Nicolas Constant | a198f3d81b | |
Nicolas Constant | 27312dd3c4 | |
Nicolas Constant | 467135e41f | |
Nicolas Constant | dbb485be5d | |
Nicolas Constant | 9038573419 | |
Nicolas Constant | be5ff31069 | |
Nicolas Constant | e448bd250c | |
Nicolas Constant | bab263f048 | |
Nicolas Constant | 962eac4215 | |
Nicolas Constant | c84b7937a2 | |
Nicolas Constant | 26584fc5c8 | |
Nicolas Constant | 7503f34882 | |
Nicolas Constant | 7abc4a3b3e | |
Nicolas Constant | 67e5e3418b | |
Nicolas Constant | ed8242834e | |
Nicolas Constant | 45255fa39d | |
Nicolas Constant | bf660b80f6 | |
Nicolas Constant | 3896afc380 | |
Nicolas Constant | 892be2c2b8 | |
Nicolas Constant | 67801ea631 | |
Nicolas Constant | 56f0a3396a | |
Nicolas Constant | 2de826a023 | |
Nicolas Constant | 2ff6128d70 | |
Nicolas Constant | abce8e5ab5 | |
Nicolas Constant | 4fbb691672 | |
Nicolas Constant | bde9c0c34d | |
Nicolas Constant | f0dc54ad01 | |
Nicolas Constant | 922de367a8 | |
Nicolas Constant | aea0244b2a | |
Nicolas Constant | 0871243a18 | |
Nicolas Constant | be9bf5ebc5 | |
Nicolas Constant | 5781af6b3b | |
Nicolas Constant | 154a1da930 | |
Nicolas Constant | 9b25a86d1d | |
Nicolas Constant | e39c226965 | |
Nicolas Constant | 5d2d5ffd52 | |
Nicolas Constant | af092d942d | |
Nicolas Constant | 537270cceb | |
Nicolas Constant | dad118d222 | |
Nicolas Constant | 7a88b8002f | |
Nicolas Constant | 27c6eb697a | |
Nicolas Constant | bc6fe94101 | |
Nicolas Constant | 30bd16447f | |
Nicolas Constant | a2597b72a9 | |
Nicolas Constant | 9e87335064 | |
Nicolas Constant | 2f6eacc524 | |
Nicolas Constant | c0049696bf | |
Nicolas Constant | 90ca06b48b | |
Nicolas Constant | 2181230d96 | |
Nicolas Constant | eaae2f1f47 | |
Nicolas Constant | 4b1aa7aa5c | |
Nicolas Constant | 0e1178f128 | |
Nicolas Constant | 16b8909abc | |
Nicolas Constant | 9bdca4e202 | |
Nicolas Constant | cd36c62935 | |
Nicolas Constant | f5fc24d2f5 | |
Nicolas Constant | d660dc990c | |
Nicolas Constant | 5aa8deb47b | |
Nicolas Constant | 7fec7e765e | |
Nicolas Constant | 5f480b4baa | |
Nicolas Constant | e73d76bdd8 | |
Nicolas Constant | 567f02a4e2 | |
Nicolas Constant | 674fde74bd | |
Nicolas Constant | fcc7bbaa44 | |
Nicolas Constant | c02b4804f5 | |
Nicolas Constant | 6d5fe3089e | |
Nicolas Constant | 130ad6ff63 | |
Nicolas Constant | 39419fd50c | |
Nicolas Constant | 9920316863 | |
Nicolas Constant | 0bb7e8912f | |
Nicolas Constant | 5077819722 | |
Nicolas Constant | cee46df117 | |
Nicolas Constant | 392c7ca494 | |
Nicolas Constant | 3e772f2cd4 | |
Nicolas Constant | 024327ffe9 | |
Nicolas Constant | 2d61ae9ae3 | |
Nicolas Constant | 4b0fe65776 | |
Nicolas Constant | bcf207acb5 | |
Nicolas Constant | f0fce82d27 | |
Nicolas Constant | 1d8e622ab5 |
|
@ -0,0 +1 @@
|
|||
patreon: nicolasconstant
|
|
@ -16,7 +16,7 @@ jobs:
|
|||
- name: Setup .NET Core
|
||||
uses: actions/setup-dotnet@v1
|
||||
with:
|
||||
dotnet-version: 3.1.101
|
||||
dotnet-version: 6.0.x
|
||||
- name: Install dependencies
|
||||
run: dotnet restore
|
||||
working-directory: ${{env.working-directory}}
|
||||
|
@ -24,5 +24,5 @@ jobs:
|
|||
run: dotnet build --configuration Release --no-restore
|
||||
working-directory: ${{env.working-directory}}
|
||||
- name: Test
|
||||
run: dotnet test --no-restore --verbosity quiet
|
||||
run: dotnet test --no-restore --verbosity minimal
|
||||
working-directory: ${{env.working-directory}}
|
||||
|
|
|
@ -351,3 +351,4 @@ MigrationBackup/
|
|||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
/src/BSLManager/Properties/launchSettings.json
|
||||
|
|
|
@ -0,0 +1,51 @@
|
|||
# BSLManager
|
||||
|
||||
A CLI is provided in the Docker image so that admins can manage their instance.
|
||||
|
||||
## Access to the CLI
|
||||
|
||||
Since the CLI is packaged into the docker image, you'll have to open a shell from the container. To do so, list first your running containers:
|
||||
|
||||
```
|
||||
docker ps
|
||||
```
|
||||
|
||||
This should display you something equivalent to this:
|
||||
|
||||
```
|
||||
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
|
||||
3734c41af5a7 postgres:9.6 "docker-entrypoint.s…" 2 weeks ago Up 2 weeks 5432/tcp db_1
|
||||
be6870fe103e nicolasconstant/birdsitelive:latest "dotnet BirdsiteLive…" 6 weeks ago Up 2 weeks 443/tcp, 0.0.0.0:5000->80/tcp birdsitelive
|
||||
```
|
||||
|
||||
Find the BSL container and keep the ID, here it's `be6870fe103e`. And you only need the three first char to identify it, so we'll be using `be6`.
|
||||
|
||||
Then open a shell inside the container (change `be6` with your own id):
|
||||
|
||||
```
|
||||
docker exec -it be6 /bin/bash
|
||||
```
|
||||
|
||||
And you should now be inside the container, and all you have to do is calling the CLI:
|
||||
|
||||
```
|
||||
./BSLManager
|
||||
```
|
||||
|
||||
## Setting up the CLI
|
||||
|
||||
The manager will ask you to provide information about the database and the instance.
|
||||
Those must be same than the ones in the `docker-compose.yml` file.
|
||||
Provide the information, review it and validate it. Then the CLI UI should shows up.
|
||||
|
||||
## Using the CLI
|
||||
|
||||
You can navigate between the sections with the arrows and tab keys.
|
||||
|
||||
The **filter** permits to filter the list of users with a pattern.
|
||||
|
||||
All users have their followings count provided next to them.
|
||||
You can select any user by using the up/down arrow keys and hitting the `Enter` key, this will display more information about the user.
|
||||
You can also remove a user and all their followings by hitting the `Del` key. You will be prompted by a confirmation message, and you'll be able to remove this user.
|
||||
|
||||
Deleting users having a lots of followings can take some time: after the prompt has closed the process is still running and will update the list after that. Let the software do its thing and it will go through.
|
|
@ -1,17 +1,14 @@
|
|||
#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-buster-slim AS base
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base
|
||||
WORKDIR /app
|
||||
EXPOSE 80
|
||||
EXPOSE 443
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/core/sdk:3.1-buster AS build
|
||||
FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish
|
||||
COPY ./src/ ./src/
|
||||
RUN dotnet restore "/src/BirdsiteLive/BirdsiteLive.csproj"
|
||||
RUN dotnet build "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/build
|
||||
|
||||
FROM build AS publish
|
||||
RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
|
||||
RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
|
||||
|
||||
FROM base AS final
|
||||
WORKDIR /app
|
||||
|
|
106
INSTALLATION.md
106
INSTALLATION.md
|
@ -4,6 +4,9 @@
|
|||
|
||||
You will need a Twitter API key to make BirdsiteLIVE working. First create an **Standalone App** in the [Twitter developer portal](https://developer.twitter.com/en/portal/projects-and-apps) and retrieve the API Key and API Secret Key.
|
||||
|
||||
Please make sure you are using a **Standalone App** API Key and not a **Project App** API Key (that will NOT work with BirdsiteLIVE), if you don't see the **Standalone App** section, you might need to [apply for Elevated Access](https://developer.twitter.com/en/portal/products/elevated) as described in the [API documentation](https://developer.twitter.com/en/support/twitter-api/developer-account).
|
||||
|
||||
|
||||
## Server prerequisites
|
||||
|
||||
Your instance will need [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed and working.
|
||||
|
@ -116,7 +119,7 @@ sudo certbot --nginx -d {your-domain-name.com}
|
|||
|
||||
Make sure you're redirecting all traffic to https when asked.
|
||||
|
||||
Finally check that the auto-revewal will work as espected:
|
||||
Finally check that the auto-renewal will work as espected:
|
||||
|
||||
```
|
||||
sudo certbot renew --dry-run
|
||||
|
@ -138,11 +141,11 @@ sudo ufw status
|
|||
|
||||
You should now have an up and running BirdsiteLIVE instance!
|
||||
|
||||
## Upgrading
|
||||
## Updating
|
||||
|
||||
Make sure your data belong outside the containers before migrating (set by default).
|
||||
|
||||
To upgrade your installation to the latest release:
|
||||
To update your installation to the latest release:
|
||||
|
||||
```
|
||||
# Edit `docker-compose.yml` to update the version, if you have one specified
|
||||
|
@ -152,6 +155,101 @@ docker-compose pull
|
|||
docker-compose up -d
|
||||
```
|
||||
|
||||
## Auto-Updating
|
||||
|
||||
To set auto-updates on your deployment, add to the `docker-compose.yml` file this section:
|
||||
|
||||
```diff
|
||||
version: "3"
|
||||
|
||||
networks:
|
||||
birdsitelivenetwork:
|
||||
external: false
|
||||
|
||||
services:
|
||||
server:
|
||||
image: nicolasconstant/birdsitelive:latest
|
||||
[...]
|
||||
|
||||
db:
|
||||
image: postgres:9.6
|
||||
[...]
|
||||
|
||||
+ watchtower:
|
||||
+ image: containrrr/watchtower
|
||||
+ restart: always
|
||||
+ container_name: watchtower
|
||||
+ environment:
|
||||
+ - WATCHTOWER_CLEANUP=true
|
||||
+ volumes:
|
||||
+ - /var/run/docker.sock:/var/run/docker.sock
|
||||
+ command: --interval 300
|
||||
```
|
||||
|
||||
## IP Whitelisting
|
||||
|
||||
If you want to use the IP Whitelisting functionality (see related [variable](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)) and you are using the nginx reverse proxy set as before, please add the following:
|
||||
|
||||
```
|
||||
sudo nano /etc/nginx/sites-enabled/{your-domain-name.com}
|
||||
```
|
||||
|
||||
``` diff
|
||||
server {
|
||||
listen 80;
|
||||
server_name {your-domain-name.com};
|
||||
location / {
|
||||
proxy_pass http://localhost:5000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection keep-alive;
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
+ proxy_set_header X-Real-IP $remote_addr;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And edit the docker-compose file as follow:
|
||||
|
||||
```diff
|
||||
version: "3"
|
||||
|
||||
networks:
|
||||
birdsitelivenetwork:
|
||||
external: false
|
||||
|
||||
services:
|
||||
server:
|
||||
image: nicolasconstant/birdsitelive:latest
|
||||
restart: always
|
||||
container_name: birdsitelive
|
||||
environment:
|
||||
- Instance:Domain=domain.name
|
||||
- Instance:AdminEmail=name@domain.ext
|
||||
+ - Instance:IpWhiteListing=127.0.0.1;127.0.0.2
|
||||
+ - Instance:EnableXRealIpHeader=true
|
||||
- Db:Type=postgres
|
||||
- Db:Host=db
|
||||
- Db:Name=birdsitelive
|
||||
- Db:User=birdsitelive
|
||||
- Db:Password=birdsitelive
|
||||
- Twitter:ConsumerKey=twitter.api.key
|
||||
- Twitter:ConsumerSecret=twitter.api.key
|
||||
networks:
|
||||
- birdsitelivenetwork
|
||||
ports:
|
||||
- "5000:80"
|
||||
depends_on:
|
||||
- db
|
||||
|
||||
db:
|
||||
image: postgres:9.6
|
||||
[...]
|
||||
```
|
||||
|
||||
## More options
|
||||
|
||||
You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)
|
||||
You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)
|
||||
|
|
|
@ -8,15 +8,17 @@ BirdsiteLIVE is an ActivityPub bridge from Twitter, it's mostly a pet project/pl
|
|||
|
||||
## State of development
|
||||
|
||||
The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not provide a good state-of-the-art codebase. But I might refactor it to make it cleaner.
|
||||
The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not to provide a good state-of-the-art codebase. But I might refactor it to make it cleaner.
|
||||
|
||||
## Official instance
|
||||
|
||||
You can find an official (and temporary) instance here: [beta.birdsite.live](https://beta.birdsite.live). This instance can disapear at any time, if you want a long term instance you should install your own or use another one.
|
||||
There's none! Please read [here why I've stopped it](https://write.as/nicolas-constant/closing-the-official-bsl-instance).
|
||||
|
||||
## Installation
|
||||
|
||||
I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available.
|
||||
I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive) (linux/amd64 only). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available.
|
||||
|
||||
Also a [CLI](https://github.com/NicolasConstant/BirdsiteLive/blob/master/BSLManager.md) is available for adminitrative tasks.
|
||||
|
||||
## License
|
||||
|
||||
|
|
87
VARIABLES.md
87
VARIABLES.md
|
@ -2,6 +2,40 @@
|
|||
|
||||
You can configure some of BirdsiteLIVE's settings via environment variables (those are optionnals):
|
||||
|
||||
## Blacklisting & Whitelisting
|
||||
|
||||
### Fediverse users and instances
|
||||
|
||||
Here are the supported patterns to describe Fediverse users and/or instances:
|
||||
|
||||
* `@user@instance.ext` to describe a Fediverse user
|
||||
* `instance.ext` to describe an instance under a domain name
|
||||
* `*.instance.ext` to describe instances from all subdomains of a domain name (this doesn't include the instance.ext, if you want both you need to add both)
|
||||
|
||||
You can whitelist or blacklist fediverses users by settings the followings variables with the above patterns separated by `;`:
|
||||
|
||||
* `Moderation:FollowersWhiteListing` Fediverse Whitelisting
|
||||
* `Moderation:FollowersBlackListing` Fediverse Blacklisting
|
||||
|
||||
If the whitelisting is set, only given patterns can follow twitter accounts on the instance.
|
||||
If blacklisted, the given patterns can't follow twitter accounts on the instance.
|
||||
If both whitelisting and blacklisting are set, only the whitelisting will be active.
|
||||
|
||||
### Twitter users
|
||||
|
||||
Here is the supported pattern to describe Twitter users:
|
||||
|
||||
* `twitter_handle` to describe a Twitter user
|
||||
|
||||
You can whitelist or blacklist twitter users by settings the followings variables with the above pattern separated by `;`:
|
||||
|
||||
* `Moderation:TwitterAccountsWhiteListing` Twitter Whitelisting
|
||||
* `Moderation:TwitterAccountsBlackListing` Twitter Blacklisting
|
||||
|
||||
If the whitelisting is set, only given patterns can be followed on the instance.
|
||||
If blacklisted, the given patterns can't be followed on the instance.
|
||||
If both whitelisting and blacklisting are set, only the whitelisting will be active.
|
||||
|
||||
## Logging
|
||||
|
||||
* `Logging:Type` (default: none) set the type of the logging and monitoring system, currently the only type supported is `insights` for *Azure Application Insights* (PR welcome to support other types)
|
||||
|
@ -11,4 +45,55 @@ You can configure some of BirdsiteLIVE's settings via environment variables (tho
|
|||
|
||||
* `Instance:Name` (default: BirdsiteLIVE) the name of the instance
|
||||
* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
|
||||
* `Instance:PublishReplies` (default: false) to enable or disable replies publishing.
|
||||
* `Instance:PublishReplies` (default: false) to enable or disable replies publishing.
|
||||
* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
|
||||
* `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`.
|
||||
* `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
|
||||
* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
|
||||
* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
|
||||
* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
|
||||
* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
|
||||
* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
|
||||
|
||||
# Docker Compose full example
|
||||
|
||||
In order to illustrate above variables, here is an example of an updated `docker-compose.yml` file:
|
||||
|
||||
```diff
|
||||
version: "3"
|
||||
|
||||
networks:
|
||||
[...]
|
||||
|
||||
services:
|
||||
server:
|
||||
image: nicolasconstant/birdsitelive:latest
|
||||
[...]
|
||||
environment:
|
||||
- Instance:Domain=domain.name
|
||||
- Instance:AdminEmail=name@domain.ext
|
||||
- Db:Type=postgres
|
||||
- Db:Host=db
|
||||
- Db:Name=birdsitelive
|
||||
- Db:User=birdsitelive
|
||||
- Db:Password=birdsitelive
|
||||
- Twitter:ConsumerKey=twitter.api.key
|
||||
- Twitter:ConsumerSecret=twitter.api.key
|
||||
+ - Moderation:FollowersWhiteListing=@me@my-instance.ca;friend-instance.com;*.friend-instance.com
|
||||
+ - Moderation:TwitterAccountsBlackListing=douchebag;jerk_88;theRealIdiot
|
||||
+ - Instance:Name=MyTwitterRelay
|
||||
+ - Instance:ResolveMentionsInProfiles=false
|
||||
+ - Instance:PublishReplies=true
|
||||
+ - Instance:UnlistedTwitterAccounts=cocacola;twitter
|
||||
+ - Instance:SensitiveTwitterAccounts=archillect
|
||||
networks:
|
||||
[...]
|
||||
|
||||
db:
|
||||
image: postgres:9.6
|
||||
[...]
|
||||
```
|
||||
|
||||
## Apply the modifications
|
||||
|
||||
After the modification of the `docker-compose.yml` file, you will need to run `docker-compose up -d` to apply the changes.
|
||||
|
|
|
@ -0,0 +1,252 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BSLManager.Domain;
|
||||
using BSLManager.Tools;
|
||||
using Terminal.Gui;
|
||||
|
||||
namespace BSLManager
|
||||
{
|
||||
public class App
|
||||
{
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly IRemoveFollowerAction _removeFollowerAction;
|
||||
|
||||
private readonly FollowersListState _state = new FollowersListState();
|
||||
|
||||
#region Ctor
|
||||
public App(IFollowersDal followersDal, IRemoveFollowerAction removeFollowerAction)
|
||||
{
|
||||
_followersDal = followersDal;
|
||||
_removeFollowerAction = removeFollowerAction;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public void Run()
|
||||
{
|
||||
Application.Init();
|
||||
var top = Application.Top;
|
||||
|
||||
// Creates the top-level window to show
|
||||
var win = new Window("BSL Manager")
|
||||
{
|
||||
X = 0,
|
||||
Y = 1, // Leave one row for the toplevel menu
|
||||
|
||||
// By using Dim.Fill(), it will automatically resize without manual intervention
|
||||
Width = Dim.Fill(),
|
||||
Height = Dim.Fill()
|
||||
};
|
||||
|
||||
top.Add(win);
|
||||
|
||||
// Creates a menubar, the item "New" has a help menu.
|
||||
var menu = new MenuBar(new MenuBarItem[]
|
||||
{
|
||||
new MenuBarItem("_File", new MenuItem[]
|
||||
{
|
||||
new MenuItem("_Quit", "", () =>
|
||||
{
|
||||
if (Quit()) top.Running = false;
|
||||
})
|
||||
}),
|
||||
//new MenuBarItem ("_Edit", new MenuItem [] {
|
||||
// new MenuItem ("_Copy", "", null),
|
||||
// new MenuItem ("C_ut", "", null),
|
||||
// new MenuItem ("_Paste", "", null)
|
||||
//})
|
||||
});
|
||||
top.Add(menu);
|
||||
|
||||
static bool Quit()
|
||||
{
|
||||
var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
|
||||
return n == 0;
|
||||
}
|
||||
|
||||
RetrieveUserList();
|
||||
|
||||
var list = new ListView(_state.GetDisplayableList())
|
||||
{
|
||||
X = 1,
|
||||
Y = 3,
|
||||
Width = Dim.Fill(),
|
||||
Height = Dim.Fill()
|
||||
};
|
||||
|
||||
list.KeyDown += _ =>
|
||||
{
|
||||
if (_.KeyEvent.Key == Key.Enter)
|
||||
{
|
||||
OpenFollowerDialog(list.SelectedItem);
|
||||
}
|
||||
else if (_.KeyEvent.Key == Key.Delete
|
||||
|| _.KeyEvent.Key == Key.DeleteChar
|
||||
|| _.KeyEvent.Key == Key.Backspace
|
||||
|| _.KeyEvent.Key == Key.D)
|
||||
{
|
||||
OpenDeleteDialog(list.SelectedItem);
|
||||
}
|
||||
};
|
||||
|
||||
var listingFollowersLabel = new Label(1, 0, "Listing followers");
|
||||
var filterLabel = new Label("Filter: ") { X = 1, Y = 1 };
|
||||
var filterText = new TextField("")
|
||||
{
|
||||
X = Pos.Right(filterLabel),
|
||||
Y = 1,
|
||||
Width = 40
|
||||
};
|
||||
|
||||
filterText.KeyDown += _ =>
|
||||
{
|
||||
var text = filterText.Text.ToString();
|
||||
if (_.KeyEvent.Key == Key.Enter && !string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
_state.FilterBy(text);
|
||||
ConsoleGui.RefreshUI();
|
||||
}
|
||||
};
|
||||
|
||||
win.Add(
|
||||
listingFollowersLabel,
|
||||
filterLabel,
|
||||
filterText,
|
||||
list
|
||||
);
|
||||
|
||||
Application.Run();
|
||||
}
|
||||
|
||||
private void OpenFollowerDialog(int selectedIndex)
|
||||
{
|
||||
var close = new Button(3, 14, "Close");
|
||||
close.Clicked += () => Application.RequestStop();
|
||||
|
||||
var dialog = new Dialog("Info", 60, 18, close);
|
||||
|
||||
var follower = _state.GetElementAt(selectedIndex);
|
||||
|
||||
var name = new Label($"User: @{follower.Acct}@{follower.Host}")
|
||||
{
|
||||
X = 1,
|
||||
Y = 1,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
var following = new Label($"Following Count: {follower.Followings.Count}")
|
||||
{
|
||||
X = 1,
|
||||
Y = 3,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
|
||||
{
|
||||
X = 1,
|
||||
Y = 4,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
var inbox = new Label($"Inbox: {follower.InboxRoute}")
|
||||
{
|
||||
X = 1,
|
||||
Y = 5,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
|
||||
{
|
||||
X = 1,
|
||||
Y = 6,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
|
||||
dialog.Add(name);
|
||||
dialog.Add(following);
|
||||
dialog.Add(errors);
|
||||
dialog.Add(inbox);
|
||||
dialog.Add(sharedInbox);
|
||||
dialog.Add(close);
|
||||
Application.Run(dialog);
|
||||
}
|
||||
|
||||
private void OpenDeleteDialog(int selectedIndex)
|
||||
{
|
||||
bool okpressed = false;
|
||||
var ok = new Button(10, 14, "Yes");
|
||||
ok.Clicked += () =>
|
||||
{
|
||||
Application.RequestStop();
|
||||
okpressed = true;
|
||||
};
|
||||
|
||||
var cancel = new Button(3, 14, "No");
|
||||
cancel.Clicked += () => Application.RequestStop();
|
||||
|
||||
var dialog = new Dialog("Delete", 60, 18, cancel, ok);
|
||||
|
||||
var follower = _state.GetElementAt(selectedIndex);
|
||||
var name = new Label($"User: @{follower.Acct}@{follower.Host}")
|
||||
{
|
||||
X = 1,
|
||||
Y = 1,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
var entry = new Label("Delete user and remove all their followings?")
|
||||
{
|
||||
X = 1,
|
||||
Y = 3,
|
||||
Width = Dim.Fill(),
|
||||
Height = 1
|
||||
};
|
||||
dialog.Add(name);
|
||||
dialog.Add(entry);
|
||||
Application.Run(dialog);
|
||||
|
||||
if (okpressed)
|
||||
{
|
||||
DeleteAndRemoveUser(selectedIndex);
|
||||
}
|
||||
}
|
||||
|
||||
private void DeleteAndRemoveUser(int el)
|
||||
{
|
||||
Application.MainLoop.Invoke(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var userToDelete = _state.GetElementAt(el);
|
||||
|
||||
BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
|
||||
await _removeFollowerAction.ProcessAsync(userToDelete);
|
||||
BasicLogger.Log($"Remove user from list");
|
||||
_state.RemoveAt(el);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
BasicLogger.Log(e.Message);
|
||||
}
|
||||
|
||||
ConsoleGui.RefreshUI();
|
||||
});
|
||||
}
|
||||
|
||||
private void RetrieveUserList()
|
||||
{
|
||||
Application.MainLoop.Invoke(async () =>
|
||||
{
|
||||
var followers = await _followersDal.GetAllFollowersAsync();
|
||||
_state.Load(followers.ToList());
|
||||
ConsoleGui.RefreshUI();
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,28 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Lamar" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
|
||||
<PackageReference Include="Terminal.Gui" Version="1.9.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
|
||||
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="key.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
|
@ -0,0 +1,94 @@
|
|||
using System;
|
||||
using System.Net.Http;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Common.Structs;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
|
||||
using BirdsiteLive.DAL.Postgres.Settings;
|
||||
using Lamar;
|
||||
using Lamar.Scanning.Conventions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BSLManager
|
||||
{
|
||||
public class Bootstrapper
|
||||
{
|
||||
private readonly DbSettings _dbSettings;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public Bootstrapper(DbSettings dbSettings, InstanceSettings instanceSettings)
|
||||
{
|
||||
_dbSettings = dbSettings;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public Container Init()
|
||||
{
|
||||
var container = new Container(x =>
|
||||
{
|
||||
x.For<DbSettings>().Use(x => _dbSettings);
|
||||
|
||||
x.For<InstanceSettings>().Use(x => _instanceSettings);
|
||||
|
||||
if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
|
||||
var postgresSettings = new PostgresSettings
|
||||
{
|
||||
ConnString = connString
|
||||
};
|
||||
x.For<PostgresSettings>().Use(x => postgresSettings);
|
||||
|
||||
x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
|
||||
x.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
|
||||
x.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotImplementedException($"{_dbSettings.Type} is not supported");
|
||||
}
|
||||
|
||||
var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
|
||||
x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
|
||||
|
||||
x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
|
||||
|
||||
x.Scan(_ =>
|
||||
{
|
||||
_.Assembly("BirdsiteLive.Twitter");
|
||||
_.Assembly("BirdsiteLive.Domain");
|
||||
_.Assembly("BirdsiteLive.DAL");
|
||||
_.Assembly("BirdsiteLive.DAL.Postgres");
|
||||
_.Assembly("BirdsiteLive.Moderation");
|
||||
|
||||
_.TheCallingAssembly();
|
||||
|
||||
_.WithDefaultConventions();
|
||||
|
||||
_.LookForRegistries();
|
||||
});
|
||||
});
|
||||
return container;
|
||||
}
|
||||
|
||||
public class DummyLogger<T> : ILogger<T>
|
||||
{
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
|
||||
{
|
||||
}
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
public IDisposable BeginScope<TState>(TState state)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,81 @@
|
|||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BSLManager.Domain
|
||||
{
|
||||
public class FollowersListState
|
||||
{
|
||||
private readonly List<string> _filteredDisplayableUserList = new List<string>();
|
||||
|
||||
private List<Follower> _sourceUserList = new List<Follower>();
|
||||
private List<Follower> _filteredSourceUserList = new List<Follower>();
|
||||
|
||||
public void Load(List<Follower> followers)
|
||||
{
|
||||
_sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
|
||||
|
||||
ResetLists();
|
||||
}
|
||||
|
||||
private void ResetLists()
|
||||
{
|
||||
_filteredSourceUserList = _sourceUserList.ToList();
|
||||
|
||||
_filteredDisplayableUserList.Clear();
|
||||
|
||||
foreach (var follower in _sourceUserList)
|
||||
{
|
||||
var displayedUser = $"{GetFullHandle(follower)} ({follower.Followings.Count}) (err:{follower.PostingErrorCount})";
|
||||
_filteredDisplayableUserList.Add(displayedUser);
|
||||
}
|
||||
}
|
||||
|
||||
public List<string> GetDisplayableList()
|
||||
{
|
||||
return _filteredDisplayableUserList;
|
||||
}
|
||||
|
||||
public void FilterBy(string pattern)
|
||||
{
|
||||
ResetLists();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(pattern))
|
||||
{
|
||||
var elToRemove = _filteredSourceUserList
|
||||
.Where(x => !GetFullHandle(x).Contains(pattern))
|
||||
.Select(x => x)
|
||||
.ToList();
|
||||
|
||||
foreach (var el in elToRemove)
|
||||
{
|
||||
_filteredSourceUserList.Remove(el);
|
||||
|
||||
var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
|
||||
_filteredDisplayableUserList.Remove(dElToRemove);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string GetFullHandle(Follower follower)
|
||||
{
|
||||
return $"@{follower.Acct}@{follower.Host}";
|
||||
}
|
||||
|
||||
public void RemoveAt(int index)
|
||||
{
|
||||
var displayableUser = _filteredDisplayableUserList[index];
|
||||
var sourceUser = _filteredSourceUserList[index];
|
||||
|
||||
_filteredDisplayableUserList.Remove(displayableUser);
|
||||
|
||||
_filteredSourceUserList.Remove(sourceUser);
|
||||
_sourceUserList.Remove(sourceUser);
|
||||
}
|
||||
|
||||
public Follower GetElementAt(int index)
|
||||
{
|
||||
return _filteredSourceUserList[index];
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BSLManager.Tools;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using NStack;
|
||||
using Terminal.Gui;
|
||||
|
||||
namespace BSLManager
|
||||
{
|
||||
class Program
|
||||
{
|
||||
static async Task Main(string[] args)
|
||||
{
|
||||
Console.OutputEncoding = Encoding.Default;
|
||||
|
||||
var settingsManager = new SettingsManager();
|
||||
var settings = settingsManager.GetSettings();
|
||||
|
||||
//var builder = new ConfigurationBuilder()
|
||||
// .AddEnvironmentVariables();
|
||||
//var configuration = builder.Build();
|
||||
|
||||
//var dbSettings = configuration.GetSection("Db").Get<DbSettings>();
|
||||
//var instanceSettings = configuration.GetSection("Instance").Get<InstanceSettings>();
|
||||
|
||||
var bootstrapper = new Bootstrapper(settings.dbSettings, settings.instanceSettings);
|
||||
var container = bootstrapper.Init();
|
||||
|
||||
var app = container.GetInstance<App>();
|
||||
app.Run();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,13 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace BSLManager.Tools
|
||||
{
|
||||
public static class BasicLogger
|
||||
{
|
||||
public static void Log(string log)
|
||||
{
|
||||
File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
using System.Reflection;
|
||||
using Terminal.Gui;
|
||||
|
||||
namespace BSLManager.Tools
|
||||
{
|
||||
public static class ConsoleGui
|
||||
{
|
||||
public static void RefreshUI()
|
||||
{
|
||||
typeof(Application)
|
||||
.GetMethod("TerminalResized", BindingFlags.Static | BindingFlags.NonPublic)
|
||||
.Invoke(null, null);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,124 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using Newtonsoft.Json;
|
||||
using Org.BouncyCastle.Asn1.IsisMtt.X509;
|
||||
|
||||
namespace BSLManager.Tools
|
||||
{
|
||||
public class SettingsManager
|
||||
{
|
||||
private const string LocalFileName = "ManagerSettings.json";
|
||||
|
||||
public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
|
||||
{
|
||||
var localSettingsData = GetLocalSettingsFile();
|
||||
if (localSettingsData != null) return Convert(localSettingsData);
|
||||
|
||||
Console.WriteLine("We need to set up the manager");
|
||||
Console.WriteLine("Please provide the following information as provided in the docker-compose file");
|
||||
|
||||
LocalSettingsData data;
|
||||
do
|
||||
{
|
||||
data = GetDataFromUser();
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("Please check if all is ok:");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Db Host: {data.DbHost}");
|
||||
Console.WriteLine($"Db Name: {data.DbName}");
|
||||
Console.WriteLine($"Db User: {data.DbUser}");
|
||||
Console.WriteLine($"Db Password: {data.DbPassword}");
|
||||
Console.WriteLine($"Instance Domain: {data.InstanceDomain}");
|
||||
Console.WriteLine();
|
||||
|
||||
string resp;
|
||||
do
|
||||
{
|
||||
Console.WriteLine("Is it valid? (yes, no)");
|
||||
resp = Console.ReadLine()?.Trim().ToLowerInvariant();
|
||||
|
||||
if (resp == "n" || resp == "no") data = null;
|
||||
|
||||
} while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
|
||||
|
||||
} while (data == null);
|
||||
|
||||
SaveLocalSettings(data);
|
||||
return Convert(data);
|
||||
}
|
||||
|
||||
private LocalSettingsData GetDataFromUser()
|
||||
{
|
||||
var data = new LocalSettingsData();
|
||||
|
||||
Console.WriteLine("Db Host:");
|
||||
data.DbHost = Console.ReadLine();
|
||||
|
||||
Console.WriteLine("Db Name:");
|
||||
data.DbName = Console.ReadLine();
|
||||
|
||||
Console.WriteLine("Db User:");
|
||||
data.DbUser = Console.ReadLine();
|
||||
|
||||
Console.WriteLine("Db Password:");
|
||||
data.DbPassword = Console.ReadLine();
|
||||
|
||||
Console.WriteLine("Instance Domain:");
|
||||
data.InstanceDomain = Console.ReadLine();
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
|
||||
{
|
||||
var dbSettings = new DbSettings
|
||||
{
|
||||
Type = data.DbType,
|
||||
Host = data.DbHost,
|
||||
Name = data.DbName,
|
||||
User = data.DbUser,
|
||||
Password = data.DbPassword
|
||||
};
|
||||
var instancesSettings = new InstanceSettings
|
||||
{
|
||||
Domain = data.InstanceDomain
|
||||
};
|
||||
return (dbSettings, instancesSettings);
|
||||
}
|
||||
|
||||
private LocalSettingsData GetLocalSettingsFile()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(LocalFileName)) return null;
|
||||
|
||||
var jsonContent = File.ReadAllText(LocalFileName);
|
||||
var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
|
||||
return content;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveLocalSettings(LocalSettingsData data)
|
||||
{
|
||||
var jsonContent = JsonConvert.SerializeObject(data);
|
||||
File.WriteAllText(LocalFileName, jsonContent);
|
||||
}
|
||||
}
|
||||
|
||||
internal class LocalSettingsData
|
||||
{
|
||||
public string DbType { get; set; } = "postgres";
|
||||
public string DbHost { get; set; }
|
||||
public string DbName { get; set; }
|
||||
public string DbUser { get; set; }
|
||||
public string DbPassword { get; set; }
|
||||
|
||||
public string InstanceDomain { get; set; }
|
||||
}
|
||||
}
|
|
@ -1,4 +1,5 @@
|
|||
using System;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace BirdsiteLive.ActivityPub
|
||||
|
@ -19,6 +20,8 @@ namespace BirdsiteLive.ActivityPub
|
|||
if(a.apObject.type == "Follow")
|
||||
return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
|
||||
break;
|
||||
case "Delete":
|
||||
return JsonConvert.DeserializeObject<ActivityDelete>(json);
|
||||
case "Accept":
|
||||
var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
|
||||
//var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
|
||||
|
@ -41,7 +44,6 @@ namespace BirdsiteLive.ActivityPub
|
|||
}
|
||||
};
|
||||
return acceptFollow;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="System.Text.Json" Version="4.7.2" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
|
||||
<PackageReference Include="System.Text.Json" Version="7.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -0,0 +1,11 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace BirdsiteLive.ActivityPub.Models
|
||||
{
|
||||
public class ActivityDelete : Activity
|
||||
{
|
||||
public string[] to { get; set; }
|
||||
[JsonProperty("object")]
|
||||
public object apObject { get; set; }
|
||||
}
|
||||
}
|
|
@ -17,6 +17,7 @@ namespace BirdsiteLive.ActivityPub
|
|||
public string name { get; set; }
|
||||
public string summary { get; set; }
|
||||
public string url { get; set; }
|
||||
public string movedTo { get; set; }
|
||||
public bool manuallyApprovesFollowers { get; set; }
|
||||
public string inbox { get; set; }
|
||||
public bool? discoverable { get; set; } = true;
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
namespace BirdsiteLive.ActivityPub.Models
|
||||
{
|
||||
public class Tombstone
|
||||
{
|
||||
public string id { get; set; }
|
||||
public readonly string type = "Tombstone";
|
||||
public string atomUrl { get; set; }
|
||||
}
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BirdsiteLive.Common.Regexes
|
||||
{
|
||||
public class HeaderRegexes
|
||||
{
|
||||
public static readonly Regex HeaderSignature = new Regex(@"^([a-zA-Z0-9]+)=""(.+)""$");
|
||||
}
|
||||
}
|
|
@ -4,6 +4,8 @@ namespace BirdsiteLive.Common.Regexes
|
|||
{
|
||||
public class UrlRegexes
|
||||
{
|
||||
public static readonly Regex Url = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
|
||||
public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
|
||||
|
||||
public static readonly Regex Domain = new Regex(@"^[a-zA-Z0-9\-_]+(\.[a-zA-Z0-9\-_]+)+$");
|
||||
}
|
||||
}
|
|
@ -8,5 +8,17 @@
|
|||
public bool ResolveMentionsInProfiles { get; set; }
|
||||
public bool PublishReplies { get; set; }
|
||||
public int MaxUsersCapacity { get; set; }
|
||||
|
||||
public string UnlistedTwitterAccounts { get; set; }
|
||||
public string SensitiveTwitterAccounts { get; set; }
|
||||
|
||||
public int FailingTwitterUserCleanUpThreshold { get; set; }
|
||||
public int FailingFollowerCleanUpThreshold { get; set; } = -1;
|
||||
|
||||
public int UserCacheCapacity { get; set; }
|
||||
public string IpWhiteListing { get; set; }
|
||||
public bool EnableXRealIpHeader { get; set; }
|
||||
|
||||
public int MaxTweetRetention { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
namespace BirdsiteLive.Common.Settings
|
||||
{
|
||||
public class ModerationSettings
|
||||
{
|
||||
public string FollowersWhiteListing { get; set; }
|
||||
public string FollowersBlackListing { get; set; }
|
||||
public string TwitterAccountsWhiteListing { get; set; }
|
||||
public string TwitterAccountsBlackListing { get; set; }
|
||||
}
|
||||
}
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Asn1" Version="1.0.9" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
|
||||
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -9,17 +9,27 @@ using BirdsiteLive.ActivityPub;
|
|||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Org.BouncyCastle.Bcpg;
|
||||
|
||||
namespace BirdsiteLive.Domain
|
||||
{
|
||||
public interface IActivityPubService
|
||||
{
|
||||
Task<string> GetUserIdAsync(string acct);
|
||||
Task<Actor> GetUser(string objectId);
|
||||
Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
|
||||
Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
|
||||
string targetInbox);
|
||||
Task DeleteUserAsync(string username, string targetHost, string targetInbox);
|
||||
Task DeleteNoteAsync(SyncTweet tweet);
|
||||
}
|
||||
|
||||
public class WebFinger
|
||||
{
|
||||
public string subject { get; set; }
|
||||
public string[] aliases { get; set; }
|
||||
}
|
||||
|
||||
public class ActivityPubService : IActivityPubService
|
||||
|
@ -27,47 +37,133 @@ namespace BirdsiteLive.Domain
|
|||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly ICryptoService _cryptoService;
|
||||
private readonly ILogger<ActivityPubService> _logger;
|
||||
|
||||
#region Ctor
|
||||
public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings, IHttpClientFactory httpClientFactory)
|
||||
public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings, IHttpClientFactory httpClientFactory, ILogger<ActivityPubService> logger)
|
||||
{
|
||||
_cryptoService = cryptoService;
|
||||
_instanceSettings = instanceSettings;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<string> GetUserIdAsync(string acct)
|
||||
{
|
||||
var splittedAcct = acct.Trim('@').Split('@');
|
||||
|
||||
var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}";
|
||||
|
||||
var httpClient = _httpClientFactory.CreateClient();
|
||||
httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
var result = await httpClient.GetAsync(url);
|
||||
|
||||
result.EnsureSuccessStatusCode();
|
||||
|
||||
var content = await result.Content.ReadAsStringAsync();
|
||||
|
||||
var actor = JsonConvert.DeserializeObject<WebFinger>(content);
|
||||
return actor.aliases.FirstOrDefault();
|
||||
}
|
||||
|
||||
public async Task<Actor> GetUser(string objectId)
|
||||
{
|
||||
var httpClient = _httpClientFactory.CreateClient();
|
||||
httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json");
|
||||
var result = await httpClient.GetAsync(objectId);
|
||||
|
||||
if (result.StatusCode == HttpStatusCode.Gone)
|
||||
throw new FollowerIsGoneException();
|
||||
|
||||
result.EnsureSuccessStatusCode();
|
||||
|
||||
var content = await result.Content.ReadAsStringAsync();
|
||||
return JsonConvert.DeserializeObject<Actor>(content);
|
||||
|
||||
var actor = JsonConvert.DeserializeObject<Actor>(content);
|
||||
if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
|
||||
return actor;
|
||||
}
|
||||
|
||||
public async Task DeleteUserAsync(string username, string targetHost, string targetInbox)
|
||||
{
|
||||
try
|
||||
{
|
||||
var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
|
||||
|
||||
var deleteUser = new ActivityDelete
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{actor}#delete",
|
||||
type = "Delete",
|
||||
actor = actor,
|
||||
to = new [] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
apObject = actor
|
||||
};
|
||||
|
||||
await PostDataAsync(deleteUser, targetHost, actor, targetInbox);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteNoteAsync(SyncTweet tweet)
|
||||
{
|
||||
var acct = tweet.Acct.ToLowerInvariant().Trim();
|
||||
|
||||
var actor = $"https://{_instanceSettings.Domain}/users/{acct}";
|
||||
var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}";
|
||||
|
||||
var delete = new ActivityDelete
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{noteId}#delete",
|
||||
type = "Delete",
|
||||
actor = actor,
|
||||
to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
apObject = new Tombstone
|
||||
{
|
||||
id = noteId,
|
||||
atomUrl = noteId
|
||||
}
|
||||
};
|
||||
|
||||
await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox);
|
||||
}
|
||||
|
||||
public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
|
||||
{
|
||||
var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
|
||||
var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var nowString = now.ToString("s") + "Z";
|
||||
|
||||
var noteActivity = new ActivityCreateNote()
|
||||
try
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{noteUri}/activity",
|
||||
type = "Create",
|
||||
actor = actor,
|
||||
published = nowString,
|
||||
var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
|
||||
var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId);
|
||||
|
||||
to = note.to,
|
||||
cc = note.cc,
|
||||
apObject = note
|
||||
};
|
||||
var now = DateTime.UtcNow;
|
||||
var nowString = now.ToString("s") + "Z";
|
||||
|
||||
await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
|
||||
var noteActivity = new ActivityCreateNote()
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{noteUri}/activity",
|
||||
type = "Create",
|
||||
actor = actor,
|
||||
published = nowString,
|
||||
|
||||
to = note.to,
|
||||
cc = note.cc,
|
||||
apObject = note
|
||||
};
|
||||
|
||||
await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error sending {Username} post ({NoteId}) to {Host}{Inbox}", username, noteId, targetHost, targetInbox);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
@ -15,4 +15,8 @@
|
|||
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Enum\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -0,0 +1,61 @@
|
|||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Domain.BusinessUseCases
|
||||
{
|
||||
public interface IProcessDeleteUser
|
||||
{
|
||||
Task ExecuteAsync(Follower follower);
|
||||
Task ExecuteAsync(string followerUsername, string followerDomain);
|
||||
Task ExecuteAsync(string actorId);
|
||||
}
|
||||
|
||||
public class ProcessDeleteUser : IProcessDeleteUser
|
||||
{
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
|
||||
#region Ctor
|
||||
public ProcessDeleteUser(IFollowersDal followersDal, ITwitterUserDal twitterUserDal)
|
||||
{
|
||||
_followersDal = followersDal;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ExecuteAsync(string followerUsername, string followerDomain)
|
||||
{
|
||||
// Get Follower and Twitter Users
|
||||
var follower = await _followersDal.GetFollowerAsync(followerUsername, followerDomain);
|
||||
if (follower == null) return;
|
||||
|
||||
await ExecuteAsync(follower);
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(string actorId)
|
||||
{
|
||||
// Get Follower and Twitter Users
|
||||
var follower = await _followersDal.GetFollowerAsync(actorId);
|
||||
if (follower == null) return;
|
||||
|
||||
await ExecuteAsync(follower);
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Follower follower)
|
||||
{
|
||||
// Remove twitter users if no more followers
|
||||
var followings = follower.Followings;
|
||||
foreach (var following in followings)
|
||||
{
|
||||
var followers = await _followersDal.GetFollowersAsync(following);
|
||||
if (followers.Length == 1 && followers.First().Id == follower.Id)
|
||||
await _twitterUserDal.DeleteTwitterUserAsync(following);
|
||||
}
|
||||
|
||||
// Remove follower from DB
|
||||
await _followersDal.DeleteFollowerAsync(follower.Id);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -5,7 +5,7 @@ namespace BirdsiteLive.Domain.BusinessUseCases
|
|||
{
|
||||
public interface IProcessFollowUser
|
||||
{
|
||||
Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox);
|
||||
Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox, string followerActorId);
|
||||
}
|
||||
|
||||
public class ProcessFollowUser : IProcessFollowUser
|
||||
|
@ -21,13 +21,13 @@ namespace BirdsiteLive.Domain.BusinessUseCases
|
|||
}
|
||||
#endregion
|
||||
|
||||
public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox)
|
||||
public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox, string followerActorId)
|
||||
{
|
||||
// Get Follower and Twitter Users
|
||||
var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
|
||||
if (follower == null)
|
||||
{
|
||||
await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox, sharedInbox);
|
||||
await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox, sharedInbox, followerActorId);
|
||||
follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
namespace BirdsiteLive.Domain.Enum
|
||||
{
|
||||
public enum MigrationTypeEnum
|
||||
{
|
||||
Unknown = 0,
|
||||
Migration = 1,
|
||||
Deletion = 2
|
||||
}
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
using System;
|
||||
|
||||
namespace BirdsiteLive.Domain
|
||||
{
|
||||
public class FollowerIsGoneException : Exception
|
||||
{
|
||||
}
|
||||
}
|
|
@ -0,0 +1,352 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using BirdsiteLive.Twitter;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain.Enum;
|
||||
using System.Net.Http;
|
||||
using BirdsiteLive.Common.Regexes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Domain
|
||||
{
|
||||
public class MigrationService
|
||||
{
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly ITheFedInfoService _theFedInfoService;
|
||||
private readonly ITwitterTweetsService _twitterTweetsService;
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly ILogger<MigrationService> _logger;
|
||||
|
||||
#region Ctor
|
||||
public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ITheFedInfoService theFedInfoService, IHttpClientFactory httpClientFactory, ILogger<MigrationService> logger)
|
||||
{
|
||||
_twitterTweetsService = twitterTweetsService;
|
||||
_activityPubService = activityPubService;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_followersDal = followersDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
_theFedInfoService = theFedInfoService;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public string GetMigrationCode(string acct)
|
||||
{
|
||||
var hash = GetHashString(acct);
|
||||
return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]";
|
||||
}
|
||||
|
||||
public string GetDeletionCode(string acct)
|
||||
{
|
||||
var hash = GetHashString(acct);
|
||||
return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]";
|
||||
}
|
||||
|
||||
public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type)
|
||||
{
|
||||
string code;
|
||||
if (type == MigrationTypeEnum.Migration)
|
||||
code = GetMigrationCode(acct);
|
||||
else if (type == MigrationTypeEnum.Deletion)
|
||||
code = GetDeletionCode(acct);
|
||||
else
|
||||
throw new NotImplementedException();
|
||||
|
||||
var castedTweetId = ExtractedTweetId(tweetId);
|
||||
var tweet = _twitterTweetsService.GetTweet(castedTweetId);
|
||||
|
||||
if (tweet == null)
|
||||
throw new Exception("Tweet not found");
|
||||
|
||||
if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant())
|
||||
throw new Exception($"Tweet not published by @{acct}");
|
||||
|
||||
if (!tweet.MessageContent.Contains(code))
|
||||
{
|
||||
var message = "Tweet don't have migration code";
|
||||
if (type == MigrationTypeEnum.Deletion)
|
||||
message = "Tweet don't have deletion code";
|
||||
|
||||
throw new Exception(message);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private long ExtractedTweetId(string tweetId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tweetId))
|
||||
throw new ArgumentException("No provided Tweet ID");
|
||||
|
||||
long castedId;
|
||||
if (long.TryParse(tweetId, out castedId))
|
||||
return castedId;
|
||||
|
||||
var urlPart = tweetId.Split('/').LastOrDefault();
|
||||
if (long.TryParse(urlPart, out castedId))
|
||||
return castedId;
|
||||
|
||||
throw new ArgumentException("Unvalid Tweet ID");
|
||||
}
|
||||
|
||||
public async Task<ValidatedFediverseUser> ValidateFediverseAcctAsync(string fediverseAcct)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(fediverseAcct))
|
||||
throw new ArgumentException("Please provide Fediverse account");
|
||||
|
||||
if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2)
|
||||
throw new ArgumentException("Please provide valid Fediverse handle");
|
||||
|
||||
var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct);
|
||||
var user = await _activityPubService.GetUser(objectId);
|
||||
|
||||
var result = new ValidatedFediverseUser
|
||||
{
|
||||
FediverseAcct = fediverseAcct,
|
||||
ObjectId = objectId,
|
||||
User = user,
|
||||
IsValid = user != null
|
||||
};
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct)
|
||||
{
|
||||
// Apply moved to
|
||||
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
|
||||
if (twitterAccount == null)
|
||||
{
|
||||
await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct);
|
||||
twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
|
||||
}
|
||||
|
||||
twitterAccount.MovedTo = validatedUser.User.id;
|
||||
twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
|
||||
twitterAccount.LastSync = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
|
||||
|
||||
// Notify Followers
|
||||
var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been disabled by its original owner.<br/>It has been redirected to {validatedUser.FediverseAcct}.</p>";
|
||||
NotifyFollowers(acct, twitterAccount, message);
|
||||
}
|
||||
|
||||
private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message)
|
||||
{
|
||||
var t = Task.Run(async () =>
|
||||
{
|
||||
var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id);
|
||||
foreach (var follower in followers)
|
||||
{
|
||||
try
|
||||
{
|
||||
var noteId = Guid.NewGuid().ToString();
|
||||
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct);
|
||||
var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId);
|
||||
|
||||
//var to = validatedUser.ObjectId;
|
||||
var to = follower.ActorId;
|
||||
var cc = new string[0];
|
||||
|
||||
var note = new Note
|
||||
{
|
||||
id = noteUrl,
|
||||
|
||||
published = DateTime.UtcNow.ToString("s") + "Z",
|
||||
url = noteUrl,
|
||||
attributedTo = actorUrl,
|
||||
|
||||
to = new[] { to },
|
||||
cc = cc,
|
||||
|
||||
content = message,
|
||||
tag = new Tag[]{
|
||||
new Tag()
|
||||
{
|
||||
type = "Mention",
|
||||
href = follower.ActorId,
|
||||
name = $"@{follower.Acct}@{follower.Host}"
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute))
|
||||
await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute);
|
||||
else
|
||||
await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async Task DeleteAccountAsync(string acct)
|
||||
{
|
||||
// Apply deleted state
|
||||
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
|
||||
if (twitterAccount == null)
|
||||
{
|
||||
await _twitterUserDal.CreateTwitterUserAsync(acct, -1);
|
||||
twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
|
||||
}
|
||||
|
||||
twitterAccount.Deleted = true;
|
||||
twitterAccount.LastSync = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
|
||||
|
||||
// Notify Followers
|
||||
var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been deleted by its original owner.<br/></p>";
|
||||
NotifyFollowers(acct, twitterAccount, message);
|
||||
|
||||
// Delete remote accounts
|
||||
DeleteRemoteAccounts(acct);
|
||||
}
|
||||
|
||||
private void DeleteRemoteAccounts(string acct)
|
||||
{
|
||||
var t = Task.Run(async () =>
|
||||
{
|
||||
var allUsers = await _followersDal.GetAllFollowersAsync();
|
||||
|
||||
var followersWtSharedInbox = allUsers
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
|
||||
.GroupBy(x => x.Host)
|
||||
.ToList();
|
||||
foreach (var followerGroup in followersWtSharedInbox)
|
||||
{
|
||||
var host = followerGroup.First().Host;
|
||||
var sharedInbox = followerGroup.First().SharedInboxRoute;
|
||||
|
||||
var t1 = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
var followerWtInbox = allUsers
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
|
||||
.ToList();
|
||||
foreach (var followerGroup in followerWtInbox)
|
||||
{
|
||||
var host = followerGroup.Host;
|
||||
var sharedInbox = followerGroup.InboxRoute;
|
||||
|
||||
var t1 = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async Task TriggerRemoteMigrationAsync(string id, string tweetIdStg, string handle)
|
||||
{
|
||||
var url = $"https://{{0}}/migration/move/{{1}}/{{2}}/{handle}";
|
||||
await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
|
||||
|
||||
}
|
||||
|
||||
public async Task TriggerRemoteDeleteAsync(string id, string tweetIdStg)
|
||||
{
|
||||
var url = $"https://{{0}}/migration/delete/{{1}}/{{2}}";
|
||||
await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
|
||||
}
|
||||
|
||||
private async Task ProcessRemoteMigrationAsync(string id, string tweetIdStg, string urlPattern)
|
||||
{
|
||||
try
|
||||
{
|
||||
var instances = await RetrieveCompatibleBslInstancesAsync();
|
||||
var tweetId = ExtractedTweetId(tweetIdStg);
|
||||
|
||||
foreach (var instance in instances)
|
||||
{
|
||||
try
|
||||
{
|
||||
var host = instance.Host;
|
||||
if(!UrlRegexes.Domain.IsMatch(host)) continue;
|
||||
|
||||
var url = string.Format(urlPattern, host, id, tweetId);
|
||||
|
||||
var client = _httpClientFactory.CreateClient();
|
||||
var result = await client.PostAsync(url, null);
|
||||
result.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<List<BslInstanceInfo>> RetrieveCompatibleBslInstancesAsync()
|
||||
{
|
||||
var instances = await _theFedInfoService.GetBslInstanceListAsync();
|
||||
var filteredInstances = instances
|
||||
.Where(x => x.Version >= new Version(0, 21, 0))
|
||||
.Where(x => string.Compare(x.Host,
|
||||
_instanceSettings.Domain,
|
||||
StringComparison.InvariantCultureIgnoreCase) != 0)
|
||||
.ToList();
|
||||
return filteredInstances;
|
||||
}
|
||||
|
||||
private byte[] GetHash(string inputString)
|
||||
{
|
||||
using (HashAlgorithm algorithm = SHA256.Create())
|
||||
return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
|
||||
}
|
||||
|
||||
private string GetHashString(string inputString)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
foreach (byte b in GetHash(inputString))
|
||||
sb.Append(b.ToString("X2"));
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public class ValidatedFediverseUser
|
||||
{
|
||||
public string FediverseAcct { get; set; }
|
||||
public string ObjectId { get; set; }
|
||||
public Actor User { get; set; }
|
||||
public bool IsValid { get; set; }
|
||||
}
|
||||
}
|
|
@ -0,0 +1,148 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Domain.Tools;
|
||||
|
||||
namespace BirdsiteLive.Domain.Repository
|
||||
{
|
||||
public interface IModerationRepository
|
||||
{
|
||||
ModerationTypeEnum GetModerationType(ModerationEntityTypeEnum type);
|
||||
ModeratedTypeEnum CheckStatus(ModerationEntityTypeEnum type, string entity);
|
||||
}
|
||||
|
||||
public class ModerationRepository : IModerationRepository
|
||||
{
|
||||
private readonly Regex[] _followersWhiteListing;
|
||||
private readonly Regex[] _followersBlackListing;
|
||||
private readonly Regex[] _twitterAccountsWhiteListing;
|
||||
private readonly Regex[] _twitterAccountsBlackListing;
|
||||
|
||||
private readonly Dictionary<ModerationEntityTypeEnum, ModerationTypeEnum> _modMode =
|
||||
new Dictionary<ModerationEntityTypeEnum, ModerationTypeEnum>();
|
||||
|
||||
#region Ctor
|
||||
public ModerationRepository(ModerationSettings settings)
|
||||
{
|
||||
var parsedFollowersWhiteListing = PatternsParser.Parse(settings.FollowersWhiteListing);
|
||||
var parsedFollowersBlackListing = PatternsParser.Parse(settings.FollowersBlackListing);
|
||||
var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(settings.TwitterAccountsWhiteListing);
|
||||
var parsedTwitterAccountsBlackListing = PatternsParser.Parse(settings.TwitterAccountsBlackListing);
|
||||
|
||||
_followersWhiteListing = parsedFollowersWhiteListing
|
||||
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.Follower, x))
|
||||
.ToArray();
|
||||
_followersBlackListing = parsedFollowersBlackListing
|
||||
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.Follower, x))
|
||||
.ToArray();
|
||||
_twitterAccountsWhiteListing = parsedTwitterAccountsWhiteListing
|
||||
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x))
|
||||
.ToArray();
|
||||
_twitterAccountsBlackListing = parsedTwitterAccountsBlackListing
|
||||
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x))
|
||||
.ToArray();
|
||||
|
||||
// Set Follower moderation politic
|
||||
if (_followersWhiteListing.Any())
|
||||
_modMode.Add(ModerationEntityTypeEnum.Follower, ModerationTypeEnum.WhiteListing);
|
||||
else if (_followersBlackListing.Any())
|
||||
_modMode.Add(ModerationEntityTypeEnum.Follower, ModerationTypeEnum.BlackListing);
|
||||
else
|
||||
_modMode.Add(ModerationEntityTypeEnum.Follower, ModerationTypeEnum.None);
|
||||
|
||||
// Set Twitter account moderation politic
|
||||
if (_twitterAccountsWhiteListing.Any())
|
||||
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.WhiteListing);
|
||||
else if (_twitterAccountsBlackListing.Any())
|
||||
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.BlackListing);
|
||||
else
|
||||
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.None);
|
||||
}
|
||||
#endregion
|
||||
|
||||
public ModerationTypeEnum GetModerationType(ModerationEntityTypeEnum type)
|
||||
{
|
||||
return _modMode[type];
|
||||
}
|
||||
|
||||
public ModeratedTypeEnum CheckStatus(ModerationEntityTypeEnum type, string entity)
|
||||
{
|
||||
if (_modMode[type] == ModerationTypeEnum.None) return ModeratedTypeEnum.None;
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case ModerationEntityTypeEnum.Follower:
|
||||
return ProcessFollower(entity);
|
||||
case ModerationEntityTypeEnum.TwitterAccount:
|
||||
return ProcessTwitterAccount(entity);
|
||||
}
|
||||
|
||||
throw new NotImplementedException($"Type {type} is not supported");
|
||||
}
|
||||
|
||||
private ModeratedTypeEnum ProcessFollower(string entity)
|
||||
{
|
||||
var politic = _modMode[ModerationEntityTypeEnum.Follower];
|
||||
|
||||
switch (politic)
|
||||
{
|
||||
case ModerationTypeEnum.None:
|
||||
return ModeratedTypeEnum.None;
|
||||
case ModerationTypeEnum.BlackListing:
|
||||
if (_followersBlackListing.Any(x => x.IsMatch(entity)))
|
||||
return ModeratedTypeEnum.BlackListed;
|
||||
return ModeratedTypeEnum.None;
|
||||
case ModerationTypeEnum.WhiteListing:
|
||||
if (_followersWhiteListing.Any(x => x.IsMatch(entity)))
|
||||
return ModeratedTypeEnum.WhiteListed;
|
||||
return ModeratedTypeEnum.None;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException();
|
||||
}
|
||||
}
|
||||
|
||||
private ModeratedTypeEnum ProcessTwitterAccount(string entity)
|
||||
{
|
||||
var politic = _modMode[ModerationEntityTypeEnum.TwitterAccount];
|
||||
|
||||
switch (politic)
|
||||
{
|
||||
case ModerationTypeEnum.None:
|
||||
return ModeratedTypeEnum.None;
|
||||
case ModerationTypeEnum.BlackListing:
|
||||
if (_twitterAccountsBlackListing.Any(x => x.IsMatch(entity)))
|
||||
return ModeratedTypeEnum.BlackListed;
|
||||
return ModeratedTypeEnum.None;
|
||||
case ModerationTypeEnum.WhiteListing:
|
||||
if (_twitterAccountsWhiteListing.Any(x => x.IsMatch(entity)))
|
||||
return ModeratedTypeEnum.WhiteListed;
|
||||
return ModeratedTypeEnum.None;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum ModerationEntityTypeEnum
|
||||
{
|
||||
Unknown = 0,
|
||||
Follower = 1,
|
||||
TwitterAccount = 2
|
||||
}
|
||||
|
||||
public enum ModerationTypeEnum
|
||||
{
|
||||
None = 0,
|
||||
BlackListing = 1,
|
||||
WhiteListing = 2
|
||||
}
|
||||
|
||||
public enum ModeratedTypeEnum
|
||||
{
|
||||
None = 0,
|
||||
BlackListed = 1,
|
||||
WhiteListed = 2
|
||||
}
|
||||
}
|
|
@ -0,0 +1,40 @@
|
|||
using System.Linq;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Domain.Tools;
|
||||
|
||||
namespace BirdsiteLive.Domain.Repository
|
||||
{
|
||||
public interface IPublicationRepository
|
||||
{
|
||||
bool IsUnlisted(string twitterAcct);
|
||||
bool IsSensitive(string twitterAcct);
|
||||
}
|
||||
|
||||
public class PublicationRepository : IPublicationRepository
|
||||
{
|
||||
private readonly string[] _unlistedAccounts;
|
||||
private readonly string[] _sensitiveAccounts;
|
||||
|
||||
#region Ctor
|
||||
public PublicationRepository(InstanceSettings settings)
|
||||
{
|
||||
_unlistedAccounts = PatternsParser.Parse(settings.UnlistedTwitterAccounts);
|
||||
_sensitiveAccounts = PatternsParser.Parse(settings.SensitiveTwitterAccounts);
|
||||
}
|
||||
#endregion
|
||||
|
||||
public bool IsUnlisted(string twitterAcct)
|
||||
{
|
||||
if (_unlistedAccounts == null || !_unlistedAccounts.Any()) return false;
|
||||
|
||||
return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
|
||||
}
|
||||
|
||||
public bool IsSensitive(string twitterAcct)
|
||||
{
|
||||
if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
|
||||
|
||||
return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
|
||||
}
|
||||
}
|
||||
}
|
|
@ -7,6 +7,7 @@ using BirdsiteLive.ActivityPub;
|
|||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Domain.Statistics;
|
||||
using BirdsiteLive.Domain.Tools;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
|
@ -25,13 +26,15 @@ namespace BirdsiteLive.Domain
|
|||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly IStatusExtractor _statusExtractor;
|
||||
private readonly IExtractionStatisticsHandler _statisticsHandler;
|
||||
|
||||
private readonly IPublicationRepository _publicationRepository;
|
||||
|
||||
#region Ctor
|
||||
public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
|
||||
public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, IPublicationRepository publicationRepository)
|
||||
{
|
||||
_instanceSettings = instanceSettings;
|
||||
_statusExtractor = statusExtractor;
|
||||
_statisticsHandler = statisticsHandler;
|
||||
_publicationRepository = publicationRepository;
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
@ -41,7 +44,16 @@ namespace BirdsiteLive.Domain
|
|||
var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
|
||||
|
||||
var to = $"{actorUrl}/followers";
|
||||
var apPublic = "https://www.w3.org/ns/activitystreams#Public";
|
||||
|
||||
var isUnlisted = _publicationRepository.IsUnlisted(username);
|
||||
var cc = new string[0];
|
||||
if (isUnlisted)
|
||||
cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
|
||||
|
||||
string summary = null;
|
||||
var sensitive = _publicationRepository.IsSensitive(username);
|
||||
if (sensitive)
|
||||
summary = "Potential Content Warning";
|
||||
|
||||
var extractedTags = _statusExtractor.Extract(tweet.MessageContent);
|
||||
_statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
|
||||
|
@ -70,14 +82,12 @@ namespace BirdsiteLive.Domain
|
|||
attributedTo = actorUrl,
|
||||
|
||||
inReplyTo = inReplyTo,
|
||||
//to = new [] {to},
|
||||
//cc = new [] { apPublic },
|
||||
|
||||
to = new[] { to },
|
||||
//cc = new[] { apPublic },
|
||||
cc = new string[0],
|
||||
cc = cc,
|
||||
|
||||
sensitive = false,
|
||||
sensitive = sensitive,
|
||||
summary = summary,
|
||||
content = $"<p>{content}</p>",
|
||||
attachment = Convert(tweet.Media),
|
||||
tag = extractedTags.tags
|
||||
|
@ -100,4 +110,4 @@ namespace BirdsiteLive.Domain
|
|||
}).ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,162 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace BirdsiteLive.Domain
|
||||
{
|
||||
public interface ITheFedInfoService
|
||||
{
|
||||
Task<List<BslInstanceInfo>> GetBslInstanceListAsync();
|
||||
}
|
||||
|
||||
public class TheFedInfoService : ITheFedInfoService
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
|
||||
#region Ctor
|
||||
public TheFedInfoService(IHttpClientFactory httpClientFactory)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<List<BslInstanceInfo>> GetBslInstanceListAsync()
|
||||
{
|
||||
var cancellationToken = CancellationToken.None;
|
||||
|
||||
var result = await CallGraphQlAsync<MyResponseData>(
|
||||
new Uri("https://the-federation.info/graphql"),
|
||||
HttpMethod.Get,
|
||||
"query ($platform: String!) { nodes(platform: $platform) { host, version } }",
|
||||
new
|
||||
{
|
||||
platform = "birdsitelive",
|
||||
},
|
||||
cancellationToken);
|
||||
|
||||
var convertedResults = ConvertResults(result);
|
||||
return convertedResults;
|
||||
}
|
||||
|
||||
private List<BslInstanceInfo> ConvertResults(GraphQLResponse<MyResponseData> qlData)
|
||||
{
|
||||
var results = new List<BslInstanceInfo>();
|
||||
|
||||
foreach (var instanceInfo in qlData.Data.Nodes)
|
||||
{
|
||||
try
|
||||
{
|
||||
var rawVersion = instanceInfo.Version.Split('+').First();
|
||||
if (string.IsNullOrWhiteSpace(rawVersion)) continue;
|
||||
var version = Version.Parse(rawVersion);
|
||||
if(version <= new Version(0,1,0)) continue;
|
||||
|
||||
var instance = new BslInstanceInfo
|
||||
{
|
||||
Host = instanceInfo.Host,
|
||||
Version = version
|
||||
};
|
||||
results.Add(instance);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private async Task<GraphQLResponse<TResponse>> CallGraphQlAsync<TResponse>(Uri endpoint, HttpMethod method, string query, object variables, CancellationToken cancellationToken)
|
||||
{
|
||||
var content = new StringContent(SerializeGraphQlCall(query, variables), Encoding.UTF8, "application/json");
|
||||
var httpRequestMessage = new HttpRequestMessage
|
||||
{
|
||||
Method = method,
|
||||
Content = content,
|
||||
RequestUri = endpoint,
|
||||
};
|
||||
//add authorization headers if necessary here
|
||||
httpRequestMessage.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
var httpClient = _httpClientFactory.CreateClient();
|
||||
using (var response = await httpClient.SendAsync(httpRequestMessage, cancellationToken))
|
||||
{
|
||||
//if (response.IsSuccessStatusCode)
|
||||
if (response?.Content.Headers.ContentType?.MediaType == "application/json")
|
||||
{
|
||||
var responseString = await response.Content.ReadAsStringAsync(); //cancellationToken supported for .NET 5/6
|
||||
return DeserializeGraphQlCall<TResponse>(responseString);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new ApplicationException($"Unable to contact '{endpoint}': {response.StatusCode} - {response.ReasonPhrase}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string SerializeGraphQlCall(string query, object variables)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var textWriter = new StringWriter(sb);
|
||||
var serializer = new JsonSerializer();
|
||||
serializer.Serialize(textWriter, new
|
||||
{
|
||||
query = query,
|
||||
variables = variables,
|
||||
});
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private GraphQLResponse<TResponse> DeserializeGraphQlCall<TResponse>(string response)
|
||||
{
|
||||
var serializer = new JsonSerializer();
|
||||
var stringReader = new StringReader(response);
|
||||
var jsonReader = new JsonTextReader(stringReader);
|
||||
var result = serializer.Deserialize<GraphQLResponse<TResponse>>(jsonReader);
|
||||
return result;
|
||||
}
|
||||
|
||||
private class GraphQLResponse<TResponse>
|
||||
{
|
||||
public List<GraphQLError> Errors { get; set; }
|
||||
public TResponse Data { get; set; }
|
||||
}
|
||||
|
||||
private class GraphQLError
|
||||
{
|
||||
public string Message { get; set; }
|
||||
public List<GraphQLErrorLocation> Locations { get; set; }
|
||||
public List<object> Path { get; set; } //either int or string
|
||||
}
|
||||
|
||||
private class GraphQLErrorLocation
|
||||
{
|
||||
public int Line { get; set; }
|
||||
public int Column { get; set; }
|
||||
}
|
||||
|
||||
private class MyResponseData
|
||||
{
|
||||
public Node[] Nodes { get; set; }
|
||||
}
|
||||
|
||||
private class Node
|
||||
{
|
||||
public string Host { get; set; }
|
||||
public string Version { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
public class BslInstanceInfo
|
||||
{
|
||||
public string Host { get; set; }
|
||||
public Version Version { get; set; }
|
||||
}
|
||||
}
|
|
@ -0,0 +1,28 @@
|
|||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using Org.BouncyCastle.Pkcs;
|
||||
|
||||
namespace BirdsiteLive.Domain.Tools
|
||||
{
|
||||
public class ModerationRegexParser
|
||||
{
|
||||
public static Regex Parse(ModerationEntityTypeEnum type, string data)
|
||||
{
|
||||
data = data.ToLowerInvariant().Trim();
|
||||
|
||||
if (type == ModerationEntityTypeEnum.Follower)
|
||||
{
|
||||
if (data.StartsWith("@"))
|
||||
return new Regex($@"^{data}$");
|
||||
|
||||
if (data.StartsWith("*"))
|
||||
data = data.Replace("*", "(.+)");
|
||||
|
||||
return new Regex($@"^@(.+)@{data}$");
|
||||
}
|
||||
|
||||
return new Regex($@"^{data}$");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace BirdsiteLive.Domain.Tools
|
||||
{
|
||||
public class PatternsParser
|
||||
{
|
||||
public static string[] Parse(string entry)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(entry)) return new string[0];
|
||||
|
||||
var separationChar = '|';
|
||||
if (entry.Contains(";")) separationChar = ';';
|
||||
else if (entry.Contains(",")) separationChar = ',';
|
||||
|
||||
var splitEntries = entry
|
||||
.Split(new[] {separationChar}, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Select(x => x.ToLowerInvariant().Trim());
|
||||
return splitEntries.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
using System.Linq;
|
||||
|
||||
namespace BirdsiteLive.Domain.Tools
|
||||
{
|
||||
public class SigValidationResultExtractor
|
||||
{
|
||||
public static string GetUserName(SignatureValidationResult result)
|
||||
{
|
||||
return result.User.preferredUsername.ToLowerInvariant().Trim();
|
||||
}
|
||||
|
||||
public static string GetHost(SignatureValidationResult result)
|
||||
{
|
||||
return result.User.url.Replace("https://", string.Empty).Split('/').First();
|
||||
}
|
||||
|
||||
public static string GetSharedInbox(SignatureValidationResult result)
|
||||
{
|
||||
return result.User?.endpoints?.sharedInbox;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -44,11 +44,8 @@ namespace BirdsiteLive.Domain.Tools
|
|||
var urlMatch = UrlRegexes.Url.Matches(messageContent);
|
||||
foreach (Match m in urlMatch)
|
||||
{
|
||||
var url = m.ToString().Replace("\n", string.Empty).Trim();
|
||||
|
||||
var protocol = "https://";
|
||||
if (url.StartsWith("http://")) protocol = "http://";
|
||||
else if (url.StartsWith("ftp://")) protocol = "ftp://";
|
||||
var url = m.Groups[2].ToString();
|
||||
var protocol = m.Groups[3].ToString();
|
||||
|
||||
var truncatedUrl = url.Replace(protocol, string.Empty);
|
||||
|
||||
|
@ -67,8 +64,8 @@ namespace BirdsiteLive.Domain.Tools
|
|||
secondPart = truncatedUrl.Substring(30);
|
||||
}
|
||||
|
||||
messageContent = Regex.Replace(messageContent, m.ToString(),
|
||||
$@" <a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
|
||||
messageContent = Regex.Replace(messageContent, Regex.Escape(m.ToString()),
|
||||
$@"{m.Groups[1]}<a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
|
||||
}
|
||||
|
||||
// Extract Hashtags
|
||||
|
@ -84,12 +81,16 @@ namespace BirdsiteLive.Domain.Tools
|
|||
}
|
||||
|
||||
var url = $"https://{_instanceSettings.Domain}/tags/{tag}";
|
||||
tags.Add(new Tag
|
||||
|
||||
if (tags.All(x => x.href != url))
|
||||
{
|
||||
name = $"#{tag}",
|
||||
href = url,
|
||||
type = "Hashtag"
|
||||
});
|
||||
tags.Add(new Tag
|
||||
{
|
||||
name = $"#{tag}",
|
||||
href = url,
|
||||
type = "Hashtag"
|
||||
});
|
||||
}
|
||||
|
||||
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
|
||||
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{m.Groups[3]}");
|
||||
|
@ -99,7 +100,7 @@ namespace BirdsiteLive.Domain.Tools
|
|||
if (extractMentions)
|
||||
{
|
||||
var mentionMatch = OrderByLength(UserRegexes.Mention.Matches(messageContent));
|
||||
foreach (Match m in mentionMatch.OrderByDescending(x => x.Length))
|
||||
foreach (Match m in mentionMatch)
|
||||
{
|
||||
var mention = m.Groups[2].ToString();
|
||||
|
||||
|
@ -112,13 +113,16 @@ namespace BirdsiteLive.Domain.Tools
|
|||
var url = $"https://{_instanceSettings.Domain}/users/{mention}";
|
||||
var name = $"@{mention}@{_instanceSettings.Domain}";
|
||||
|
||||
tags.Add(new Tag
|
||||
if (tags.All(x => x.href != url))
|
||||
{
|
||||
name = name,
|
||||
href = url,
|
||||
type = "Mention"
|
||||
});
|
||||
|
||||
tags.Add(new Tag
|
||||
{
|
||||
name = name,
|
||||
href = url,
|
||||
type = "Mention"
|
||||
});
|
||||
}
|
||||
|
||||
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
|
||||
$@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}");
|
||||
}
|
||||
|
@ -126,13 +130,17 @@ namespace BirdsiteLive.Domain.Tools
|
|||
|
||||
return (messageContent.Trim(), tags.ToArray());
|
||||
}
|
||||
|
||||
|
||||
private IEnumerable<Match> OrderByLength(MatchCollection matches)
|
||||
{
|
||||
var result = new List<Match>();
|
||||
|
||||
foreach (Match m in matches) result.Add(m);
|
||||
result = result.OrderByDescending(x => x.Length).ToList();
|
||||
|
||||
result = result
|
||||
.OrderBy(x => x.Length)
|
||||
.GroupBy(p => p.Value)
|
||||
.Select(g => g.First())
|
||||
.ToList();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
|
@ -7,9 +7,13 @@ using System.Text;
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub;
|
||||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Regexes;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Cryptography;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain.BusinessUseCases;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Domain.Statistics;
|
||||
using BirdsiteLive.Domain.Tools;
|
||||
using BirdsiteLive.Twitter;
|
||||
|
@ -21,13 +25,17 @@ namespace BirdsiteLive.Domain
|
|||
{
|
||||
public interface IUserService
|
||||
{
|
||||
Actor GetUser(TwitterUser twitterUser);
|
||||
Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser);
|
||||
Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
|
||||
Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
|
||||
|
||||
Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost);
|
||||
Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityDelete activity, string body);
|
||||
}
|
||||
|
||||
public class UserService : IUserService
|
||||
{
|
||||
private readonly IProcessDeleteUser _processDeleteUser;
|
||||
private readonly IProcessFollowUser _processFollowUser;
|
||||
private readonly IProcessUndoFollowUser _processUndoFollowUser;
|
||||
|
||||
|
@ -39,8 +47,10 @@ namespace BirdsiteLive.Domain
|
|||
|
||||
private readonly ITwitterUserService _twitterUserService;
|
||||
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
|
||||
#region Ctor
|
||||
public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService)
|
||||
public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IProcessDeleteUser processDeleteUser)
|
||||
{
|
||||
_instanceSettings = instanceSettings;
|
||||
_cryptoService = cryptoService;
|
||||
|
@ -50,10 +60,12 @@ namespace BirdsiteLive.Domain
|
|||
_statusExtractor = statusExtractor;
|
||||
_statisticsHandler = statisticsHandler;
|
||||
_twitterUserService = twitterUserService;
|
||||
_moderationRepository = moderationRepository;
|
||||
_processDeleteUser = processDeleteUser;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public Actor GetUser(TwitterUser twitterUser)
|
||||
public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser)
|
||||
{
|
||||
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct);
|
||||
var acct = twitterUser.Acct.ToLowerInvariant();
|
||||
|
@ -76,9 +88,10 @@ namespace BirdsiteLive.Domain
|
|||
preferredUsername = acct,
|
||||
name = twitterUser.Name,
|
||||
inbox = $"{actorUrl}/inbox",
|
||||
summary = description,
|
||||
summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]<br/><br/>" + description,
|
||||
url = actorUrl,
|
||||
manuallyApprovesFollowers = twitterUser.Protected,
|
||||
discoverable = false,
|
||||
publicKey = new PublicKey()
|
||||
{
|
||||
id = $"{actorUrl}#main-key",
|
||||
|
@ -100,14 +113,27 @@ namespace BirdsiteLive.Domain
|
|||
new UserAttachment
|
||||
{
|
||||
type = "PropertyValue",
|
||||
name = "Official",
|
||||
name = "Official Account",
|
||||
value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
|
||||
},
|
||||
new UserAttachment
|
||||
{
|
||||
type = "PropertyValue",
|
||||
name = "Disclaimer",
|
||||
value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it."
|
||||
},
|
||||
new UserAttachment
|
||||
{
|
||||
type = "PropertyValue",
|
||||
name = "Take control of this account",
|
||||
value = $"<a href=\"https://{_instanceSettings.Domain}/migration/move/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\">MANAGE</a>"
|
||||
}
|
||||
},
|
||||
endpoints = new EndPoints
|
||||
{
|
||||
sharedInbox = $"https://{_instanceSettings.Domain}/inbox"
|
||||
}
|
||||
},
|
||||
movedTo = dbTwitterUser?.MovedTo
|
||||
};
|
||||
return user;
|
||||
}
|
||||
|
@ -118,63 +144,95 @@ namespace BirdsiteLive.Domain
|
|||
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
|
||||
if (!sigValidation.SignatureIsValidated) return false;
|
||||
|
||||
// Save Follow in DB
|
||||
var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant();
|
||||
var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
|
||||
// Prepare data
|
||||
var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
|
||||
var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
|
||||
var followerInbox = sigValidation.User.inbox;
|
||||
var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox;
|
||||
var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty);
|
||||
var followerSharedInbox = SigValidationResultExtractor.GetSharedInbox(sigValidation);
|
||||
var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty).ToLowerInvariant().Trim();
|
||||
|
||||
// Make sure to only keep routes
|
||||
followerInbox = OnlyKeepRoute(followerInbox, followerHost);
|
||||
followerSharedInbox = OnlyKeepRoute(followerSharedInbox, followerHost);
|
||||
|
||||
// Validate Moderation status
|
||||
var followerModPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
|
||||
if (followerModPolicy != ModerationTypeEnum.None)
|
||||
{
|
||||
var followerStatus = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.Follower, $"@{followerUserName}@{followerHost}");
|
||||
|
||||
if(followerModPolicy == ModerationTypeEnum.WhiteListing && followerStatus != ModeratedTypeEnum.WhiteListed ||
|
||||
followerModPolicy == ModerationTypeEnum.BlackListing && followerStatus == ModeratedTypeEnum.BlackListed)
|
||||
return await SendRejectFollowAsync(activity, followerHost);
|
||||
}
|
||||
|
||||
// Validate TwitterAccount status
|
||||
var twitterAccountModPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
|
||||
if (twitterAccountModPolicy != ModerationTypeEnum.None)
|
||||
{
|
||||
var twitterUserStatus = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.TwitterAccount, twitterUser);
|
||||
if (twitterAccountModPolicy == ModerationTypeEnum.WhiteListing && twitterUserStatus != ModeratedTypeEnum.WhiteListed ||
|
||||
twitterAccountModPolicy == ModerationTypeEnum.BlackListing && twitterUserStatus == ModeratedTypeEnum.BlackListed)
|
||||
return await SendRejectFollowAsync(activity, followerHost);
|
||||
}
|
||||
|
||||
// Validate User Protected
|
||||
var user = _twitterUserService.GetUser(twitterUser);
|
||||
if (!user.Protected)
|
||||
{
|
||||
// Execute
|
||||
await _processFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser, followerInbox, followerSharedInbox);
|
||||
await _processFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser, followerInbox, followerSharedInbox, activity.actor);
|
||||
|
||||
// Send Accept Activity
|
||||
var acceptFollow = new ActivityAcceptFollow()
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
|
||||
type = "Accept",
|
||||
actor = activity.apObject,
|
||||
apObject = new ActivityFollow()
|
||||
{
|
||||
id = activity.id,
|
||||
type = activity.type,
|
||||
actor = activity.actor,
|
||||
apObject = activity.apObject
|
||||
}
|
||||
};
|
||||
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
|
||||
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
|
||||
return await SendAcceptFollowAsync(activity, followerHost);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Send Reject Activity
|
||||
var acceptFollow = new ActivityRejectFollow()
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{activity.apObject}#rejects/follows/{Guid.NewGuid()}",
|
||||
type = "Reject",
|
||||
actor = activity.apObject,
|
||||
apObject = new ActivityFollow()
|
||||
{
|
||||
id = activity.id,
|
||||
type = activity.type,
|
||||
actor = activity.actor,
|
||||
apObject = activity.apObject
|
||||
}
|
||||
};
|
||||
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
|
||||
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
|
||||
return await SendRejectFollowAsync(activity, followerHost);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
|
||||
{
|
||||
var acceptFollow = new ActivityAcceptFollow()
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
|
||||
type = "Accept",
|
||||
actor = activity.apObject,
|
||||
apObject = new ActivityFollow()
|
||||
{
|
||||
id = activity.id,
|
||||
type = activity.type,
|
||||
actor = activity.actor,
|
||||
apObject = activity.apObject
|
||||
}
|
||||
};
|
||||
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
|
||||
return result == HttpStatusCode.Accepted ||
|
||||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
|
||||
}
|
||||
|
||||
public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
|
||||
{
|
||||
var acceptFollow = new ActivityRejectFollow()
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{activity.apObject}#rejects/follows/{Guid.NewGuid()}",
|
||||
type = "Reject",
|
||||
actor = activity.apObject,
|
||||
apObject = new ActivityFollow()
|
||||
{
|
||||
id = activity.id,
|
||||
type = activity.type,
|
||||
actor = activity.actor,
|
||||
apObject = activity.apObject
|
||||
}
|
||||
};
|
||||
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
|
||||
return result == HttpStatusCode.Accepted ||
|
||||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
|
||||
}
|
||||
|
||||
private string OnlyKeepRoute(string inbox, string host)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(inbox))
|
||||
|
@ -219,6 +277,28 @@ namespace BirdsiteLive.Domain
|
|||
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders,
|
||||
ActivityDelete activity, string body)
|
||||
{
|
||||
if (activity.apObject is string apObject)
|
||||
{
|
||||
if (!string.Equals(activity.actor.Trim(), apObject.Trim(), StringComparison.InvariantCultureIgnoreCase)) return true;
|
||||
|
||||
try
|
||||
{
|
||||
// Validate
|
||||
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
|
||||
if (!sigValidation.SignatureIsValidated) return false;
|
||||
}
|
||||
catch (FollowerIsGoneException){}
|
||||
|
||||
// Remove user and followings
|
||||
await _processDeleteUser.ExecuteAsync(activity.actor.Trim());
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
|
||||
{
|
||||
//Check Date Validity
|
||||
|
@ -239,33 +319,24 @@ namespace BirdsiteLive.Domain
|
|||
var signature_header = new Dictionary<string, string>();
|
||||
foreach (var signature in signatures)
|
||||
{
|
||||
var splitSig = signature.Replace("\"", string.Empty).Split('=');
|
||||
signature_header.Add(splitSig[0], splitSig[1]);
|
||||
var m = HeaderRegexes.HeaderSignature.Match(signature);
|
||||
signature_header.Add(m.Groups[1].ToString(), m.Groups[2].ToString());
|
||||
}
|
||||
|
||||
signature_header["signature"] = signature_header["signature"] + "==";
|
||||
|
||||
var key_id = signature_header["keyId"];
|
||||
var headers = signature_header["headers"];
|
||||
var algorithm = signature_header["algorithm"];
|
||||
var sig = Convert.FromBase64String(signature_header["signature"]);
|
||||
|
||||
|
||||
// Retrieve User
|
||||
var remoteUser = await _activityPubService.GetUser(actor);
|
||||
|
||||
// Prepare Key data
|
||||
var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
|
||||
toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", "");
|
||||
var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode));
|
||||
|
||||
var toSign = new StringBuilder();
|
||||
//var comparisonString = headers.Split(' ').Select(signed_header_name =>
|
||||
//{
|
||||
// if (signed_header_name == "(request-target)")
|
||||
// return "(request-target): post /inbox";
|
||||
// else
|
||||
// return $"{signed_header_name}: {r.Headers[signed_header_name.ToUpperInvariant()]}";
|
||||
//});
|
||||
|
||||
foreach (var headerKey in headers.Split(' '))
|
||||
{
|
||||
if (headerKey == "(request-target)") toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n");
|
||||
|
@ -273,21 +344,13 @@ namespace BirdsiteLive.Domain
|
|||
}
|
||||
toSign.Remove(toSign.Length - 1, 1);
|
||||
|
||||
//var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode));
|
||||
|
||||
//new RSACryptoServiceProvider(keyId.publicKey.publicKeyPem);
|
||||
|
||||
//Create a new instance of RSACryptoServiceProvider.
|
||||
RSACryptoServiceProvider key = new RSACryptoServiceProvider();
|
||||
|
||||
//Get an instance of RSAParameters from ExportParameters function.
|
||||
RSAParameters RSAKeyInfo = key.ExportParameters(false);
|
||||
|
||||
//Set RSAKeyInfo to the public key values.
|
||||
RSAKeyInfo.Modulus = Convert.FromBase64String(toDecode);
|
||||
|
||||
key.ImportParameters(RSAKeyInfo);
|
||||
// Import key
|
||||
var key = new RSACryptoServiceProvider();
|
||||
var rsaKeyInfo = key.ExportParameters(false);
|
||||
rsaKeyInfo.Modulus = Convert.FromBase64String(toDecode);
|
||||
key.ImportParameters(rsaKeyInfo);
|
||||
|
||||
// Trust and Verify
|
||||
var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
|
||||
|
||||
return new SignatureValidationResult()
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub;
|
||||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
|
||||
namespace BirdsiteLive.Moderation.Actions
|
||||
{
|
||||
public interface IRejectAllFollowingsAction
|
||||
{
|
||||
Task ProcessAsync(Follower follower);
|
||||
}
|
||||
|
||||
public class RejectAllFollowingsAction : IRejectAllFollowingsAction
|
||||
{
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IUserService _userService;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public RejectAllFollowingsAction(ITwitterUserDal twitterUserDal, IUserService userService, InstanceSettings instanceSettings)
|
||||
{
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_userService = userService;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(Follower follower)
|
||||
{
|
||||
foreach (var following in follower.Followings)
|
||||
{
|
||||
try
|
||||
{
|
||||
var f = await _twitterUserDal.GetTwitterUserAsync(following);
|
||||
var activityFollowing = new ActivityFollow
|
||||
{
|
||||
type = "Follow",
|
||||
actor = follower.ActorId,
|
||||
apObject = UrlFactory.GetActorUrl(_instanceSettings.Domain, f.Acct)
|
||||
};
|
||||
|
||||
await _userService.SendRejectFollowAsync(activityFollowing, follower.Host);
|
||||
}
|
||||
catch (Exception) { }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,44 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub;
|
||||
using BirdsiteLive.ActivityPub.Converters;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
|
||||
namespace BirdsiteLive.Moderation.Actions
|
||||
{
|
||||
public interface IRejectFollowingAction
|
||||
{
|
||||
Task ProcessAsync(Follower follower, SyncTwitterUser twitterUser);
|
||||
}
|
||||
|
||||
public class RejectFollowingAction : IRejectFollowingAction
|
||||
{
|
||||
private readonly IUserService _userService;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public RejectFollowingAction(IUserService userService, InstanceSettings instanceSettings)
|
||||
{
|
||||
_userService = userService;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(Follower follower, SyncTwitterUser twitterUser)
|
||||
{
|
||||
try
|
||||
{
|
||||
var activityFollowing = new ActivityFollow
|
||||
{
|
||||
type = "Follow",
|
||||
actor = follower.ActorId,
|
||||
apObject = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct)
|
||||
};
|
||||
await _userService.SendRejectFollowAsync(activityFollowing, follower.Host);
|
||||
}
|
||||
catch (Exception) { }
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,34 @@
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain.BusinessUseCases;
|
||||
|
||||
namespace BirdsiteLive.Moderation.Actions
|
||||
{
|
||||
public interface IRemoveFollowerAction
|
||||
{
|
||||
Task ProcessAsync(Follower follower);
|
||||
}
|
||||
|
||||
public class RemoveFollowerAction : IRemoveFollowerAction
|
||||
{
|
||||
private readonly IRejectAllFollowingsAction _rejectAllFollowingsAction;
|
||||
private readonly IProcessDeleteUser _processDeleteUser;
|
||||
|
||||
#region Ctor
|
||||
public RemoveFollowerAction(IRejectAllFollowingsAction rejectAllFollowingsAction, IProcessDeleteUser processDeleteUser)
|
||||
{
|
||||
_rejectAllFollowingsAction = rejectAllFollowingsAction;
|
||||
_processDeleteUser = processDeleteUser;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(Follower follower)
|
||||
{
|
||||
// Perform undo following to user instance
|
||||
await _rejectAllFollowingsAction.ProcessAsync(follower);
|
||||
|
||||
// Remove twitter users if no more followers
|
||||
await _processDeleteUser.ExecuteAsync(follower);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,57 @@
|
|||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Moderation.Actions
|
||||
{
|
||||
public interface IRemoveTwitterAccountAction
|
||||
{
|
||||
Task ProcessAsync(SyncTwitterUser twitterUser);
|
||||
}
|
||||
|
||||
public class RemoveTwitterAccountAction : IRemoveTwitterAccountAction
|
||||
{
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IRejectFollowingAction _rejectFollowingAction;
|
||||
|
||||
#region Ctor
|
||||
public RemoveTwitterAccountAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectFollowingAction rejectFollowingAction)
|
||||
{
|
||||
_followersDal = followersDal;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_rejectFollowingAction = rejectFollowingAction;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(SyncTwitterUser twitterUser)
|
||||
{
|
||||
// Check Followers
|
||||
var twitterUserId = twitterUser.Id;
|
||||
var followers = await _followersDal.GetFollowersAsync(twitterUserId);
|
||||
|
||||
// Remove all Followers
|
||||
foreach (var follower in followers)
|
||||
{
|
||||
// Perform undo following to user instance
|
||||
await _rejectFollowingAction.ProcessAsync(follower, twitterUser);
|
||||
|
||||
// Remove following from DB
|
||||
if (follower.Followings.Contains(twitterUserId))
|
||||
follower.Followings.Remove(twitterUserId);
|
||||
|
||||
if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
|
||||
follower.FollowingsSyncStatus.Remove(twitterUserId);
|
||||
|
||||
if (follower.Followings.Any())
|
||||
await _followersDal.UpdateFollowerAsync(follower);
|
||||
else
|
||||
await _followersDal.DeleteFollowerAsync(follower.Id);
|
||||
}
|
||||
|
||||
// Remove twitter user
|
||||
await _twitterUserDal.DeleteTwitterUserAsync(twitterUserId);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\BirdsiteLive.Domain\BirdsiteLive.Domain.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Actions\" />
|
||||
<Folder Include="Processors\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
|
@ -0,0 +1,61 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Moderation.Processors;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Moderation
|
||||
{
|
||||
public interface IModerationPipeline
|
||||
{
|
||||
Task ApplyModerationSettingsAsync();
|
||||
}
|
||||
|
||||
public class ModerationPipeline : IModerationPipeline
|
||||
{
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
private readonly IFollowerModerationProcessor _followerModerationProcessor;
|
||||
private readonly ITwitterAccountModerationProcessor _twitterAccountModerationProcessor;
|
||||
|
||||
private readonly ILogger<ModerationPipeline> _logger;
|
||||
|
||||
#region Ctor
|
||||
public ModerationPipeline(IModerationRepository moderationRepository, IFollowerModerationProcessor followerModerationProcessor, ITwitterAccountModerationProcessor twitterAccountModerationProcessor, ILogger<ModerationPipeline> logger)
|
||||
{
|
||||
_moderationRepository = moderationRepository;
|
||||
_followerModerationProcessor = followerModerationProcessor;
|
||||
_twitterAccountModerationProcessor = twitterAccountModerationProcessor;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ApplyModerationSettingsAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await CheckFollowerModerationPolicyAsync();
|
||||
await CheckTwitterAccountModerationPolicyAsync();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogCritical(e, "ModerationPipeline execution failed.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CheckFollowerModerationPolicyAsync()
|
||||
{
|
||||
var followerPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
|
||||
if (followerPolicy == ModerationTypeEnum.None) return;
|
||||
|
||||
await _followerModerationProcessor.ProcessAsync(followerPolicy);
|
||||
}
|
||||
|
||||
private async Task CheckTwitterAccountModerationPolicyAsync()
|
||||
{
|
||||
var twitterAccountPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
|
||||
if (twitterAccountPolicy == ModerationTypeEnum.None) return;
|
||||
|
||||
await _twitterAccountModerationProcessor.ProcessAsync(twitterAccountPolicy);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,49 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
|
||||
namespace BirdsiteLive.Moderation.Processors
|
||||
{
|
||||
public interface IFollowerModerationProcessor
|
||||
{
|
||||
Task ProcessAsync(ModerationTypeEnum type);
|
||||
}
|
||||
|
||||
public class FollowerModerationProcessor : IFollowerModerationProcessor
|
||||
{
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
private readonly IRemoveFollowerAction _removeFollowerAction;
|
||||
|
||||
#region Ctor
|
||||
public FollowerModerationProcessor(IFollowersDal followersDal, IModerationRepository moderationRepository, IRemoveFollowerAction removeFollowerAction)
|
||||
{
|
||||
_followersDal = followersDal;
|
||||
_moderationRepository = moderationRepository;
|
||||
_removeFollowerAction = removeFollowerAction;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(ModerationTypeEnum type)
|
||||
{
|
||||
if (type == ModerationTypeEnum.None) return;
|
||||
|
||||
var followers = await _followersDal.GetAllFollowersAsync();
|
||||
|
||||
foreach (var follower in followers)
|
||||
{
|
||||
var followerHandle = $"@{follower.Acct.Trim()}@{follower.Host.Trim()}".ToLowerInvariant();
|
||||
var status = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.Follower, followerHandle);
|
||||
|
||||
if (type == ModerationTypeEnum.WhiteListing && status != ModeratedTypeEnum.WhiteListed ||
|
||||
type == ModerationTypeEnum.BlackListing && status == ModeratedTypeEnum.BlackListed)
|
||||
{
|
||||
Console.WriteLine($"Remove {followerHandle}");
|
||||
await _removeFollowerAction.ProcessAsync(follower);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,45 @@
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
|
||||
namespace BirdsiteLive.Moderation.Processors
|
||||
{
|
||||
public interface ITwitterAccountModerationProcessor
|
||||
{
|
||||
Task ProcessAsync(ModerationTypeEnum type);
|
||||
}
|
||||
|
||||
public class TwitterAccountModerationProcessor : ITwitterAccountModerationProcessor
|
||||
{
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
|
||||
|
||||
#region Ctor
|
||||
public TwitterAccountModerationProcessor(ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository, IRemoveTwitterAccountAction removeTwitterAccountAction)
|
||||
{
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_moderationRepository = moderationRepository;
|
||||
_removeTwitterAccountAction = removeTwitterAccountAction;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(ModerationTypeEnum type)
|
||||
{
|
||||
if (type == ModerationTypeEnum.None) return;
|
||||
|
||||
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
|
||||
|
||||
foreach (var user in twitterUsers)
|
||||
{
|
||||
var userHandle = user.Acct.ToLowerInvariant().Trim();
|
||||
var status = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.TwitterAccount, userHandle);
|
||||
|
||||
if (type == ModerationTypeEnum.WhiteListing && status != ModeratedTypeEnum.WhiteListed ||
|
||||
type == ModerationTypeEnum.BlackListing && status == ModeratedTypeEnum.BlackListed)
|
||||
await _removeTwitterAccountAction.ProcessAsync(user);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,23 +1,25 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<LangVersion>latest</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
|
||||
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
|
||||
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\BirdsiteLive.Domain\BirdsiteLive.Domain.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
|
||||
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Processors\TweetsCleanUp\Base\" />
|
||||
<Folder Include="Tools\" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRefreshTwitterUserStatusProcessor
|
||||
{
|
||||
Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -3,11 +3,11 @@ using System.Threading;
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRetrieveFollowersProcessor
|
||||
{
|
||||
Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
|
||||
Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct);
|
||||
//IAsyncEnumerable<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -3,10 +3,10 @@ using System.Threading.Tasks;
|
|||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRetrieveTweetsProcessor
|
||||
{
|
||||
Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
|
||||
Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -3,7 +3,7 @@ using System.Threading.Tasks;
|
|||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface IRetrieveTwitterUsersProcessor
|
||||
{
|
|
@ -2,10 +2,10 @@
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface ISaveProgressionProcessor
|
||||
{
|
||||
Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
|
||||
Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.Federation
|
||||
{
|
||||
public interface ISendTweetsToFollowersProcessor
|
||||
{
|
||||
Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -1,11 +0,0 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts
|
||||
{
|
||||
public interface ISendTweetsToFollowersProcessor
|
||||
{
|
||||
Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
|
||||
{
|
||||
public interface IDeleteTweetsProcessor
|
||||
{
|
||||
Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
|
||||
{
|
||||
public interface IRetrieveTweetsToDeleteProcessor
|
||||
{
|
||||
Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
|
||||
{
|
||||
public interface ISaveDeletedTweetStatusProcessor
|
||||
{
|
||||
Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Models
|
||||
{
|
||||
public class TweetToDelete
|
||||
{
|
||||
public SyncTweet Tweet { get; set; }
|
||||
public bool DeleteSuccessful { get; set; }
|
||||
}
|
||||
}
|
|
@ -4,7 +4,7 @@ using Tweetinvi.Models;
|
|||
|
||||
namespace BirdsiteLive.Pipeline.Models
|
||||
{
|
||||
public class UserWithTweetsToSync
|
||||
public class UserWithDataToSync
|
||||
{
|
||||
public SyncTwitterUser User { get; set; }
|
||||
public ExtractedTweet[] Tweets { get; set; }
|
|
@ -0,0 +1,109 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
|
||||
{
|
||||
private readonly ICachedTwitterUserService _twitterUserService;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
|
||||
{
|
||||
_twitterUserService = twitterUserService;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_removeTwitterAccountAction = removeTwitterAccountAction;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
|
||||
{
|
||||
var usersWtData = new List<UserWithDataToSync>();
|
||||
|
||||
foreach (var user in syncTwitterUsers)
|
||||
{
|
||||
TwitterUser userView = null;
|
||||
|
||||
try
|
||||
{
|
||||
userView = _twitterUserService.GetUser(user.Acct);
|
||||
}
|
||||
catch (UserNotFoundException)
|
||||
{
|
||||
await ProcessNotFoundUserAsync(user);
|
||||
continue;
|
||||
}
|
||||
catch (UserHasBeenSuspendedException)
|
||||
{
|
||||
await ProcessNotFoundUserAsync(user);
|
||||
continue;
|
||||
}
|
||||
catch (RateLimitExceededException)
|
||||
{
|
||||
await ProcessRateLimitExceededAsync(user);
|
||||
continue;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// ignored
|
||||
}
|
||||
|
||||
if (userView == null || userView.Protected)
|
||||
{
|
||||
await ProcessFailingUserAsync(user);
|
||||
continue;
|
||||
}
|
||||
|
||||
user.FetchingErrorCount = 0;
|
||||
var userWtData = new UserWithDataToSync
|
||||
{
|
||||
User = user
|
||||
};
|
||||
usersWtData.Add(userWtData);
|
||||
}
|
||||
return usersWtData.ToArray();
|
||||
}
|
||||
|
||||
private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user)
|
||||
{
|
||||
var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
|
||||
dbUser.LastSync = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
|
||||
}
|
||||
|
||||
private async Task ProcessNotFoundUserAsync(SyncTwitterUser user)
|
||||
{
|
||||
await _removeTwitterAccountAction.ProcessAsync(user);
|
||||
}
|
||||
|
||||
private async Task ProcessFailingUserAsync(SyncTwitterUser user)
|
||||
{
|
||||
var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
|
||||
dbUser.FetchingErrorCount++;
|
||||
dbUser.LastSync = DateTime.UtcNow;
|
||||
|
||||
if (dbUser.FetchingErrorCount > _instanceSettings.FailingTwitterUserCleanUpThreshold)
|
||||
{
|
||||
await _removeTwitterAccountAction.ProcessAsync(user);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,10 +2,10 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor
|
||||
{
|
||||
|
@ -18,7 +18,7 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
}
|
||||
#endregion
|
||||
|
||||
public async Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct)
|
||||
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
|
||||
{
|
||||
//TODO multithread this
|
||||
foreach (var user in userWithTweetsToSyncs)
|
|
@ -5,14 +5,14 @@ using System.Threading;
|
|||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Tweetinvi.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
|
||||
{
|
||||
|
@ -31,33 +31,30 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
}
|
||||
#endregion
|
||||
|
||||
public async Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
|
||||
public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
|
||||
{
|
||||
var usersWtTweets = new List<UserWithTweetsToSync>();
|
||||
var usersWtTweets = new List<UserWithDataToSync>();
|
||||
|
||||
//TODO multithread this
|
||||
foreach (var user in syncTwitterUsers)
|
||||
foreach (var userWtData in syncTwitterUsers)
|
||||
{
|
||||
var user = userWtData.User;
|
||||
var tweets = RetrieveNewTweets(user);
|
||||
if (tweets.Length > 0 && user.LastTweetPostedId != -1)
|
||||
{
|
||||
var userWtTweets = new UserWithTweetsToSync
|
||||
{
|
||||
User = user,
|
||||
Tweets = tweets
|
||||
};
|
||||
usersWtTweets.Add(userWtTweets);
|
||||
userWtData.Tweets = tweets;
|
||||
usersWtTweets.Add(userWtData);
|
||||
}
|
||||
else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
|
||||
{
|
||||
var tweetId = tweets.Last().Id;
|
||||
var now = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now);
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
|
||||
}
|
||||
else
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, now);
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
|
||||
}
|
||||
}
|
||||
|
|
@ -7,18 +7,18 @@ using BirdsiteLive.Common.Extensions;
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Tools;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
|
||||
{
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
|
||||
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
|
||||
|
||||
|
||||
public int WaitFactor = 1000 * 60; //1 min
|
||||
|
||||
#region Ctor
|
||||
|
@ -39,10 +39,10 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
try
|
||||
{
|
||||
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
|
||||
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
|
||||
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
|
||||
|
||||
var userCount = users.Any() ? users.Length : 1;
|
||||
var splitNumber = (int) Math.Ceiling(userCount / 15d);
|
||||
var splitNumber = (int)Math.Ceiling(userCount / 15d);
|
||||
var splitUsers = users.Split(splitNumber).ToList();
|
||||
|
||||
foreach (var u in splitUsers)
|
||||
|
@ -55,7 +55,12 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
}
|
||||
|
||||
var splitCount = splitUsers.Count();
|
||||
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct);
|
||||
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
|
||||
|
||||
//// Extra wait time to fit 100.000/day limit
|
||||
//var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
|
||||
//if (extraWaitTime < 0) extraWaitTime = 0;
|
||||
//await Task.Delay(extraWaitTime * 1000, ct);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
|
@ -0,0 +1,66 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class SaveProgressionProcessor : ISaveProgressionProcessor
|
||||
{
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly ILogger<SaveProgressionProcessor> _logger;
|
||||
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
|
||||
|
||||
#region Ctor
|
||||
public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger, IRemoveTwitterAccountAction removeTwitterAccountAction)
|
||||
{
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_logger = logger;
|
||||
_removeTwitterAccountAction = removeTwitterAccountAction;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (userWithTweetsToSync.Tweets.Length == 0)
|
||||
{
|
||||
_logger.LogInformation("No tweets synchronized");
|
||||
await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
|
||||
return;
|
||||
}
|
||||
if (userWithTweetsToSync.Followers.Length == 0)
|
||||
{
|
||||
_logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
|
||||
await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
|
||||
return;
|
||||
}
|
||||
|
||||
var userId = userWithTweetsToSync.User.Id;
|
||||
var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
|
||||
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
|
||||
var minimumSync = followingSyncStatuses.Min();
|
||||
var now = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateUserSyncDateAsync(SyncTwitterUser user)
|
||||
{
|
||||
user.LastSync = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(user);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -5,10 +5,12 @@ using System.Net;
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Moderation.Actions;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.SubTasks;
|
||||
using BirdsiteLive.Twitter;
|
||||
|
@ -16,24 +18,30 @@ using BirdsiteLive.Twitter.Models;
|
|||
using Microsoft.Extensions.Logging;
|
||||
using Tweetinvi.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
namespace BirdsiteLive.Pipeline.Processors.Federation
|
||||
{
|
||||
public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor
|
||||
{
|
||||
private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask;
|
||||
private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox;
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
|
||||
private readonly IRemoveFollowerAction _removeFollowerAction;
|
||||
|
||||
#region Ctor
|
||||
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, ILogger<SendTweetsToFollowersProcessor> logger)
|
||||
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction)
|
||||
{
|
||||
_sendTweetsToInboxTask = sendTweetsToInboxTask;
|
||||
_sendTweetsToSharedInbox = sendTweetsToSharedInbox;
|
||||
_logger = logger;
|
||||
_instanceSettings = instanceSettings;
|
||||
_removeFollowerAction = removeFollowerAction;
|
||||
_followersDal = followersDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
|
||||
public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
|
||||
{
|
||||
var user = userWithTweetsToSync.User;
|
||||
|
||||
|
@ -41,18 +49,18 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
var followersWtSharedInbox = userWithTweetsToSync.Followers
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
|
||||
.ToList();
|
||||
await ProcessFollowersWithSharedInbox(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
|
||||
await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
|
||||
|
||||
// Process Inbox
|
||||
var followerWtInbox = userWithTweetsToSync.Followers
|
||||
.Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
|
||||
.ToList();
|
||||
await ProcessFollowersWithInbox(userWithTweetsToSync.Tweets, followerWtInbox, user);
|
||||
await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
|
||||
|
||||
return userWithTweetsToSync;
|
||||
}
|
||||
|
||||
private async Task ProcessFollowersWithSharedInbox(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
|
||||
private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
|
||||
{
|
||||
var followersPerInstances = followers.GroupBy(x => x.Host);
|
||||
|
||||
|
@ -61,28 +69,61 @@ namespace BirdsiteLive.Pipeline.Processors
|
|||
try
|
||||
{
|
||||
await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
|
||||
|
||||
foreach (var f in followersPerInstance)
|
||||
await ProcessWorkingUserAsync(f);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
var follower = followersPerInstance.First();
|
||||
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute);
|
||||
|
||||
foreach (var f in followersPerInstance)
|
||||
await ProcessFailingUserAsync(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessFollowersWithInbox(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
|
||||
|
||||
private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
|
||||
{
|
||||
foreach (var follower in followerWtInbox)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _sendTweetsToInboxTask.ExecuteAsync(tweets, follower, user);
|
||||
await ProcessWorkingUserAsync(follower);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute);
|
||||
await ProcessFailingUserAsync(follower);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessWorkingUserAsync(Follower follower)
|
||||
{
|
||||
if (follower.PostingErrorCount > 0)
|
||||
{
|
||||
follower.PostingErrorCount = 0;
|
||||
await _followersDal.UpdateFollowerAsync(follower);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessFailingUserAsync(Follower follower)
|
||||
{
|
||||
follower.PostingErrorCount++;
|
||||
|
||||
if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold
|
||||
&& _instanceSettings.FailingFollowerCleanUpThreshold > 0
|
||||
|| follower.PostingErrorCount > 2147483600)
|
||||
{
|
||||
await _removeFollowerAction.ProcessAsync(follower);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _followersDal.UpdateFollowerAsync(follower);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,32 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
||||
{
|
||||
public class SendTweetsTaskBase
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
|
||||
#region Ctor
|
||||
protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox)
|
||||
{
|
||||
var tweet = new SyncTweet
|
||||
{
|
||||
Acct = acct,
|
||||
TweetId = tweetId,
|
||||
PublishedAt = DateTime.UtcNow,
|
||||
Inbox = inbox,
|
||||
Host = host
|
||||
};
|
||||
await _syncTweetsPostgresDal.SaveTweetAsync(tweet);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -17,17 +17,16 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user);
|
||||
}
|
||||
|
||||
public class SendTweetsToInboxTask : ISendTweetsToInboxTask
|
||||
public class SendTweetsToInboxTask : SendTweetsTaskBase, ISendTweetsToInboxTask
|
||||
{
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
private readonly IStatusService _statusService;
|
||||
private readonly IFollowersDal _followersDal;
|
||||
private readonly InstanceSettings _settings;
|
||||
private readonly ILogger<SendTweetsToInboxTask> _logger;
|
||||
|
||||
|
||||
|
||||
#region Ctor
|
||||
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
|
||||
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal)
|
||||
{
|
||||
_activityPubService = activityPubService;
|
||||
_statusService = statusService;
|
||||
|
@ -61,6 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
{
|
||||
var note = _statusService.GetStatus(user.Acct, tweet);
|
||||
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
|
||||
await SaveSyncTweetAsync(user.Acct, tweet.Id, follower.Host, inbox);
|
||||
}
|
||||
}
|
||||
catch (ArgumentException e)
|
|
@ -2,6 +2,7 @@
|
|||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
|
@ -16,7 +17,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance);
|
||||
}
|
||||
|
||||
public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
|
||||
public class SendTweetsToSharedInboxTask : SendTweetsTaskBase, ISendTweetsToSharedInboxTask
|
||||
{
|
||||
private readonly IStatusService _statusService;
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
|
@ -25,7 +26,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
|
||||
|
||||
#region Ctor
|
||||
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
|
||||
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal)
|
||||
{
|
||||
_activityPubService = activityPubService;
|
||||
_statusService = statusService;
|
||||
|
@ -61,6 +62,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
|
|||
{
|
||||
var note = _statusService.GetStatus(user.Acct, tweet);
|
||||
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
|
||||
await SaveSyncTweetAsync(user.Acct, tweet.Id, host, inbox);
|
||||
}
|
||||
}
|
||||
catch (ArgumentException e)
|
|
@ -1,31 +0,0 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors
|
||||
{
|
||||
public class SaveProgressionProcessor : ISaveProgressionProcessor
|
||||
{
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
|
||||
#region Ctor
|
||||
public SaveProgressionProcessor(ITwitterUserDal twitterUserDal)
|
||||
{
|
||||
_twitterUserDal = twitterUserDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
|
||||
{
|
||||
var userId = userWithTweetsToSync.User.Id;
|
||||
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
|
||||
var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min();
|
||||
var now = DateTime.UtcNow;
|
||||
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using System;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base
|
||||
{
|
||||
public class RetentionBase
|
||||
{
|
||||
protected int GetRetentionTime(InstanceSettings settings)
|
||||
{
|
||||
var retentionTime = Math.Abs(settings.MaxTweetRetention);
|
||||
if (retentionTime < 1) retentionTime = 1;
|
||||
if (retentionTime > 90) retentionTime = 90;
|
||||
return retentionTime;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,54 @@
|
|||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class DeleteTweetsProcessor : IDeleteTweetsProcessor
|
||||
{
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
private readonly ILogger<DeleteTweetsProcessor> _logger;
|
||||
|
||||
#region Ctor
|
||||
public DeleteTweetsProcessor(IActivityPubService activityPubService, ILogger<DeleteTweetsProcessor> logger)
|
||||
{
|
||||
_activityPubService = activityPubService;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _activityPubService.DeleteNoteAsync(tweet.Tweet);
|
||||
tweet.DeleteSuccessful = true;
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
var code = e.StatusCode;
|
||||
if (code is HttpStatusCode.Gone or HttpStatusCode.NotFound)
|
||||
{
|
||||
_logger.LogInformation("Tweet already deleted");
|
||||
tweet.DeleteSuccessful = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(e.Message, e);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e.Message, e);
|
||||
}
|
||||
|
||||
return tweet;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,64 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct)
|
||||
{
|
||||
var batchSize = 100;
|
||||
|
||||
for (;;)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
|
||||
var from = now.AddDays(-GetRetentionTime(_instanceSettings));
|
||||
var dbBrowsingEnded = false;
|
||||
var lastId = -1L;
|
||||
|
||||
do
|
||||
{
|
||||
var tweets = await _syncTweetsPostgresDal.GetTweetsOlderThanAsync(from, lastId, batchSize);
|
||||
|
||||
foreach (var syncTweet in tweets)
|
||||
{
|
||||
var tweet = new TweetToDelete
|
||||
{
|
||||
Tweet = syncTweet
|
||||
};
|
||||
await tweetsBufferBlock.SendAsync(tweet, ct);
|
||||
}
|
||||
|
||||
if (tweets.Any()) lastId = tweets.Last().Id;
|
||||
if (tweets.Count < batchSize) dbBrowsingEnded = true;
|
||||
|
||||
} while (!dbBrowsingEnded);
|
||||
|
||||
await Task.Delay(TimeSpan.FromHours(3), ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,36 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
|
||||
|
||||
namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
|
||||
{
|
||||
public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor
|
||||
{
|
||||
private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
|
||||
#region Ctor
|
||||
public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
|
||||
{
|
||||
_syncTweetsPostgresDal = syncTweetsPostgresDal;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
|
||||
{
|
||||
var retentionTime = GetRetentionTime(_instanceSettings);
|
||||
retentionTime += 20; // Delay until last retry
|
||||
var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime);
|
||||
if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
|
||||
{
|
||||
await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,9 +1,10 @@
|
|||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Pipeline.Contracts;
|
||||
using BirdsiteLive.Pipeline.Contracts.Federation;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
|
@ -17,6 +18,7 @@ namespace BirdsiteLive.Pipeline
|
|||
public class StatusPublicationPipeline : IStatusPublicationPipeline
|
||||
{
|
||||
private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor;
|
||||
private readonly IRefreshTwitterUserStatusProcessor _refreshTwitterUserStatusProcessor;
|
||||
private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
|
||||
private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
|
||||
private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
|
||||
|
@ -24,13 +26,14 @@ namespace BirdsiteLive.Pipeline
|
|||
private readonly ILogger<StatusPublicationPipeline> _logger;
|
||||
|
||||
#region Ctor
|
||||
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, ILogger<StatusPublicationPipeline> logger)
|
||||
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
|
||||
{
|
||||
_retrieveTweetsProcessor = retrieveTweetsProcessor;
|
||||
_retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
|
||||
_retrieveFollowersProcessor = retrieveFollowersProcessor;
|
||||
_sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
|
||||
_saveProgressionProcessor = saveProgressionProcessor;
|
||||
_refreshTwitterUserStatusProcessor = refreshTwitterUserStatusProcessor;
|
||||
|
||||
_logger = logger;
|
||||
}
|
||||
|
@ -39,16 +42,21 @@ namespace BirdsiteLive.Pipeline
|
|||
public async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
// Create blocks
|
||||
var twitterUsersBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
|
||||
var retrieveTweetsBlock = new TransformBlock<SyncTwitterUser[], UserWithTweetsToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
|
||||
var retrieveTweetsBufferBlock = new BufferBlock<UserWithTweetsToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
|
||||
var retrieveFollowersBlock = new TransformManyBlock<UserWithTweetsToSync[], UserWithTweetsToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
|
||||
var retrieveFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
|
||||
var sendTweetsToFollowersBlock = new TransformBlock<UserWithTweetsToSync, UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
|
||||
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
|
||||
var saveProgressionBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
|
||||
var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
|
||||
{ BoundedCapacity = 1, CancellationToken = ct });
|
||||
var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
|
||||
var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
|
||||
var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
|
||||
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
|
||||
var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
|
||||
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
|
||||
var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
|
||||
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
|
||||
var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
|
||||
|
||||
// Link pipeline
|
||||
twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
|
||||
twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
|
||||
twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
|
||||
retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
|
||||
retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
|
||||
|
@ -58,7 +66,7 @@ namespace BirdsiteLive.Pipeline
|
|||
sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
|
||||
|
||||
// Launch twitter user retriever
|
||||
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct);
|
||||
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
|
||||
|
||||
// Wait
|
||||
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
|
||||
|
|
|
@ -16,6 +16,7 @@ namespace BirdsiteLive.Pipeline.Tools
|
|||
|
||||
private int _totalUsersCount = -1;
|
||||
private int _warmUpIterations;
|
||||
private const int WarmUpMaxCapacity = 200;
|
||||
|
||||
#region Ctor
|
||||
public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
|
||||
|
@ -31,8 +32,7 @@ namespace BirdsiteLive.Pipeline.Tools
|
|||
if (_totalUsersCount == -1)
|
||||
{
|
||||
_totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
|
||||
var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4;
|
||||
_warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int)(_totalUsersCount / (float)warmUpMaxCapacity);
|
||||
_warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
|
||||
}
|
||||
|
||||
// Return if warm up ended
|
||||
|
@ -40,7 +40,7 @@ namespace BirdsiteLive.Pipeline.Tools
|
|||
|
||||
// Calculate warm up value
|
||||
var maxUsers = _warmUpIterations > 0
|
||||
? _instanceSettings.MaxUsersCapacity / 4
|
||||
? WarmUpMaxCapacity
|
||||
: _instanceSettings.MaxUsersCapacity;
|
||||
_warmUpIterations--;
|
||||
return maxUsers;
|
||||
|
|
|
@ -0,0 +1,76 @@
|
|||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
|
||||
using BirdsiteLive.Pipeline.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Pipeline
|
||||
{
|
||||
public interface ITweetCleanUpPipeline
|
||||
{
|
||||
Task ExecuteAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
public class TweetCleanUpPipeline : ITweetCleanUpPipeline
|
||||
{
|
||||
private readonly IRetrieveTweetsToDeleteProcessor _retrieveTweetsToDeleteProcessor;
|
||||
private readonly IDeleteTweetsProcessor _deleteTweetsProcessor;
|
||||
private readonly ISaveDeletedTweetStatusProcessor _saveDeletedTweetStatusProcessor;
|
||||
private readonly ILogger<TweetCleanUpPipeline> _logger;
|
||||
|
||||
#region Ctor
|
||||
public TweetCleanUpPipeline(IRetrieveTweetsToDeleteProcessor retrieveTweetsToDeleteProcessor, IDeleteTweetsProcessor deleteTweetsProcessor, ISaveDeletedTweetStatusProcessor saveDeletedTweetStatusProcessor, ILogger<TweetCleanUpPipeline> logger)
|
||||
{
|
||||
_retrieveTweetsToDeleteProcessor = retrieveTweetsToDeleteProcessor;
|
||||
_deleteTweetsProcessor = deleteTweetsProcessor;
|
||||
_saveDeletedTweetStatusProcessor = saveDeletedTweetStatusProcessor;
|
||||
_logger = logger;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
// Create blocks
|
||||
var tweetsToDeleteBufferBlock = new BufferBlock<TweetToDelete>(new DataflowBlockOptions
|
||||
{
|
||||
BoundedCapacity = 200,
|
||||
CancellationToken = ct
|
||||
});
|
||||
var deleteTweetsBlock = new TransformBlock<TweetToDelete, TweetToDelete>(
|
||||
async x => await _deleteTweetsProcessor.ProcessAsync(x, ct),
|
||||
new ExecutionDataflowBlockOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = 5,
|
||||
CancellationToken = ct
|
||||
});
|
||||
var deletedTweetsBufferBlock = new BufferBlock<TweetToDelete>(new DataflowBlockOptions
|
||||
{
|
||||
BoundedCapacity = 200,
|
||||
CancellationToken = ct
|
||||
});
|
||||
var saveProgressionBlock = new ActionBlock<TweetToDelete>(
|
||||
async x => await _saveDeletedTweetStatusProcessor.ProcessAsync(x, ct),
|
||||
new ExecutionDataflowBlockOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = 5,
|
||||
CancellationToken = ct
|
||||
});
|
||||
|
||||
// Link pipeline
|
||||
var dataflowLinkOptions = new DataflowLinkOptions { PropagateCompletion = true };
|
||||
tweetsToDeleteBufferBlock.LinkTo(deleteTweetsBlock, dataflowLinkOptions);
|
||||
deleteTweetsBlock.LinkTo(deletedTweetsBufferBlock, dataflowLinkOptions);
|
||||
deletedTweetsBufferBlock.LinkTo(saveProgressionBlock, dataflowLinkOptions);
|
||||
|
||||
// Launch tweet retriever
|
||||
var retrieveTweetsToDeleteTask = _retrieveTweetsToDeleteProcessor.GetTweetsAsync(tweetsToDeleteBufferBlock, ct);
|
||||
|
||||
// Wait
|
||||
await Task.WhenAny(new[] { retrieveTweetsToDeleteTask, saveProgressionBlock.Completion });
|
||||
|
||||
var ex = retrieveTweetsToDeleteTask.IsFaulted ? retrieveTweetsToDeleteTask.Exception : saveProgressionBlock.Completion.Exception;
|
||||
_logger.LogCritical(ex, "An error occurred, pipeline stopped");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -5,7 +5,7 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
|
||||
<PackageReference Include="TweetinviAPI" Version="4.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
using System;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
|
@ -13,23 +14,25 @@ namespace BirdsiteLive.Twitter
|
|||
{
|
||||
private readonly ITwitterUserService _twitterService;
|
||||
|
||||
private MemoryCache _userCache = new MemoryCache(new MemoryCacheOptions()
|
||||
{
|
||||
SizeLimit = 5000
|
||||
});
|
||||
private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
|
||||
private readonly MemoryCache _userCache;
|
||||
private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
|
||||
.SetSize(1)//Size amount
|
||||
//Priority on removing when reaching size limit (memory pressure)
|
||||
.SetPriority(CacheItemPriority.High)
|
||||
// Keep in cache for this time, reset time if accessed.
|
||||
.SetSlidingExpiration(TimeSpan.FromHours(24))
|
||||
// Remove from cache after this time, regardless of sliding expiration
|
||||
.SetAbsoluteExpiration(TimeSpan.FromDays(30));
|
||||
.SetAbsoluteExpiration(TimeSpan.FromDays(7));
|
||||
|
||||
#region Ctor
|
||||
public CachedTwitterUserService(ITwitterUserService twitterService)
|
||||
public CachedTwitterUserService(ITwitterUserService twitterService, InstanceSettings settings)
|
||||
{
|
||||
_twitterService = twitterService;
|
||||
|
||||
_userCache = new MemoryCache(new MemoryCacheOptions()
|
||||
{
|
||||
SizeLimit = settings.UserCacheCapacity
|
||||
});
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
@ -44,6 +47,11 @@ namespace BirdsiteLive.Twitter
|
|||
return user;
|
||||
}
|
||||
|
||||
public bool IsUserApiRateLimited()
|
||||
{
|
||||
return _twitterService.IsUserApiRateLimited();
|
||||
}
|
||||
|
||||
public void PurgeUser(string username)
|
||||
{
|
||||
_userCache.Remove(username);
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
using System;
|
||||
|
||||
namespace BirdsiteLive.Twitter
|
||||
{
|
||||
public class RateLimitExceededException : Exception
|
||||
{
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
using System;
|
||||
|
||||
namespace BirdsiteLive.Twitter
|
||||
{
|
||||
public class UserHasBeenSuspendedException : Exception
|
||||
{
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
using System;
|
||||
|
||||
namespace BirdsiteLive.Twitter
|
||||
{
|
||||
public class UserNotFoundException : Exception
|
||||
{
|
||||
|
||||
}
|
||||
}
|
|
@ -28,7 +28,8 @@ namespace BirdsiteLive.Twitter.Extractors
|
|||
IsReply = tweet.InReplyToUserId != null,
|
||||
IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
|
||||
IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null,
|
||||
RetweetUrl = ExtractRetweetUrl(tweet)
|
||||
RetweetUrl = ExtractRetweetUrl(tweet),
|
||||
CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName
|
||||
};
|
||||
|
||||
return extractedTweet;
|
||||
|
@ -57,7 +58,7 @@ namespace BirdsiteLive.Twitter.Extractors
|
|||
var message = tweet.FullText;
|
||||
var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
|
||||
|
||||
if (tweet.IsRetweet && tweet.QuotedStatusId == null && message.StartsWith("RT") && tweet.RetweetedTweet != null)
|
||||
if (tweet.IsRetweet && message.StartsWith("RT") && tweet.RetweetedTweet != null)
|
||||
{
|
||||
message = tweet.RetweetedTweet.FullText;
|
||||
tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct();
|
||||
|
|
|
@ -15,5 +15,6 @@ namespace BirdsiteLive.Twitter.Models
|
|||
public bool IsThread { get; set; }
|
||||
public bool IsRetweet { get; set; }
|
||||
public string RetweetUrl { get; set; }
|
||||
public string CreatorName { get; set; }
|
||||
}
|
||||
}
|
|
@ -13,6 +13,8 @@ namespace BirdsiteLive.Statistics.Domain
|
|||
void CalledTweetApi();
|
||||
void CalledTimelineApi();
|
||||
ApiStatistics GetStatistics();
|
||||
|
||||
int GetCurrentUserCalls();
|
||||
}
|
||||
|
||||
//Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
|
||||
|
@ -60,7 +62,12 @@ namespace BirdsiteLive.Statistics.Domain
|
|||
foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data);
|
||||
}
|
||||
|
||||
public void CalledUserApi() //GET users/show - 900/15mins
|
||||
public int GetCurrentUserCalls()
|
||||
{
|
||||
return _userCalls;
|
||||
}
|
||||
|
||||
public void CalledUserApi() //GET users/show - 300/15mins
|
||||
{
|
||||
Interlocked.Increment(ref _userCalls);
|
||||
}
|
||||
|
@ -87,7 +94,7 @@ namespace BirdsiteLive.Statistics.Domain
|
|||
UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0,
|
||||
UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0,
|
||||
UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0,
|
||||
UserCallsMax = 900,
|
||||
UserCallsMax = 300,
|
||||
TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0,
|
||||
TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0,
|
||||
TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0,
|
||||
|
@ -95,7 +102,7 @@ namespace BirdsiteLive.Statistics.Domain
|
|||
TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0,
|
||||
TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0,
|
||||
TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0,
|
||||
TimelineCallsMax = 1500
|
||||
TimelineCallsMax = 1000
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ using BirdsiteLive.Twitter.Models;
|
|||
using BirdsiteLive.Twitter.Tools;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Tweetinvi;
|
||||
using Tweetinvi.Exceptions;
|
||||
using Tweetinvi.Models;
|
||||
|
||||
namespace BirdsiteLive.Twitter
|
||||
|
@ -13,6 +14,7 @@ namespace BirdsiteLive.Twitter
|
|||
public interface ITwitterUserService
|
||||
{
|
||||
TwitterUser GetUser(string username);
|
||||
bool IsUserApiRateLimited();
|
||||
}
|
||||
|
||||
public class TwitterUserService : ITwitterUserService
|
||||
|
@ -32,24 +34,46 @@ namespace BirdsiteLive.Twitter
|
|||
|
||||
public TwitterUser GetUser(string username)
|
||||
{
|
||||
//Check if API is saturated
|
||||
if (IsUserApiRateLimited()) throw new RateLimitExceededException();
|
||||
|
||||
//Proceed to account retrieval
|
||||
_twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
|
||||
ExceptionHandler.SwallowWebExceptions = false;
|
||||
RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
|
||||
|
||||
IUser user;
|
||||
try
|
||||
{
|
||||
user = User.GetUserFromScreenName(username);
|
||||
_statisticsHandler.CalledUserApi();
|
||||
if (user == null)
|
||||
}
|
||||
catch (TwitterException e)
|
||||
{
|
||||
if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant())))
|
||||
{
|
||||
_logger.LogWarning("User {username} not found", username);
|
||||
return null;
|
||||
throw new UserHasBeenSuspendedException();
|
||||
}
|
||||
else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant())))
|
||||
{
|
||||
throw new UserNotFoundException();
|
||||
}
|
||||
else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant())))
|
||||
{
|
||||
throw new RateLimitExceededException();
|
||||
}
|
||||
else
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error retrieving user {Username}", username);
|
||||
return null;
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_statisticsHandler.CalledUserApi();
|
||||
}
|
||||
|
||||
// Expand URLs
|
||||
|
@ -64,11 +88,38 @@ namespace BirdsiteLive.Twitter
|
|||
Name = user.Name,
|
||||
Description = description,
|
||||
Url = $"https://twitter.com/{username}",
|
||||
ProfileImageUrl = user.ProfileImageUrlFullSize,
|
||||
ProfileImageUrl = user.ProfileImageUrlFullSize.Replace("http://", "https://"),
|
||||
ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,
|
||||
ProfileBannerURL = user.ProfileBannerURL,
|
||||
Protected = user.Protected
|
||||
};
|
||||
}
|
||||
|
||||
public bool IsUserApiRateLimited()
|
||||
{
|
||||
// Retrieve limit from tooling
|
||||
_twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
|
||||
ExceptionHandler.SwallowWebExceptions = false;
|
||||
RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
|
||||
|
||||
try
|
||||
{
|
||||
var queryRateLimits = RateLimit.GetQueryRateLimit("https://api.twitter.com/1.1/users/show.json?screen_name=mastodon");
|
||||
|
||||
if (queryRateLimits != null)
|
||||
{
|
||||
return queryRateLimits.Remaining <= 0;
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error retrieving rate limits");
|
||||
}
|
||||
|
||||
// Fallback
|
||||
var currentCalls = _statisticsHandler.GetCurrentUserCalls();
|
||||
var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax;
|
||||
return currentCalls >= maxCalls;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -39,7 +39,17 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests",
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}"
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation", "BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj", "{4BE541AC-8A93-4FA3-98AC-956CC2D5B748}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation.Tests", "Tests\BirdsiteLive.Moderation.Tests\BirdsiteLive.Moderation.Tests.csproj", "{0A311BF3-4FD9-4303-940A-A3778890561C}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BSLManager", "BSLManager\BSLManager.csproj", "{4A84D351-E91B-4E58-8E20-211F0F4991D7}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSLManager.Tests", "Tests\BSLManager.Tests\BSLManager.Tests.csproj", "{D4457271-620E-465A-B08E-7FC63C99A2F6}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
|
@ -107,6 +117,26 @@ Global
|
|||
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4BE541AC-8A93-4FA3-98AC-956CC2D5B748}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4BE541AC-8A93-4FA3-98AC-956CC2D5B748}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4BE541AC-8A93-4FA3-98AC-956CC2D5B748}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4BE541AC-8A93-4FA3-98AC-956CC2D5B748}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{0A311BF3-4FD9-4303-940A-A3778890561C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{0A311BF3-4FD9-4303-940A-A3778890561C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{0A311BF3-4FD9-4303-940A-A3778890561C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{0A311BF3-4FD9-4303-940A-A3778890561C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
@ -126,6 +156,10 @@ Global
|
|||
{F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
|
||||
{BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
|
||||
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
|
||||
{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
|
||||
{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
|
||||
{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
|
||||
{D4457271-620E-465A-B08E-7FC63C99A2F6} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}
|
||||
|
|
|
@ -1,23 +1,24 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
|
||||
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
|
||||
<Version>0.13.0</Version>
|
||||
<Version>0.23.0</Version>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />
|
||||
<PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.17.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.11" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Cryptography\BirdsiteLive.Cryptography.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Domain\BirdsiteLive.Domain.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
|
||||
<ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
|
||||
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
|
||||
|
|
|
@ -0,0 +1,59 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Services;
|
||||
using BirdsiteLive.Statistics.Domain;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Razor.Language.Intermediate;
|
||||
|
||||
namespace BirdsiteLive.Component
|
||||
{
|
||||
public class NodeInfoViewComponent : ViewComponent
|
||||
{
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
private readonly ICachedStatisticsService _cachedStatisticsService;
|
||||
|
||||
#region Ctor
|
||||
public NodeInfoViewComponent(IModerationRepository moderationRepository, ICachedStatisticsService cachedStatisticsService)
|
||||
{
|
||||
_moderationRepository = moderationRepository;
|
||||
_cachedStatisticsService = cachedStatisticsService;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<IViewComponentResult> InvokeAsync()
|
||||
{
|
||||
var followerPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
|
||||
var twitterAccountPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
|
||||
|
||||
var statistics = await _cachedStatisticsService.GetStatisticsAsync();
|
||||
|
||||
var viewModel = new NodeInfoViewModel
|
||||
{
|
||||
BlacklistingEnabled = followerPolicy == ModerationTypeEnum.BlackListing ||
|
||||
twitterAccountPolicy == ModerationTypeEnum.BlackListing,
|
||||
WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing ||
|
||||
twitterAccountPolicy == ModerationTypeEnum.WhiteListing,
|
||||
InstanceSaturation = statistics.Saturation
|
||||
};
|
||||
|
||||
//viewModel = new NodeInfoViewModel
|
||||
//{
|
||||
// BlacklistingEnabled = false,
|
||||
// WhitelistingEnabled = false,
|
||||
// InstanceSaturation = 175
|
||||
//};
|
||||
return View(viewModel);
|
||||
}
|
||||
}
|
||||
|
||||
public class NodeInfoViewModel
|
||||
{
|
||||
public bool BlacklistingEnabled { get; set; }
|
||||
public bool WhitelistingEnabled { get; set; }
|
||||
public int InstanceSaturation { get; set; }
|
||||
}
|
||||
}
|
|
@ -0,0 +1,58 @@
|
|||
using Microsoft.AspNetCore.Mvc;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Services;
|
||||
|
||||
namespace BirdsiteLive.Controllers
|
||||
{
|
||||
public class AboutController : Controller
|
||||
{
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
private readonly ICachedStatisticsService _cachedStatisticsService;
|
||||
|
||||
#region Ctor
|
||||
public AboutController(IModerationRepository moderationRepository, ICachedStatisticsService cachedStatisticsService)
|
||||
{
|
||||
_moderationRepository = moderationRepository;
|
||||
_cachedStatisticsService = cachedStatisticsService;
|
||||
}
|
||||
#endregion
|
||||
|
||||
public async Task<IActionResult> Index()
|
||||
{
|
||||
var stats = await _cachedStatisticsService.GetStatisticsAsync();
|
||||
return View(stats);
|
||||
}
|
||||
|
||||
public IActionResult Blacklisting()
|
||||
{
|
||||
var status = GetModerationStatus();
|
||||
return View("Blacklisting", status);
|
||||
}
|
||||
|
||||
public IActionResult Whitelisting()
|
||||
{
|
||||
var status = GetModerationStatus();
|
||||
return View("Whitelisting", status);
|
||||
}
|
||||
|
||||
private ModerationStatus GetModerationStatus()
|
||||
{
|
||||
var status = new ModerationStatus
|
||||
{
|
||||
Followers = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower),
|
||||
TwitterAccounts = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount)
|
||||
};
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
public class ModerationStatus
|
||||
{
|
||||
public ModerationTypeEnum Followers { get; set; }
|
||||
public ModerationTypeEnum TwitterAccounts { get; set; }
|
||||
}
|
||||
}
|
|
@ -14,18 +14,21 @@ using Newtonsoft.Json;
|
|||
|
||||
namespace BirdsiteLive.Controllers
|
||||
{
|
||||
#if DEBUG
|
||||
public class DebugingController : Controller
|
||||
{
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly ICryptoService _cryptoService;
|
||||
private readonly IActivityPubService _activityPubService;
|
||||
private readonly IUserService _userService;
|
||||
|
||||
#region Ctor
|
||||
public DebugingController(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService)
|
||||
public DebugingController(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IUserService userService)
|
||||
{
|
||||
_instanceSettings = instanceSettings;
|
||||
_cryptoService = cryptoService;
|
||||
_activityPubService = activityPubService;
|
||||
_userService = userService;
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
@ -53,21 +56,56 @@ namespace BirdsiteLive.Controllers
|
|||
return View("Index");
|
||||
}
|
||||
|
||||
private static string _noteId;
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> DeleteNote()
|
||||
{
|
||||
var username = "twitter";
|
||||
var actor = $"https://{_instanceSettings.Domain}/users/{username}";
|
||||
var targetHost = "ioc.exchange";
|
||||
var target = $"https://{targetHost}/users/test";
|
||||
var inbox = $"/inbox";
|
||||
|
||||
var delete = new ActivityDelete
|
||||
{
|
||||
context = "https://www.w3.org/ns/activitystreams",
|
||||
id = $"{_noteId}#delete",
|
||||
type = "Delete",
|
||||
actor = actor,
|
||||
to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
apObject = new Tombstone
|
||||
{
|
||||
id = _noteId,
|
||||
atomUrl = _noteId
|
||||
}
|
||||
};
|
||||
|
||||
await _activityPubService.PostDataAsync(delete, targetHost, actor, inbox);
|
||||
return View("Index");
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> PostNote()
|
||||
{
|
||||
var username = "gra";
|
||||
var username = "twitter";
|
||||
var actor = $"https://{_instanceSettings.Domain}/users/{username}";
|
||||
var targetHost = "mastodon.technology";
|
||||
var target = $"{targetHost}/users/testtest";
|
||||
var inbox = $"/users/testtest/inbox";
|
||||
var targetHost = "ioc.exchange";
|
||||
var target = $"https://{targetHost}/users/test";
|
||||
//var inbox = $"/users/testtest/inbox";
|
||||
var inbox = $"/inbox";
|
||||
|
||||
var noteGuid = Guid.NewGuid();
|
||||
var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}";
|
||||
var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}";
|
||||
|
||||
|
||||
_noteId = noteId;
|
||||
|
||||
var to = $"{actor}/followers";
|
||||
var apPublic = "https://www.w3.org/ns/activitystreams#Public";
|
||||
to = target;
|
||||
|
||||
var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" };
|
||||
cc = new string[0];
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var nowString = now.ToString("s") + "Z";
|
||||
|
@ -79,22 +117,38 @@ namespace BirdsiteLive.Controllers
|
|||
type = "Create",
|
||||
actor = actor,
|
||||
published = nowString,
|
||||
to = new []{ to },
|
||||
//cc = new [] { apPublic },
|
||||
to = new[] { to },
|
||||
cc = cc,
|
||||
apObject = new Note()
|
||||
{
|
||||
id = noteId,
|
||||
summary = null,
|
||||
summary = null,
|
||||
inReplyTo = null,
|
||||
published = nowString,
|
||||
url = noteUrl,
|
||||
attributedTo = actor,
|
||||
|
||||
// Unlisted
|
||||
to = new[] { to },
|
||||
//cc = new [] { apPublic },
|
||||
cc = cc,
|
||||
//cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
|
||||
//// Public
|
||||
//to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
|
||||
//cc = new[] { to },
|
||||
|
||||
sensitive = false,
|
||||
content = "<p>Woooot</p>",
|
||||
content = "<p>TEST PUBLIC</p>",
|
||||
//content = "<p><span class=\"h-card\"><a href=\"https://ioc.exchange/users/test\" class=\"u-url mention\">@<span>test</span></a></span> test</p>",
|
||||
attachment = new Attachment[0],
|
||||
tag = new Tag[0]
|
||||
tag = new Tag[]{
|
||||
new Tag()
|
||||
{
|
||||
type = "Mention",
|
||||
href = target,
|
||||
name = "@test@ioc.exchange"
|
||||
}
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -102,7 +156,33 @@ namespace BirdsiteLive.Controllers
|
|||
|
||||
return View("Index");
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> PostRejectFollow()
|
||||
{
|
||||
var activityFollow = new ActivityFollow
|
||||
{
|
||||
type = "Follow",
|
||||
actor = "https://mastodon.technology/users/testtest",
|
||||
apObject = $"https://{_instanceSettings.Domain}/users/afp"
|
||||
};
|
||||
|
||||
await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology");
|
||||
return View("Index");
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> PostDeleteUser()
|
||||
{
|
||||
var userName = "twitter";
|
||||
var host = "ioc.exchange";
|
||||
var inbox = "/inbox";
|
||||
|
||||
await _activityPubService.DeleteUserAsync(userName, host, inbox);
|
||||
return View("Index");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
public static class HtmlHelperExtensions
|
||||
{
|
||||
|
|
|
@ -3,25 +3,61 @@ using System.Collections.Generic;
|
|||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.ActivityPub;
|
||||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Tools;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BirdsiteLive.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
public class InboxController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<InboxController> _logger;
|
||||
private readonly IUserService _userService;
|
||||
|
||||
#region Ctor
|
||||
public InboxController(ILogger<InboxController> logger, IUserService userService)
|
||||
{
|
||||
_logger = logger;
|
||||
_userService = userService;
|
||||
}
|
||||
#endregion
|
||||
|
||||
[Route("/inbox")]
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> Inbox()
|
||||
{
|
||||
var r = Request;
|
||||
using (var reader = new StreamReader(Request.Body))
|
||||
try
|
||||
{
|
||||
var body = await reader.ReadToEndAsync();
|
||||
//System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body);
|
||||
var r = Request;
|
||||
using (var reader = new StreamReader(Request.Body))
|
||||
{
|
||||
var body = await reader.ReadToEndAsync();
|
||||
|
||||
_logger.LogTrace("Inbox: {Body}", body);
|
||||
//System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body);
|
||||
|
||||
var activity = ApDeserializer.ProcessActivity(body);
|
||||
var signature = r.Headers["Signature"].First();
|
||||
|
||||
switch (activity?.type)
|
||||
{
|
||||
case "Delete":
|
||||
{
|
||||
var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path,
|
||||
r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers),
|
||||
activity as ActivityDelete, body);
|
||||
if (succeeded) return Accepted();
|
||||
else return Unauthorized();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (FollowerIsGoneException) { }
|
||||
|
||||
return Accepted();
|
||||
}
|
||||
|
|
|
@ -0,0 +1,226 @@
|
|||
using System;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Domain.Enum;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Models;
|
||||
using System.Reflection.Metadata;
|
||||
|
||||
namespace BirdsiteLive.Controllers
|
||||
{
|
||||
public class MigrationController : Controller
|
||||
{
|
||||
private readonly MigrationService _migrationService;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
|
||||
#region Ctor
|
||||
public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal)
|
||||
{
|
||||
_migrationService = migrationService;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
[HttpGet]
|
||||
[Route("/migration/move/{id}")]
|
||||
public IActionResult IndexMove(string id)
|
||||
{
|
||||
var migrationCode = _migrationService.GetMigrationCode(id);
|
||||
var data = new MigrationData()
|
||||
{
|
||||
Acct = id,
|
||||
MigrationCode = migrationCode
|
||||
};
|
||||
|
||||
return View("Index", data);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("/migration/delete/{id}")]
|
||||
public IActionResult IndexDelete(string id)
|
||||
{
|
||||
var migrationCode = _migrationService.GetDeletionCode(id);
|
||||
var data = new MigrationData()
|
||||
{
|
||||
Acct = id,
|
||||
MigrationCode = migrationCode
|
||||
};
|
||||
|
||||
return View("Delete", data);
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
[Route("/migration/move/{id}")]
|
||||
public async Task<IActionResult> MigrateMove(string id, string tweetid, string handle)
|
||||
{
|
||||
var migrationCode = _migrationService.GetMigrationCode(id);
|
||||
var data = new MigrationData()
|
||||
{
|
||||
Acct = id,
|
||||
MigrationCode = migrationCode,
|
||||
|
||||
IsAcctProvided = !string.IsNullOrWhiteSpace(handle),
|
||||
IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
|
||||
|
||||
TweetId = tweetid,
|
||||
FediverseAccount = handle
|
||||
};
|
||||
ValidatedFediverseUser fediverseUserValidation = null;
|
||||
|
||||
//Verify can be migrated
|
||||
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
|
||||
if (twitterAccount != null && twitterAccount.Deleted)
|
||||
{
|
||||
data.ErrorMessage = "This account has been deleted, it can't be migrated";
|
||||
return View("Index", data);
|
||||
}
|
||||
if (twitterAccount != null &&
|
||||
(!string.IsNullOrWhiteSpace(twitterAccount.MovedTo)
|
||||
|| !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
|
||||
{
|
||||
data.ErrorMessage = "This account has been moved already, it can't be migrated again";
|
||||
return View("Index", data);
|
||||
}
|
||||
|
||||
// Start migration
|
||||
try
|
||||
{
|
||||
fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
|
||||
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
|
||||
|
||||
data.IsAcctValid = fediverseUserValidation.IsValid;
|
||||
data.IsTweetValid = isTweetValid;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
data.ErrorMessage = e.Message;
|
||||
}
|
||||
|
||||
if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
|
||||
_migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle);
|
||||
data.MigrationSuccess = true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
data.ErrorMessage = e.Message;
|
||||
}
|
||||
}
|
||||
|
||||
return View("Index", data);
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
[Route("/migration/delete/{id}")]
|
||||
public async Task<IActionResult> MigrateDelete(string id, string tweetid)
|
||||
{
|
||||
var deletionCode = _migrationService.GetDeletionCode(id);
|
||||
|
||||
var data = new MigrationData()
|
||||
{
|
||||
Acct = id,
|
||||
MigrationCode = deletionCode,
|
||||
|
||||
IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
|
||||
|
||||
TweetId = tweetid
|
||||
};
|
||||
|
||||
//Verify can be deleted
|
||||
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
|
||||
if (twitterAccount != null && twitterAccount.Deleted)
|
||||
{
|
||||
data.ErrorMessage = "This account has been deleted, it can't be deleted again";
|
||||
return View("Delete", data);
|
||||
}
|
||||
|
||||
// Start deletion
|
||||
try
|
||||
{
|
||||
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
|
||||
data.IsTweetValid = isTweetValid;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
data.ErrorMessage = e.Message;
|
||||
}
|
||||
|
||||
if (data.IsTweetValid)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _migrationService.DeleteAccountAsync(id);
|
||||
_migrationService.TriggerRemoteDeleteAsync(id, tweetid);
|
||||
data.MigrationSuccess = true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
data.ErrorMessage = e.Message;
|
||||
}
|
||||
}
|
||||
|
||||
return View("Delete", data);
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
[Route("/migration/move/{id}/{tweetid}/{handle}")]
|
||||
public async Task<IActionResult> RemoteMigrateMove(string id, string tweetid, string handle)
|
||||
{
|
||||
//Check inputs
|
||||
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid) ||
|
||||
string.IsNullOrWhiteSpace(handle))
|
||||
return StatusCode(422);
|
||||
|
||||
//Verify can be migrated
|
||||
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
|
||||
if (twitterAccount != null && (twitterAccount.Deleted
|
||||
|| !string.IsNullOrWhiteSpace(twitterAccount.MovedTo)
|
||||
|| !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
|
||||
return Ok();
|
||||
|
||||
// Start migration
|
||||
var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
|
||||
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
|
||||
|
||||
if (fediverseUserValidation.IsValid && isTweetValid)
|
||||
{
|
||||
await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
|
||||
return Ok();
|
||||
}
|
||||
|
||||
return StatusCode(400);
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
[Route("/migration/delete/{id}/{tweetid}")]
|
||||
public async Task<IActionResult> RemoteMigrateDelete(string id, string tweetid)
|
||||
{
|
||||
//Check inputs
|
||||
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid))
|
||||
return StatusCode(422);
|
||||
|
||||
//Verify can be deleted
|
||||
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
|
||||
if (twitterAccount != null && twitterAccount.Deleted) return Ok();
|
||||
|
||||
// Start deletion
|
||||
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
|
||||
|
||||
if (isTweetValid)
|
||||
{
|
||||
await _migrationService.DeleteAccountAsync(id);
|
||||
return Ok();
|
||||
}
|
||||
|
||||
return StatusCode(400);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Domain.Statistics;
|
||||
|
@ -16,7 +17,6 @@ namespace BirdsiteLive.Controllers
|
|||
private readonly ITwitterStatisticsHandler _twitterStatistics;
|
||||
private readonly IExtractionStatisticsHandler _extractionStatistics;
|
||||
|
||||
|
||||
#region Ctor
|
||||
public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics)
|
||||
{
|
||||
|
@ -32,7 +32,9 @@ namespace BirdsiteLive.Controllers
|
|||
var stats = new Models.StatisticsModels.Statistics
|
||||
{
|
||||
FollowersCount = await _followersDal.GetFollowersCountAsync(),
|
||||
FailingFollowersCount = await _followersDal.GetFailingFollowersCountAsync(),
|
||||
TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(),
|
||||
FailingTwitterUserCount = await _twitterUserDal.GetFailingTwitterUsersCountAsync(),
|
||||
TwitterStatistics = _twitterStatistics.GetStatistics(),
|
||||
ExtractionStatistics = _extractionStatistics.GetStatistics(),
|
||||
};
|
||||
|
|
|
@ -3,7 +3,6 @@ using System.Collections.Generic;
|
|||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Mime;
|
||||
using System.Runtime.InteropServices.WindowsRuntime;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
@ -11,12 +10,16 @@ using BirdsiteLive.ActivityPub;
|
|||
using BirdsiteLive.ActivityPub.Models;
|
||||
using BirdsiteLive.Common.Regexes;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.DAL.Models;
|
||||
using BirdsiteLive.Domain;
|
||||
using BirdsiteLive.Models;
|
||||
using BirdsiteLive.Tools;
|
||||
using BirdsiteLive.Twitter;
|
||||
using BirdsiteLive.Twitter.Models;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
|
@ -26,18 +29,22 @@ namespace BirdsiteLive.Controllers
|
|||
{
|
||||
private readonly ITwitterUserService _twitterUserService;
|
||||
private readonly ITwitterTweetsService _twitterTweetService;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly IUserService _userService;
|
||||
private readonly IStatusService _statusService;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly ILogger<UsersController> _logger;
|
||||
|
||||
#region Ctor
|
||||
public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService)
|
||||
public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger<UsersController> logger, ITwitterUserDal twitterUserDal)
|
||||
{
|
||||
_twitterUserService = twitterUserService;
|
||||
_userService = userService;
|
||||
_statusService = statusService;
|
||||
_instanceSettings = instanceSettings;
|
||||
_twitterTweetService = twitterTweetService;
|
||||
_logger = logger;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
@ -52,18 +59,52 @@ namespace BirdsiteLive.Controllers
|
|||
}
|
||||
return View("UserNotFound");
|
||||
}
|
||||
|
||||
|
||||
[Route("/@{id}")]
|
||||
[Route("/users/{id}")]
|
||||
public IActionResult Index(string id)
|
||||
public async Task<IActionResult> Index(string id)
|
||||
{
|
||||
_logger.LogTrace("User Index: {Id}", id);
|
||||
|
||||
id = id.Trim(new[] { ' ', '@' }).ToLowerInvariant();
|
||||
|
||||
TwitterUser user = null;
|
||||
var isSaturated = false;
|
||||
var notFound = false;
|
||||
|
||||
// Ensure valid username
|
||||
// https://help.twitter.com/en/managing-your-account/twitter-username-rules
|
||||
TwitterUser user = null;
|
||||
if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15)
|
||||
user = _twitterUserService.GetUser(id);
|
||||
{
|
||||
try
|
||||
{
|
||||
user = _twitterUserService.GetUser(id);
|
||||
}
|
||||
catch (UserNotFoundException)
|
||||
{
|
||||
notFound = true;
|
||||
}
|
||||
catch (UserHasBeenSuspendedException)
|
||||
{
|
||||
notFound = true;
|
||||
}
|
||||
catch (RateLimitExceededException)
|
||||
{
|
||||
isSaturated = true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Exception getting {Id}", id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
notFound = true;
|
||||
}
|
||||
|
||||
//var isSaturated = _twitterUserService.IsUserApiRateLimited();
|
||||
var dbUser = await _twitterUserDal.GetTwitterUserAsync(id);
|
||||
|
||||
var acceptHeaders = Request.Headers["Accept"];
|
||||
if (acceptHeaders.Any())
|
||||
|
@ -71,14 +112,17 @@ namespace BirdsiteLive.Controllers
|
|||
var r = acceptHeaders.First();
|
||||
if (r.Contains("application/activity+json"))
|
||||
{
|
||||
if (user == null) return NotFound();
|
||||
var apUser = _userService.GetUser(user);
|
||||
if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
|
||||
if (notFound) return NotFound();
|
||||
if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 };
|
||||
var apUser = _userService.GetUser(user, dbUser);
|
||||
var jsonApUser = JsonConvert.SerializeObject(apUser);
|
||||
return Content(jsonApUser, "application/activity+json; charset=utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
if (user == null) return View("UserNotFound");
|
||||
if (isSaturated) return View("ApiSaturated");
|
||||
if (notFound) return View("UserNotFound");
|
||||
|
||||
var displayableUser = new DisplayTwitterUser
|
||||
{
|
||||
|
@ -88,11 +132,21 @@ namespace BirdsiteLive.Controllers
|
|||
Url = user.Url,
|
||||
ProfileImageUrl = user.ProfileImageUrl,
|
||||
Protected = user.Protected,
|
||||
|
||||
InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}",
|
||||
|
||||
InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
|
||||
MovedTo = dbUser?.MovedTo,
|
||||
MovedToAcct = dbUser?.MovedToAcct,
|
||||
Deleted = dbUser?.Deleted ?? false,
|
||||
};
|
||||
return View(displayableUser);
|
||||
}
|
||||
|
||||
[Route("/users/{id}/remote_follow")]
|
||||
public async Task<IActionResult> IndexRemoteFollow(string id)
|
||||
{
|
||||
return Redirect($"/users/{id}");
|
||||
}
|
||||
|
||||
[Route("/@{id}/{statusId}")]
|
||||
[Route("/users/{id}/statuses/{statusId}")]
|
||||
|
@ -127,45 +181,74 @@ namespace BirdsiteLive.Controllers
|
|||
[HttpPost]
|
||||
public async Task<IActionResult> Inbox()
|
||||
{
|
||||
var r = Request;
|
||||
using (var reader = new StreamReader(Request.Body))
|
||||
try
|
||||
{
|
||||
var body = await reader.ReadToEndAsync();
|
||||
//System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body);
|
||||
|
||||
var activity = ApDeserializer.ProcessActivity(body);
|
||||
// Do something
|
||||
var signature = r.Headers["Signature"].First();
|
||||
|
||||
switch (activity?.type)
|
||||
var r = Request;
|
||||
using (var reader = new StreamReader(Request.Body))
|
||||
{
|
||||
case "Follow":
|
||||
var body = await reader.ReadToEndAsync();
|
||||
|
||||
_logger.LogTrace("User Inbox: {Body}", body);
|
||||
//System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body);
|
||||
|
||||
var activity = ApDeserializer.ProcessActivity(body);
|
||||
var signature = r.Headers["Signature"].First();
|
||||
|
||||
switch (activity?.type)
|
||||
{
|
||||
case "Follow":
|
||||
{
|
||||
var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path,
|
||||
r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow, body);
|
||||
r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers),
|
||||
activity as ActivityFollow, body);
|
||||
if (succeeded) return Accepted();
|
||||
else return Unauthorized();
|
||||
}
|
||||
case "Undo":
|
||||
if (activity is ActivityUndoFollow)
|
||||
case "Undo":
|
||||
if (activity is ActivityUndoFollow)
|
||||
{
|
||||
var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path,
|
||||
r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers),
|
||||
activity as ActivityUndoFollow, body);
|
||||
if (succeeded) return Accepted();
|
||||
else return Unauthorized();
|
||||
}
|
||||
|
||||
return Accepted();
|
||||
case "Delete":
|
||||
{
|
||||
var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path,
|
||||
r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityUndoFollow, body);
|
||||
var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path,
|
||||
r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers),
|
||||
activity as ActivityDelete, body);
|
||||
if (succeeded) return Accepted();
|
||||
else return Unauthorized();
|
||||
}
|
||||
return Accepted();
|
||||
default:
|
||||
return Accepted();
|
||||
default:
|
||||
return Accepted();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Accepted();
|
||||
catch (FollowerIsGoneException) //TODO: check if user in DB
|
||||
{
|
||||
return Accepted();
|
||||
}
|
||||
catch (UserNotFoundException)
|
||||
{
|
||||
return NotFound();
|
||||
}
|
||||
catch (UserHasBeenSuspendedException)
|
||||
{
|
||||
return NotFound();
|
||||
}
|
||||
catch (RateLimitExceededException)
|
||||
{
|
||||
return new ObjectResult("Too Many Requests") { StatusCode = 429 };
|
||||
}
|
||||
}
|
||||
|
||||
[Route("/users/{id}/followers")]
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> Followers(string id)
|
||||
public IActionResult Followers(string id)
|
||||
{
|
||||
var r = Request.Headers["Accept"].First();
|
||||
if (!r.Contains("application/activity+json")) return NotFound();
|
||||
|
@ -177,10 +260,5 @@ namespace BirdsiteLive.Controllers
|
|||
var jsonApUser = JsonConvert.SerializeObject(followers);
|
||||
return Content(jsonApUser, "application/activity+json; charset=utf-8");
|
||||
}
|
||||
|
||||
private Dictionary<string, string> RequestHeaders(IHeaderDictionary header)
|
||||
{
|
||||
return header.ToDictionary<KeyValuePair<string, StringValues>, string, string>(h => h.Key.ToLowerInvariant(), h => h.Value);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -7,10 +7,12 @@ using BirdsiteLive.ActivityPub.Converters;
|
|||
using BirdsiteLive.Common.Regexes;
|
||||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.DAL.Contracts;
|
||||
using BirdsiteLive.Domain.Repository;
|
||||
using BirdsiteLive.Models;
|
||||
using BirdsiteLive.Models.WellKnownModels;
|
||||
using BirdsiteLive.Twitter;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace BirdsiteLive.Controllers
|
||||
|
@ -18,15 +20,19 @@ namespace BirdsiteLive.Controllers
|
|||
[ApiController]
|
||||
public class WellKnownController : ControllerBase
|
||||
{
|
||||
private readonly IModerationRepository _moderationRepository;
|
||||
private readonly ITwitterUserService _twitterUserService;
|
||||
private readonly ITwitterUserDal _twitterUserDal;
|
||||
private readonly InstanceSettings _settings;
|
||||
|
||||
private readonly ILogger<WellKnownController> _logger;
|
||||
|
||||
#region Ctor
|
||||
public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal)
|
||||
public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository, ILogger<WellKnownController> logger)
|
||||
{
|
||||
_twitterUserService = twitterUserService;
|
||||
_twitterUserDal = twitterUserDal;
|
||||
_moderationRepository = moderationRepository;
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
}
|
||||
#endregion
|
||||
|
@ -58,6 +64,7 @@ namespace BirdsiteLive.Controllers
|
|||
{
|
||||
var version = System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3);
|
||||
var twitterUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
|
||||
var isOpenRegistration = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower) != ModerationTypeEnum.WhiteListing;
|
||||
|
||||
if (id == "2.0")
|
||||
{
|
||||
|
@ -81,7 +88,7 @@ namespace BirdsiteLive.Controllers
|
|||
{
|
||||
"activitypub"
|
||||
},
|
||||
openRegistrations = false,
|
||||
openRegistrations = isOpenRegistration,
|
||||
services = new Models.WellKnownModels.Services()
|
||||
{
|
||||
inbound = new object[0],
|
||||
|
@ -117,7 +124,7 @@ namespace BirdsiteLive.Controllers
|
|||
{
|
||||
"activitypub"
|
||||
},
|
||||
openRegistrations = false,
|
||||
openRegistrations = isOpenRegistration,
|
||||
services = new Models.WellKnownModels.Services()
|
||||
{
|
||||
inbound = new object[0],
|
||||
|
@ -137,30 +144,54 @@ namespace BirdsiteLive.Controllers
|
|||
[Route("/.well-known/webfinger")]
|
||||
public IActionResult Webfinger(string resource = null)
|
||||
{
|
||||
var acct = resource.Split("acct:")[1].Trim();
|
||||
if (string.IsNullOrWhiteSpace(resource))
|
||||
return BadRequest();
|
||||
|
||||
string name = null;
|
||||
string domain = null;
|
||||
|
||||
var splitAcct = acct.Split('@', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (resource.StartsWith("acct:"))
|
||||
{
|
||||
var acct = resource.Split("acct:")[1].Trim();
|
||||
var splitAcct = acct.Split('@', StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
var atCount = acct.Count(x => x == '@');
|
||||
if (atCount == 1 && acct.StartsWith('@'))
|
||||
{
|
||||
name = splitAcct[1];
|
||||
var atCount = acct.Count(x => x == '@');
|
||||
if (atCount == 1 && acct.StartsWith('@'))
|
||||
{
|
||||
name = splitAcct[1];
|
||||
}
|
||||
else if (atCount == 1 || atCount == 2)
|
||||
{
|
||||
name = splitAcct[0];
|
||||
domain = splitAcct[1];
|
||||
}
|
||||
else
|
||||
{
|
||||
return BadRequest();
|
||||
}
|
||||
}
|
||||
else if (atCount == 1 || atCount == 2)
|
||||
else if (resource.StartsWith("https://"))
|
||||
{
|
||||
name = splitAcct[0];
|
||||
domain = splitAcct[1];
|
||||
try
|
||||
{
|
||||
name = resource.Split('/').Last().Trim();
|
||||
domain = resource.Split("https://", StringSplitOptions.RemoveEmptyEntries)[0].Split('/')[0].Trim();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error parsing {Resource}", resource);
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return BadRequest();
|
||||
_logger.LogError("Error parsing {Resource}", resource);
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
// Ensure lowercase
|
||||
name = name.ToLowerInvariant();
|
||||
domain = domain?.ToLowerInvariant();
|
||||
|
||||
// Ensure valid username
|
||||
// https://help.twitter.com/en/managing-your-account/twitter-username-rules
|
||||
|
@ -170,9 +201,27 @@ namespace BirdsiteLive.Controllers
|
|||
if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain)
|
||||
return NotFound();
|
||||
|
||||
var user = _twitterUserService.GetUser(name);
|
||||
if (user == null)
|
||||
try
|
||||
{
|
||||
_twitterUserService.GetUser(name);
|
||||
}
|
||||
catch (UserNotFoundException)
|
||||
{
|
||||
return NotFound();
|
||||
}
|
||||
catch (UserHasBeenSuspendedException)
|
||||
{
|
||||
return NotFound();
|
||||
}
|
||||
catch (RateLimitExceededException)
|
||||
{
|
||||
return new ObjectResult("Too Many Requests") { StatusCode = 429 };
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Exception getting {Name}", name);
|
||||
throw;
|
||||
}
|
||||
|
||||
var actorUrl = UrlFactory.GetActorUrl(_settings.Domain, name);
|
||||
|
||||
|
|
|
@ -0,0 +1,82 @@
|
|||
using BirdsiteLive.Common.Settings;
|
||||
using BirdsiteLive.Domain.Tools;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BirdsiteLive.Middlewares
|
||||
{
|
||||
public class IpWhitelistingMiddleware
|
||||
{
|
||||
private readonly RequestDelegate _next;
|
||||
private readonly ILogger<IpWhitelistingMiddleware> _logger;
|
||||
private readonly InstanceSettings _instanceSettings;
|
||||
private readonly byte[][] _safelist;
|
||||
private readonly bool _ipWhitelistingSet;
|
||||
|
||||
public IpWhitelistingMiddleware(
|
||||
RequestDelegate next,
|
||||
ILogger<IpWhitelistingMiddleware> logger,
|
||||
InstanceSettings instanceSettings)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(instanceSettings.IpWhiteListing))
|
||||
{
|
||||
var ips = PatternsParser.Parse(instanceSettings.IpWhiteListing);
|
||||
_safelist = new byte[ips.Length][];
|
||||
for (var i = 0; i < ips.Length; i++)
|
||||
{
|
||||
_safelist[i] = IPAddress.Parse(ips[i]).GetAddressBytes();
|
||||
}
|
||||
_ipWhitelistingSet = true;
|
||||
}
|
||||
|
||||
_next = next;
|
||||
_logger = logger;
|
||||
_instanceSettings = instanceSettings;
|
||||
}
|
||||
|
||||
public async Task Invoke(HttpContext context)
|
||||
{
|
||||
if (_ipWhitelistingSet)
|
||||
{
|
||||
var remoteIp = context.Connection.RemoteIpAddress;
|
||||
|
||||
if (_instanceSettings.EnableXRealIpHeader)
|
||||
{
|
||||
var forwardedIp = context.Request.Headers.FirstOrDefault(x => x.Key == "X-Real-IP").Value
|
||||
.ToString();
|
||||
if (!string.IsNullOrWhiteSpace(forwardedIp))
|
||||
{
|
||||
_logger.LogDebug("Redirected IP address detected");
|
||||
remoteIp = IPAddress.Parse(forwardedIp);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Request from Remote IP address: {RemoteIp}", remoteIp);
|
||||
|
||||
var bytes = remoteIp.GetAddressBytes();
|
||||
var badIp = true;
|
||||
foreach (var address in _safelist)
|
||||
{
|
||||
if (address.SequenceEqual(bytes))
|
||||
{
|
||||
badIp = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (badIp)
|
||||
{
|
||||
_logger.LogWarning("Forbidden Request from Remote IP address: {RemoteIp}", remoteIp);
|
||||
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await _next.Invoke(context);
|
||||
}
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue