Merge branch 'develop' into topic_backwhitelisting

INSTALLATION.md

@@ -138,11 +138,11 @@ sudo ufw status
 
 You should now have an up and running BirdsiteLIVE instance!
 
-## Upgrading 
+## Updating 
 
 Make sure your data belong outside the containers before migrating (set by default). 
 
-To upgrade your installation to the latest release:
+To update your installation to the latest release:
 
 ```
 # Edit `docker-compose.yml` to update the version, if you have one specified
@@ -152,6 +152,37 @@ docker-compose pull
 docker-compose up -d
 ```
 
+## Auto-Updating
+
+To set auto-updates on your deployment, add to the `docker-compose.yml` file this section:
+
+```diff
+version: "3"
+
+networks:
+    birdsitelivenetwork:
+        external: false
+
+services:
+    server:
+        image: nicolasconstant/birdsitelive:latest
+        [...]
+
+    db:
+        image: postgres:9.6
+        [...]
+        
++   watchtower:
++       image: containrrr/watchtower
++       restart: always
++       container_name: watchtower
++       environment:
++           - WATCHTOWER_CLEANUP=true
++       volumes:
++           - /var/run/docker.sock:/var/run/docker.sock
++       command: --interval 300
+```
+
 ## More options 
 
-You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)
+You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)

src/BirdsiteLive.Common/Regexes/HeaderRegexes.cs

@@ -0,0 +1,9 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class HeaderRegexes
+    {
+        public static readonly Regex HeaderSignature = new Regex(@"^([a-zA-Z0-9]+)=""(.+)""$");
+    }
+}

src/BirdsiteLive.Common/Regexes/UrlRegexes.cs

@@ -4,6 +4,6 @@ namespace BirdsiteLive.Common.Regexes
 {
     public class UrlRegexes
     {
-        public static readonly Regex Url = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
+        public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
     }
 }

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -40,7 +40,7 @@ namespace BirdsiteLive.Domain
         public async Task<Actor> GetUser(string objectId)
         {
             var httpClient = _httpClientFactory.CreateClient();
-            httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
+            httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json");
             var result = await httpClient.GetAsync(objectId);
             var content = await result.Content.ReadAsStringAsync();
             return JsonConvert.DeserializeObject<Actor>(content);

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -44,11 +44,8 @@ namespace BirdsiteLive.Domain.Tools
             var urlMatch = UrlRegexes.Url.Matches(messageContent);
             foreach (Match m in urlMatch)
             {
-                var url = m.ToString().Replace("\n", string.Empty).Trim();
+                var url = m.Groups[2].ToString();
-
+                var protocol = m.Groups[3].ToString();
-                var protocol = "https://";
-                if (url.StartsWith("http://")) protocol = "http://";
-                else if (url.StartsWith("ftp://")) protocol = "ftp://";
 
                 var truncatedUrl = url.Replace(protocol, string.Empty);
 
@@ -67,8 +64,8 @@ namespace BirdsiteLive.Domain.Tools
                     secondPart = truncatedUrl.Substring(30);
                 }
 
-                messageContent = Regex.Replace(messageContent, m.ToString(),
+                messageContent = Regex.Replace(messageContent, Regex.Escape(m.ToString()),
-                    $@" <a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
+                    $@"{m.Groups[1]}<a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
             }
 
             // Extract Hashtags

src/BirdsiteLive.Domain/UserService.cs

@@ -7,6 +7,7 @@ using System.Text;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
 using BirdsiteLive.Domain.BusinessUseCases;
@@ -277,33 +278,24 @@ namespace BirdsiteLive.Domain
             var signature_header = new Dictionary<string, string>();
             foreach (var signature in signatures)
             {
-                var splitSig = signature.Replace("\"", string.Empty).Split('=');
+                var m = HeaderRegexes.HeaderSignature.Match(signature);
-                signature_header.Add(splitSig[0], splitSig[1]);
+                signature_header.Add(m.Groups[1].ToString(), m.Groups[2].ToString());
             }
 
-            signature_header["signature"] = signature_header["signature"] + "==";
-
             var key_id = signature_header["keyId"];
             var headers = signature_header["headers"];
             var algorithm = signature_header["algorithm"];
             var sig = Convert.FromBase64String(signature_header["signature"]);
 
-
+            // Retrieve User
             var remoteUser = await _activityPubService.GetUser(actor);
 
+            // Prepare Key data
             var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
             toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", "");
             var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode));
 
             var toSign = new StringBuilder();
-            //var comparisonString = headers.Split(' ').Select(signed_header_name =>
-            //{
-            //    if (signed_header_name == "(request-target)")
-            //        return "(request-target): post /inbox";
-            //    else
-            //        return $"{signed_header_name}: {r.Headers[signed_header_name.ToUpperInvariant()]}";
-            //});
-
             foreach (var headerKey in headers.Split(' '))
             {
                 if (headerKey == "(request-target)") toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n");
@@ -311,21 +303,13 @@ namespace BirdsiteLive.Domain
             }
             toSign.Remove(toSign.Length - 1, 1);
 
-            //var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode));
+            // Import key
-
+            var key = new RSACryptoServiceProvider();
-            //new RSACryptoServiceProvider(keyId.publicKey.publicKeyPem);
+            var rsaKeyInfo = key.ExportParameters(false);
-
+            rsaKeyInfo.Modulus = Convert.FromBase64String(toDecode);
-            //Create a new instance of RSACryptoServiceProvider.
+            key.ImportParameters(rsaKeyInfo);
-            RSACryptoServiceProvider key = new RSACryptoServiceProvider();
-
-            //Get an instance of RSAParameters from ExportParameters function.
-            RSAParameters RSAKeyInfo = key.ExportParameters(false);
-
-            //Set RSAKeyInfo to the public key values.
-            RSAKeyInfo.Modulus = Convert.FromBase64String(toDecode);
-
-            key.ImportParameters(RSAKeyInfo);
 
+            // Trust and Verify
             var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
 
             return new SignatureValidationResult()

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -68,6 +68,10 @@ namespace BirdsiteLive.Pipeline.Processors
         {
             var tweets = new ExtractedTweet[0];
 
+            // Don't retrieve TL if protected
+            var userView = _twitterUserService.GetUser(user.Acct);
+            if (userView == null || userView.Protected) return tweets;
+
             try
             {
                 if (user.LastTweetPostedId == -1)

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -5,27 +5,57 @@ using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
+using Microsoft.Extensions.Logging;
 
 namespace BirdsiteLive.Pipeline.Processors
 {
     public class SaveProgressionProcessor : ISaveProgressionProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
-        
+        private readonly ILogger<SaveProgressionProcessor> _logger;
+
         #region Ctor
-        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal)
+        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
+            _logger = logger;
         }
         #endregion
 
         public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
         {
-            var userId = userWithTweetsToSync.User.Id;
+            try
-            var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
+            {
-            var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min();
+                if (userWithTweetsToSync.Tweets.Length == 0)
-            var now = DateTime.UtcNow;
+                {
-            await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
+                    _logger.LogWarning("No tweets synchronized");
+                    return;
+                }
+                if(userWithTweetsToSync.Followers.Length == 0)
+                {
+                    _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
+                    return;
+                }
+            
+                var userId = userWithTweetsToSync.User.Id;
+                var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
+            
+                if (followingSyncStatuses.Count == 0)
+                {
+                    _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
+                    return;
+                }
+
+                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
+                var minimumSync = followingSyncStatuses.Min();
+                var now = DateTime.UtcNow;
+                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
+                throw;
+            }
         }
     }
 }

src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs

@@ -57,7 +57,7 @@ namespace BirdsiteLive.Twitter.Extractors
             var message = tweet.FullText;
             var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
             
-            if (tweet.IsRetweet && tweet.QuotedStatusId == null && message.StartsWith("RT") && tweet.RetweetedTweet != null)
+            if (tweet.IsRetweet && message.StartsWith("RT") && tweet.RetweetedTweet != null)
             {
                 message = tweet.RetweetedTweet.FullText;
                 tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct();

src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs

@@ -87,7 +87,7 @@ namespace BirdsiteLive.Statistics.Domain
                 UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0,
                 UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0,
                 UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0,
-                UserCallsMax = 900,
+                UserCallsMax = 300,
                 TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0,
                 TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0,
                 TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0,

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -64,7 +64,7 @@ namespace BirdsiteLive.Twitter
                 Name = user.Name,
                 Description = description,
                 Url = $"https://twitter.com/{username}",
-                ProfileImageUrl = user.ProfileImageUrlFullSize,
+                ProfileImageUrl = user.ProfileImageUrlFullSize.Replace("http://", "https://"),
                 ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,
                 ProfileBannerURL = user.ProfileBannerURL,
                 Protected = user.Protected

src/BirdsiteLive.sln

@@ -44,6 +44,7 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation", "BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj", "{4BE541AC-8A93-4FA3-98AC-956CC2D5B748}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Moderation.Tests", "Tests\BirdsiteLive.Moderation.Tests\BirdsiteLive.Moderation.Tests.csproj", "{0A311BF3-4FD9-4303-940A-A3778890561C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -119,6 +120,10 @@ Global
 		{0A311BF3-4FD9-4303-940A-A3778890561C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0A311BF3-4FD9-4303-940A-A3778890561C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0A311BF3-4FD9-4303-940A-A3778890561C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -140,6 +145,7 @@ Global
 		{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
 		{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
+		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -4,7 +4,7 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.13.0</Version>
+    <Version>0.14.5</Version>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive/Services/FederationService.cs

@@ -15,21 +15,31 @@ namespace BirdsiteLive.Services
         private readonly IDatabaseInitializer _databaseInitializer;
         private readonly IModerationPipeline _moderationPipeline;
         private readonly IStatusPublicationPipeline _statusPublicationPipeline;
+        private readonly IHostApplicationLifetime _applicationLifetime;
 
         #region Ctor
-        public FederationService(IDatabaseInitializer databaseInitializer, IModerationPipeline moderationPipeline, IStatusPublicationPipeline statusPublicationPipeline)
+        public FederationService(IDatabaseInitializer databaseInitializer, IModerationPipeline moderationPipeline, IStatusPublicationPipeline statusPublicationPipeline, IHostApplicationLifetime applicationLifetime)
         {
             _databaseInitializer = databaseInitializer;
             _moderationPipeline = moderationPipeline;
             _statusPublicationPipeline = statusPublicationPipeline;
+            _applicationLifetime = applicationLifetime;
         }
         #endregion
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
-            await _databaseInitializer.InitAndMigrateDbAsync();
+            try
-            await _moderationPipeline.ApplyModerationSettingsAsync();
+            {
-            await _statusPublicationPipeline.ExecuteAsync(stoppingToken);
+                await _databaseInitializer.InitAndMigrateDbAsync();
+                await _moderationPipeline.ApplyModerationSettingsAsync();
+                await _statusPublicationPipeline.ExecuteAsync(stoppingToken);
+            }
+            finally
+            {
+                await Task.Delay(1000 * 30);
+                _applicationLifetime.StopApplication();
+            }
         }
     }
 }

src/BirdsiteLive/appsettings.json

@@ -19,8 +19,8 @@
     "Domain": "domain.name",
     "AdminEmail": "me@domain.name",
     "ResolveMentionsInProfiles": true,
-    "PublishReplies": false,
+    "PublishReplies": false, 
-    "MaxUsersCapacity": 1400
+    "MaxUsersCapacity": 800 
   },
   "Db": {
     "Type": "postgres",

src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
+    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
+  </ItemGroup>
+
+</Project>

src/Tests/BirdsiteLive.Common.Tests/Regexes/HeaderRegexTests.cs

@@ -0,0 +1,53 @@
+using BirdsiteLive.Common.Regexes;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace BirdsiteLive.Common.Tests
+{
+    [TestClass]
+    public class HeaderRegexTests
+    {
+        [TestMethod]
+        public void KeyId_Test()
+        {
+            var input = @"keyId=""https://misskey.tdl/users/8hwf6zy2k1#main-key""";
+
+            Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input));
+            var result = HeaderRegexes.HeaderSignature.Match(input);
+            Assert.AreEqual("keyId", result.Groups[1].ToString());
+            Assert.AreEqual("https://misskey.tdl/users/8hwf6zy2k1#main-key", result.Groups[2].ToString());
+        }
+
+        [TestMethod]
+        public void Algorithm_Test()
+        {
+            var input = @"algorithm=""rsa-sha256""";
+
+            Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input));
+            var result = HeaderRegexes.HeaderSignature.Match(input);
+            Assert.AreEqual("algorithm", result.Groups[1].ToString());
+            Assert.AreEqual("rsa-sha256", result.Groups[2].ToString());
+        }
+
+        [TestMethod]
+        public void Target_Test()
+        {
+            var input = @"headers=""(request-target) date host digest""";
+
+            Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input));
+            var result = HeaderRegexes.HeaderSignature.Match(input);
+            Assert.AreEqual("headers", result.Groups[1].ToString());
+            Assert.AreEqual("(request-target) date host digest", result.Groups[2].ToString());
+        }
+
+        [TestMethod]
+        public void Signature_Test()
+        {
+            var input = @"signature=""ISxp4HYhkGc83SsCC3zNZMVKBota7HqgWVg6KwaQVTQcUqt+UsWXPxB0XPhaYkqLnH3hJ+KVMdoEn3rbzcw8XZpjVt48o9OAKd0rsEZYkLoERnnFFhEw0GmVDEhdoU7gyoeOreWGsIca6Pf7TC0vGTtqez31zmvoeXvxHgqRhWQvlZM/ovFR2xN+vhmF7rZdkd6UaKOzy21K8B/Q84J7PWdbJ8i0rKieVPDIuTCy5B0iQpgs1TMaz6xKZR/KVzAr207m9Gkku2gnJ4YZHFuoa2ct5M5AtIPMPCsWTU8yaimTkPdNNezSOKV5a7T55HSvFeopLNcQKsWNMioKGpZP5hCIRKNk0Ekx0yDReE6xF/qliT7eSAGVJ/6sLQjBpBFMPKBNOrYTxueBJGtISjCZlxaIyTtJ1ErNuCrKHGjImpNvvJzTJOtu+vWnjTcUJL7N1Mw7PEreCZrNUyNuAldDWSMAFuD4HVA3+KZjpWCfjAbyelzVy2gs96CyE56o9FqJEaM5XVQhsMTpa8OSHdr2QZtKYw7Wng0d8vmbKEX1pdTVeEIhi4M9js39ZdzB4mb8JXSBE/GA6PoE5s+oH3+GoufzJYINCpk0Ulwo9g7HKm9NATnwEZZPq4NKto5mSYZKYRtqjZaa8lIALNhdvzv2+8+ifPLHlOigAUVqoupd9Aq=""";
+
+            Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input));
+            var result = HeaderRegexes.HeaderSignature.Match(input);
+            Assert.AreEqual("signature", result.Groups[1].ToString());
+            Assert.AreEqual("ISxp4HYhkGc83SsCC3zNZMVKBota7HqgWVg6KwaQVTQcUqt+UsWXPxB0XPhaYkqLnH3hJ+KVMdoEn3rbzcw8XZpjVt48o9OAKd0rsEZYkLoERnnFFhEw0GmVDEhdoU7gyoeOreWGsIca6Pf7TC0vGTtqez31zmvoeXvxHgqRhWQvlZM/ovFR2xN+vhmF7rZdkd6UaKOzy21K8B/Q84J7PWdbJ8i0rKieVPDIuTCy5B0iQpgs1TMaz6xKZR/KVzAr207m9Gkku2gnJ4YZHFuoa2ct5M5AtIPMPCsWTU8yaimTkPdNNezSOKV5a7T55HSvFeopLNcQKsWNMioKGpZP5hCIRKNk0Ekx0yDReE6xF/qliT7eSAGVJ/6sLQjBpBFMPKBNOrYTxueBJGtISjCZlxaIyTtJ1ErNuCrKHGjImpNvvJzTJOtu+vWnjTcUJL7N1Mw7PEreCZrNUyNuAldDWSMAFuD4HVA3+KZjpWCfjAbyelzVy2gs96CyE56o9FqJEaM5XVQhsMTpa8OSHdr2QZtKYw7Wng0d8vmbKEX1pdTVeEIhi4M9js39ZdzB4mb8JXSBE/GA6PoE5s+oH3+GoufzJYINCpk0Ulwo9g7HKm9NATnwEZZPq4NKto5mSYZKYRtqjZaa8lIALNhdvzv2+8+ifPLHlOigAUVqoupd9Aq=", result.Groups[2].ToString());
+        }
+    }
+}

src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs

@@ -136,7 +136,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
         }
 
         [TestMethod]
-        public void Extract_MultiUrls__Test()
+        public void Extract_MultiUrls_Test()
         {
             #region Stubs
             var message = $"https://t.co/L8BpyHgg25 Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content";
@@ -160,6 +160,82 @@ namespace BirdsiteLive.Domain.Tests.Tools
             #endregion
         }
 
+        [TestMethod]
+        public void Extract_SmallUrl_Test()
+        {
+            #region Stubs
+            var message = @"🚀 test http://GOV.UK date 🎉 data http://GOV.UK woopsi.";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            Assert.AreEqual(@"🚀 test <a href=""http://GOV.UK"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">http://</span><span class=""ellipsis"">GOV.UK</span><span class=""invisible""></span></a> date 🎉 data <a href=""http://GOV.UK"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">http://</span><span class=""ellipsis"">GOV.UK</span><span class=""invisible""></span></a> woopsi.", result.content);
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_SmallUrl_2_Test()
+        {
+            #region Stubs
+            var message = @"🚀http://GOV.UK";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            Assert.AreEqual(@"🚀<a href=""http://GOV.UK"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">http://</span><span class=""ellipsis"">GOV.UK</span><span class=""invisible""></span></a>", result.content);
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_SmallUrl_3_Test()
+        {
+            #region Stubs
+            var message = @"🚀http://GOV.UK.";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            Assert.AreEqual(@"🚀<a href=""http://GOV.UK"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">http://</span><span class=""ellipsis"">GOV.UK</span><span class=""invisible""></span></a>.", result.content);
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_UrlRegexChars_Test()
+        {
+            #region Stubs
+            var message = @"🐣 juniors & tech(http://tech.guru maker)";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            Assert.AreEqual(@"🐣 juniors & tech(<a href=""http://tech.guru"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">http://</span><span class=""ellipsis"">tech.guru</span><span class=""invisible""></span></a> maker)", result.content);
+            #endregion
+        }
+
         [TestMethod]
         public void Extract_SingleHashTag_Test()
         {
@@ -288,7 +364,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
             Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
             #endregion
         }
-
+        
         [TestMethod]
         public void Extract_SingleMentionTag_RT_Test()
         {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs

@@ -62,6 +62,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+            twitterUserServiceMock
+                .Setup(x => x.GetUser(It.Is<string>(y => y == user1.Acct)))
+                .Returns(new TwitterUser {Protected = false});
 
             var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
             #endregion
@@ -126,6 +129,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
 
             var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+            twitterUserServiceMock
+                .Setup(x => x.GetUser(It.Is<string>(y => y == user1.Acct)))
+                .Returns(new TwitterUser { Protected = false });
 
             var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
             #endregion
@@ -193,6 +199,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
 
             var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+            twitterUserServiceMock
+                .Setup(x => x.GetUser(It.Is<string>(y => y == user1.Acct)))
+                .Returns(new TwitterUser { Protected = false });
 
             var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
             #endregion

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -8,6 +8,7 @@ using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors;
 using BirdsiteLive.Twitter.Models;
 using Castle.DynamicProxy.Contributors;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 
@@ -53,6 +54,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 },
                 User = user
             };
+
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
             #endregion
 
             #region Mocks
@@ -67,11 +70,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
             #endregion
 
-            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object);
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
             #region Validations
             twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
             #endregion
         }
 
@@ -129,13 +133,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.IsAny<DateTime>()
                 ))
                 .Returns(Task.CompletedTask);
+
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
             #endregion
 
-            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object);
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
             #region Validations
             twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
             #endregion
         }
 
@@ -201,13 +208,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.IsAny<DateTime>()
                 ))
                 .Returns(Task.CompletedTask);
+
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
             #endregion
 
-            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object);
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
             #region Validations
             twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
             #endregion
         }
     }