Merge pull request #81 from NicolasConstant/develop

0.13.0 PR
2021-02-03 04:54:28 +01:00 · 2021-02-03 04:54:28 +01:00 · 6665402e50
parent 344546536f 2e8313301b
commit 6665402e50
15 changed files with 361 additions and 131 deletions
--- a/src/BirdsiteLive.Common/Regexes/EmojiRegexes.cs
+++ b/src/BirdsiteLive.Common/Regexes/EmojiRegexes.cs
--- a/src/BirdsiteLive.Common/Regexes/HashtagRegexes.cs
+++ b/src/BirdsiteLive.Common/Regexes/HashtagRegexes.cs
@ -0,0 +1,10 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class HashtagRegexes
+    {
+        public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
+        public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
+    }
+}
--- a/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs
+++ b/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs
@ -0,0 +1,9 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class UrlRegexes
+    {
+        public static readonly Regex Url = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?)");
+    }
+}
--- a/src/BirdsiteLive.Common/Regexes/UserRegexes.cs
+++ b/src/BirdsiteLive.Common/Regexes/UserRegexes.cs
@ -0,0 +1,10 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class UserRegexes
+    {
+        public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
+        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
+    }
+}
--- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs
+++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs
--- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
@ -9,6 +9,7 @@ using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;

 namespace BirdsiteLive.Pipeline.Processors
@ -16,13 +17,17 @@ namespace BirdsiteLive.Pipeline.Processors
    public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
    {
        private readonly ITwitterTweetsService _twitterTweetsService;
+        private readonly ICachedTwitterUserService _twitterUserService;
        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly ILogger<RetrieveTweetsProcessor> _logger;

        #region Ctor
-        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal)
+        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
        {
            _twitterTweetsService = twitterTweetsService;
            _twitterUserDal = twitterUserDal;
+            _twitterUserService = twitterUserService;
+            _logger = logger;
        }
        #endregion

@ -61,11 +66,20 @@ namespace BirdsiteLive.Pipeline.Processors

        private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
        {
-            ExtractedTweet[] tweets;
-            if (user.LastTweetPostedId == -1)
-                tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
-            else
-                tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+            var tweets = new ExtractedTweet[0];
+
+            try
+            {
+                if (user.LastTweetPostedId == -1)
+                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
+                else
+                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache", user.Acct, user.LastTweetPostedId);
+                _twitterUserService.PurgeUser(user.Acct);
+            }

            return tweets;
        }
--- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs
+++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs
@ -4,7 +4,12 @@ using Microsoft.Extensions.Caching.Memory;

 namespace BirdsiteLive.Twitter
 {
-    public class CachedTwitterUserService : ITwitterUserService
+    public interface ICachedTwitterUserService : ITwitterUserService
+    {
+        void PurgeUser(string username);
+    }
+
+    public class CachedTwitterUserService : ICachedTwitterUserService
    {
        private readonly ITwitterUserService _twitterService;

@ -38,5 +43,10 @@ namespace BirdsiteLive.Twitter

            return user;
        }
+
+        public void PurgeUser(string username)
+        {
+            _userCache.Remove(username);
+        }
    }
 }
--- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
+++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
@ -23,7 +23,7 @@ namespace BirdsiteLive.Twitter.Extractors
                InReplyToStatusId = tweet.InReplyToStatusId,
                InReplyToAccount = tweet.InReplyToScreenName,
                MessageContent = ExtractMessage(tweet),
-                Media = ExtractMedia(tweet.Media),
+                Media = ExtractMedia(tweet),
                CreatedAt = tweet.CreatedAt.ToUniversalTime(),
                IsReply = tweet.InReplyToUserId != null,
                IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
@ -36,10 +36,17 @@ namespace BirdsiteLive.Twitter.Extractors

        private string ExtractRetweetUrl(ITweet tweet)
        {
-            if (tweet.IsRetweet && tweet.FullText.Contains("https://t.co/"))
+            if (tweet.IsRetweet)
            {
-                var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
-                return $"https://t.co/{retweetId}";
+                if (tweet.RetweetedTweet != null)
+                {
+                    return tweet.RetweetedTweet.Url;
+                }
+                if (tweet.FullText.Contains("https://t.co/"))
+                {
+                    var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
+                    return $"https://t.co/{retweetId}";
+                }
            }

            return null;
@ -47,8 +54,15 @@ namespace BirdsiteLive.Twitter.Extractors

        public string ExtractMessage(ITweet tweet)
        {
-            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
            var message = tweet.FullText;
+            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
+            
+            if (tweet.IsRetweet && tweet.QuotedStatusId == null && message.StartsWith("RT") && tweet.RetweetedTweet != null)
+            {
+                message = tweet.RetweetedTweet.FullText;
+                tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct();
+            }
+
            foreach (var tweetUrl in tweetUrls)
            {
                if(tweet.IsRetweet)
@ -60,8 +74,10 @@ namespace BirdsiteLive.Twitter.Extractors
            if (tweet.QuotedTweet != null) message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
            if (tweet.IsRetweet)
            {
-                if (tweet.RetweetedTweet != null)
+                if (tweet.RetweetedTweet != null && !message.StartsWith("RT"))
                    message = $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{message}";
+                else if (tweet.RetweetedTweet != null && message.StartsWith($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:"))
+                    message = message.Replace($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:", $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}");
                else
                    message = message.Replace("RT", "[{{RT}}]");
            }
@ -73,10 +89,13 @@ namespace BirdsiteLive.Twitter.Extractors
            return message;
        }

-        public ExtractedMedia[] ExtractMedia(List<IMediaEntity> media)
+        public ExtractedMedia[] ExtractMedia(ITweet tweet)
        {
-            var result = new List<ExtractedMedia>();
+            var media = tweet.Media;
+            if (tweet.IsRetweet && tweet.RetweetedTweet != null)
+                media = tweet.RetweetedTweet.Media;

+            var result = new List<ExtractedMedia>();
            foreach (var m in media)
            {
                var mediaUrl = GetMediaUrl(m);
--- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
@ -61,36 +61,30 @@ namespace BirdsiteLive.Twitter
        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
        {
            var tweets = new List<ITweet>();
-            try
+
+            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
+            ExceptionHandler.SwallowWebExceptions = false;
+            TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
+
+            var user = _twitterUserService.GetUser(username);
+            if (user == null || user.Protected) return new ExtractedTweet[0];
+
+            if (fromTweetId == -1)
            {
-                _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-                ExceptionHandler.SwallowWebExceptions = false;
-                TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
-
-                var user = _twitterUserService.GetUser(username);
-                if (user == null || user.Protected) return new ExtractedTweet[0];
-
-                if (fromTweetId == -1)
-                {
-                    var timeline = Timeline.GetUserTimeline(user.Id, nberTweets);
-                    _statisticsHandler.CalledTimelineApi();
-                    if (timeline != null) tweets.AddRange(timeline);
-                }
-                else
-                {
-                    var timelineRequestParameters = new UserTimelineParameters
-                    {
-                        SinceId = fromTweetId,
-                        MaximumNumberOfTweetsToRetrieve = nberTweets
-                    };
-                    var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters);
-                    _statisticsHandler.CalledTimelineApi();
-                    if (timeline != null) tweets.AddRange(timeline);
-                }
+                var timeline = Timeline.GetUserTimeline(user.Id, nberTweets);
+                _statisticsHandler.CalledTimelineApi();
+                if (timeline != null) tweets.AddRange(timeline);
            }
-            catch (Exception e)
+            else
            {
-                _logger.LogError(e, "Error retrieving timeline from {Username}, from {TweetId}", username, fromTweetId);
+                var timelineRequestParameters = new UserTimelineParameters
+                {
+                    SinceId = fromTweetId,
+                    MaximumNumberOfTweetsToRetrieve = nberTweets
+                };
+                var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters);
+                _statisticsHandler.CalledTimelineApi();
+                if (timeline != null) tweets.AddRange(timeline);
            }

            return tweets.Select(_tweetExtractor.Extract).ToArray();
--- a/src/BirdsiteLive/BirdsiteLive.csproj
+++ b/src/BirdsiteLive/BirdsiteLive.csproj
@ -4,7 +4,7 @@
    <TargetFramework>netcoreapp3.1</TargetFramework>
    <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.12.2</Version>
+    <Version>0.13.0</Version>
  </PropertyGroup>

  <ItemGroup>
--- a/src/BirdsiteLive/Controllers/UsersController.cs
+++ b/src/BirdsiteLive/Controllers/UsersController.cs
@ -9,6 +9,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
+using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
@ -28,7 +29,6 @@ namespace BirdsiteLive.Controllers
        private readonly IUserService _userService;
        private readonly IStatusService _statusService;
        private readonly InstanceSettings _instanceSettings;
-        private readonly Regex _twitterAccountRegex = new Regex(@"^[a-zA-Z0-9_]+$");

        #region Ctor
        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService)
@ -62,7 +62,7 @@ namespace BirdsiteLive.Controllers
            // Ensure valid username 
            // https://help.twitter.com/en/managing-your-account/twitter-username-rules
            TwitterUser user = null;
-            if (!string.IsNullOrWhiteSpace(id) && _twitterAccountRegex.IsMatch(id) && id.Length <= 15)
+            if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15)
                user = _twitterUserService.GetUser(id);

            var acceptHeaders = Request.Headers["Accept"];
--- a/src/BirdsiteLive/Controllers/WellKnownController.cs
+++ b/src/BirdsiteLive/Controllers/WellKnownController.cs
@ -4,6 +4,7 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Models;
@ -20,7 +21,6 @@ namespace BirdsiteLive.Controllers
        private readonly ITwitterUserService _twitterUserService;
        private readonly ITwitterUserDal _twitterUserDal;
        private readonly InstanceSettings _settings;
-        private readonly Regex _twitterAccountRegex = new Regex(@"^[a-zA-Z0-9_]+$");

        #region Ctor
        public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal)
@ -164,7 +164,7 @@ namespace BirdsiteLive.Controllers

            // Ensure valid username 
            // https://help.twitter.com/en/managing-your-account/twitter-username-rules
-            if (string.IsNullOrWhiteSpace(name) || !_twitterAccountRegex.IsMatch(name) || name.Length > 15 )
+            if (string.IsNullOrWhiteSpace(name) || !UserRegexes.TwitterAccount.IsMatch(name) || name.Length > 15 )
                return NotFound();

            if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain)
--- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs
+++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs
@ -3,7 +3,9 @@ using System.Linq;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;

 namespace BirdsiteLive.Domain.Tests.Tools
 {
@ -28,11 +30,16 @@ namespace BirdsiteLive.Domain.Tests.Tools
            #region Stubs
            var message = "Bla.\n\n@Mention blo. https://t.co/pgtrJi9600";
            #endregion
-            
-            var service = new StatusExtractor(_settings);
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.IsTrue(result.content.Contains("Bla."));
            Assert.IsTrue(result.content.Contains("</p><p>"));
            #endregion
@ -45,10 +52,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = "Bla.\n@Mention blo. https://t.co/pgtrJi9600";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.IsTrue(result.content.Contains("Bla."));
            Assert.IsTrue(result.content.Contains("<br/>"));
            #endregion
@ -61,10 +73,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"Bla!{Environment.NewLine}https://t.co/L8BpyHgg25";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(0, result.tags.Length);

            Assert.IsTrue(result.content.Contains("Bla!"));
@ -79,10 +96,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(0, result.tags.Length);

            Assert.IsTrue(result.content.Contains("Bla!"));
@ -97,10 +119,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(0, result.tags.Length);

            Assert.IsTrue(result.content.Contains("Bla!"));
@ -115,10 +142,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"https://t.co/L8BpyHgg25 Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(0, result.tags.Length);

            Assert.IsTrue(result.content.Contains("Bla!"));
@ -132,13 +164,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
        public void Extract_SingleHashTag_Test()
        {
            #region Stubs
-            var message = $"Bla!{Environment.NewLine}#mytag⁠";
+            var message = $"Bla!{Environment.NewLine}#mytag";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
            Assert.AreEqual("#mytag", result.tags.First().name);
            Assert.AreEqual("Hashtag", result.tags.First().type);
@ -153,13 +190,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
        public void Extract_SingleHashTag_AtStart_Test()
        {
            #region Stubs
-            var message = $"#mytag⁠ Bla!";
+            var message = "#mytag Bla!";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
            Assert.AreEqual("#mytag", result.tags.First().name);
            Assert.AreEqual("Hashtag", result.tags.First().type);
@ -174,20 +216,25 @@ namespace BirdsiteLive.Domain.Tests.Tools
        public void Extract_SingleHashTag_SpecialChar_Test()
        {
            #region Stubs
-            var message = $"Bla!{Environment.NewLine}#COVIDー19⁠";
+            var message = $"Bla!{Environment.NewLine}#COVID_19";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
-            Assert.AreEqual("#COVIDー19", result.tags.First().name);
+            Assert.AreEqual("#COVID_19", result.tags.First().name);
            Assert.AreEqual("Hashtag", result.tags.First().type);
-            Assert.AreEqual("https://domain.name/tags/COVIDー19", result.tags.First().href);
+            Assert.AreEqual("https://domain.name/tags/COVID_19", result.tags.First().href);

            Assert.IsTrue(result.content.Contains("Bla!"));
-            Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/COVIDー19"" class=""mention hashtag"" rel=""tag"">#<span>COVIDー19</span></a>"));
+            Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/COVID_19"" class=""mention hashtag"" rel=""tag"">#<span>COVID_19</span></a>"));
            #endregion
        }

@ -195,13 +242,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
        public void Extract_MultiHashTags_Test()
        {
            #region Stubs
-            var message = $"Bla!{Environment.NewLine}#mytag #mytag2 #mytag3⁠{Environment.NewLine}Test #bal Test";
+            var message = $"Bla!{Environment.NewLine}#mytag #mytag2 #mytag3{Environment.NewLine}Test #bal Test";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(4, result.tags.Length);
            Assert.IsTrue(result.content.Contains("Bla!"));
            Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/mytag"" class=""mention hashtag"" rel=""tag"">#<span>mytag</span></a>"));
@ -215,13 +267,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
        public void Extract_SingleMentionTag_Test()
        {
            #region Stubs
-            var message = $"Bla!{Environment.NewLine}@mynickname⁠";
+            var message = $"Bla!{Environment.NewLine}@mynickname";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
            Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
            Assert.AreEqual("Mention", result.tags.First().type);
@ -233,16 +290,74 @@ namespace BirdsiteLive.Domain.Tests.Tools
        }

        [TestMethod]
-        public void Extract_SingleMentionTag_SpecialChar_Test()
+        public void Extract_SingleMentionTag_RT_Test()
        {
            #region Stubs
-            var message = $"Bla!{Environment.NewLine}@my___nickname⁠";
+            var message = $"[RT @mynickname]{Environment.NewLine}Bla!";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
+            Assert.AreEqual(1, result.tags.Length);
+            Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
+            Assert.AreEqual("Mention", result.tags.First().type);
+            Assert.AreEqual("https://domain.name/users/mynickname", result.tags.First().href);
+
+            Assert.IsTrue(result.content.Contains("Bla!"));
+            Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_SingleMentionTag_Dot_Test()
+        {
+            #region Stubs
+            var message = $".@mynickname Bla!{Environment.NewLine}Blo";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            logger.VerifyAll();
+            Assert.AreEqual(1, result.tags.Length);
+            Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
+            Assert.AreEqual("Mention", result.tags.First().type);
+            Assert.AreEqual("https://domain.name/users/mynickname", result.tags.First().href);
+
+            Assert.IsTrue(result.content.Contains("Bla!"));
+            Assert.IsTrue(result.content.Contains("Blo"));
+            Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_SingleMentionTag_SpecialChar_Test()
+        {
+            #region Stubs
+            var message = $"Bla!{Environment.NewLine}@my___nickname";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
            Assert.AreEqual("@my___nickname@domain.name", result.tags.First().name);
            Assert.AreEqual("Mention", result.tags.First().type);
@ -260,10 +375,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"@mynickname Bla!";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
            Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
            Assert.AreEqual("Mention", result.tags.First().type);
@ -281,10 +401,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"Bla!{Environment.NewLine}@mynickname⁠ @mynickname2 @mynickname3{Environment.NewLine}Test @dada Test";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(4, result.tags.Length);
            Assert.IsTrue(result.content.Contains("Bla!"));
            Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
@ -301,10 +426,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
            var message = $"Bla!{Environment.NewLine}@mynickname⁠ #mytag2 @mynickname3{Environment.NewLine}Test @dada #dada Test";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(5, result.tags.Length);
            Assert.IsTrue(result.content.Contains("Bla!"));
            Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
@ -314,8 +444,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
            Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/dada"" class=""mention hashtag"" rel=""tag"">#<span>dada</span></a>"));
            #endregion
        }
-
-
+        
        [TestMethod]
        public void Extract_Emoji_Test()
        {
@ -324,15 +453,20 @@ namespace BirdsiteLive.Domain.Tests.Tools
            //var message = $"tests@mynickname";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
            Assert.IsTrue(result.content.Contains(
-                @"😤 <span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+                @"😤<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));

-            Assert.IsTrue(result.content.Contains(@"😎 😍 🤗 🤩 😘"));
+            Assert.IsTrue(result.content.Contains(@"😎😍🤗🤩😘"));
            #endregion
        }

@ -344,12 +478,17 @@ namespace BirdsiteLive.Domain.Tests.Tools
            //var message = $"tests@mynickname";
            #endregion

-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
            var result = service.Extract(message);

            #region Validations
+            logger.VerifyAll();
            Assert.AreEqual(1, result.tags.Length);
-            Assert.IsTrue(result.content.Equals(@"bla ( <span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span> test)"));
+            Assert.IsTrue(result.content.Equals(@"bla (<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span> test)"));
            #endregion
        }
    }
--- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj
+++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj
@ -14,12 +14,14 @@
    <PackageReference Include="coverlet.collector" Version="1.2.0" />
  </ItemGroup>

-  <ItemGroup>
-    <ProjectReference Include="..\..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
-  </ItemGroup>
-
  <ItemGroup>
    <Folder Include="Tools\" />
  </ItemGroup>

+  <ItemGroup>
+    <ProjectReference Include="..\..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
+    <ProjectReference Include="..\..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
+    <ProjectReference Include="..\..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
+  </ItemGroup>
+
 </Project>
--- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs
+++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs
@ -7,6 +7,7 @@ using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Processors;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;

@ -59,14 +60,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                    It.IsAny<DateTime>()
                ))
                .Returns(Task.CompletedTask);
+
+            var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+
+            var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
            #endregion

-            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
+            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object, twitterUserServiceMock.Object, logger.Object);
            var usersResult = await processor.ProcessAsync(users, CancellationToken.None);

            #region Validations
            twitterServiceMock.VerifyAll();
            twitterUserDalMock.VerifyAll();
+            twitterUserServiceMock.VerifyAll();
+            logger.VerifyAll();

            Assert.AreEqual(0, usersResult.Length);
            #endregion
@ -117,14 +124,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                .Returns(tweets);

            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+
+            var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+
+            var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
            #endregion

-            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
+            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object, twitterUserServiceMock.Object, logger.Object);
            var usersResult = await processor.ProcessAsync(users, CancellationToken.None);

            #region Validations
            twitterServiceMock.VerifyAll();
            twitterUserDalMock.VerifyAll();
+            twitterUserServiceMock.VerifyAll();
+            logger.VerifyAll();
+

            Assert.AreEqual(users.Length, usersResult.Length);
            Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct);
@ -177,14 +191,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                .Returns(tweets);

            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+
+            var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+
+            var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
            #endregion

-            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
+            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object, twitterUserServiceMock.Object, logger.Object);
            var usersResult = await processor.ProcessAsync(users, CancellationToken.None);

            #region Validations
            twitterServiceMock.VerifyAll();
            twitterUserDalMock.VerifyAll();
+            twitterUserServiceMock.VerifyAll();
+            logger.VerifyAll();

            Assert.AreEqual(users.Length, usersResult.Length);
            Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct);