Store notifications. (#180)

mobilizon_reshare/dataclasses/__init__.py

@@ -2,8 +2,12 @@ from mobilizon_reshare.dataclasses.event import _MobilizonEvent
 from mobilizon_reshare.dataclasses.event_publication_status import (
     _EventPublicationStatus,
 )
-from mobilizon_reshare.dataclasses.publication import _EventPublication
+from mobilizon_reshare.dataclasses.publication import (
+    _EventPublication,
+    _PublicationNotification,
+)
 
 EventPublication = _EventPublication
 MobilizonEvent = _MobilizonEvent
 EventPublicationStatus = _EventPublicationStatus
+PublicationNotification = _PublicationNotification

mobilizon_reshare/dataclasses/publication.py

@@ -54,6 +54,11 @@ class RecapPublication(BasePublication):
     events: List[_MobilizonEvent]
 
 
+@dataclass
+class _PublicationNotification(BasePublication):
+    publication: _EventPublication
+
+
 @atomic()
 async def build_publications_for_event(
     event: _MobilizonEvent, publishers: Iterator[str]

mobilizon_reshare/main/publish.py

@@ -23,7 +23,10 @@ from mobilizon_reshare.publishers.coordinators.event_publishing.publish import (
     PublisherCoordinatorReport,
     PublisherCoordinator,
 )
-from mobilizon_reshare.storage.query.write import save_publication_report
+from mobilizon_reshare.storage.query.write import (
+    save_publication_report,
+    save_notification_report,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -31,14 +34,16 @@ logger = logging.getLogger(__name__)
 async def publish_publications(
     publications: list[_EventPublication],
 ) -> PublisherCoordinatorReport:
-    report = PublisherCoordinator(publications).run()
+    publishers_report = PublisherCoordinator(publications).run()
+    await save_publication_report(publishers_report)
 
-    await save_publication_report(report)
-    for publication_report in report.reports:
+    for publication_report in publishers_report.reports:
         if not publication_report.successful:
-            PublicationFailureNotifiersCoordinator(publication_report,).notify_failure()
+            notifiers_report = PublicationFailureNotifiersCoordinator(publication_report,).notify_failure()
+            if notifiers_report:
+                await save_notification_report(notifiers_report)
 
-    return report
+    return publishers_report
 
 
 def perform_dry_run(publications: list[_EventPublication]):

mobilizon_reshare/models/notification.py

@@ -5,10 +5,8 @@ from tortoise.models import Model
 
 
 class NotificationStatus(IntEnum):
-    WAITING = 1
-    FAILED = 2
-    PARTIAL = 3
-    COMPLETED = 4
+    FAILED = 0
+    COMPLETED = 1
 
 
 class Notification(Model):

mobilizon_reshare/publishers/coordinators/__init__.py

@@ -16,7 +16,7 @@ class BasePublicationReport:
 
     def get_failure_message(self):
         return (
-            f"Publication failed with status: {self.status}.\n" f"Reason: {self.reason}"
+            f"Publication failed with status: {self.status.name}.\n" f"Reason: {self.reason}"
         )
 
 
@@ -26,7 +26,7 @@ class BaseCoordinatorReport:
 
     @property
     def successful(self):
-        return all(r.status == PublicationStatus.COMPLETED for r in self.reports)
+        return all(r.successful for r in self.reports)
 
 
 logger = logging.getLogger(__name__)

mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py

@@ -20,7 +20,7 @@ class EventPublicationReport(BasePublicationReport):
             logger.error("Report of failure without reason.", exc_info=True)
 
         return (
-            f"Publication {self.publication.id} failed with status: {self.status}.\n"
+            f"Publication {self.publication.id} failed with status: {self.status.name}.\n"
             f"Reason: {self.reason}\n"
             f"Publisher: {self.publication.publisher.name}\n"
             f"Event: {self.publication.event.name}"

mobilizon_reshare/publishers/coordinators/event_publishing/notify.py

@@ -1,33 +1,92 @@
 from abc import ABC, abstractmethod
-from typing import List
+from dataclasses import dataclass, field
+from typing import List, Optional, Sequence
 
+from mobilizon_reshare.dataclasses import PublicationNotification, EventPublication
+from mobilizon_reshare.models.notification import NotificationStatus
 from mobilizon_reshare.models.publication import PublicationStatus
 from mobilizon_reshare.publishers import get_active_notifiers
-from mobilizon_reshare.publishers.abstract import AbstractPlatform
-from mobilizon_reshare.publishers.coordinators import logger
-from mobilizon_reshare.publishers.coordinators.event_publishing.publish import (
+from mobilizon_reshare.publishers.abstract import (
+    AbstractPlatform,
+)
+from mobilizon_reshare.publishers.coordinators import (
+    logger,
+    BasePublicationReport,
+    BaseCoordinatorReport,
+)
+from mobilizon_reshare.publishers.coordinators.event_publishing import (
     EventPublicationReport,
 )
-from mobilizon_reshare.publishers.platforms.platform_mapping import get_notifier_class
+from mobilizon_reshare.publishers.platforms.platform_mapping import (
+    get_notifier_class,
+    get_formatter_class,
+)
+
+
+@dataclass
+class PublicationNotificationReport(BasePublicationReport):
+    status: NotificationStatus
+    notification: PublicationNotification
+
+    @property
+    def successful(self):
+        return self.status == NotificationStatus.COMPLETED
+
+    def get_failure_message(self):
+        if not self.reason:
+            logger.error("Report of failure without reason.", exc_info=True)
+        return (
+            f"Failed with status: {self.status.name}.\n"
+            f"Reason: {self.reason}\n"
+            f"Publisher: {self.notification.publisher.name}\n"
+            f"Publication: {self.notification.publication.id}"
+        )
+
+
+@dataclass
+class NotifierCoordinatorReport(BaseCoordinatorReport):
+    reports: Sequence[PublicationNotificationReport]
+    notifications: Sequence[PublicationNotification] = field(default_factory=list)
 
 
 class Sender:
-    def __init__(self, message: str, platforms: List[AbstractPlatform] = None):
+    def __init__(
+        self,
+        message: str,
+        publication: EventPublication,
+        platforms: List[AbstractPlatform] = None,
+    ):
         self.message = message
         self.platforms = platforms
+        self.publication = publication
 
-    def send_to_all(self):
+    def send_to_all(self) -> NotifierCoordinatorReport:
+        reports = []
+        notifications = []
         for platform in self.platforms:
+            notification = PublicationNotification(
+                platform, get_formatter_class(platform.name)(), self.publication
+            )
             try:
                 platform.send(self.message)
+                report = PublicationNotificationReport(
+                    NotificationStatus.COMPLETED, self.message, notification
+                )
             except Exception as e:
-                logger.critical(f"Failed to send message:\n{self.message}")
+                msg = f"[{platform.name}] Failed to notify failure of message:\n{self.message}"
+                logger.critical(msg)
                 logger.exception(e)
+                report = PublicationNotificationReport(
+                    NotificationStatus.FAILED, msg, notification
+                )
+            notifications.append(notification)
+            reports.append(report)
+        return NotifierCoordinatorReport(reports=reports, notifications=notifications)
 
 
 class AbstractNotifiersCoordinator(ABC):
     def __init__(
-        self, report: EventPublicationReport, notifiers: List[AbstractPlatform] = None
+        self, report: BasePublicationReport, notifiers: List[AbstractPlatform] = None
     ):
         self.platforms = notifiers or [
             get_notifier_class(notifier)() for notifier in get_active_notifiers()
@@ -44,10 +103,17 @@ class PublicationFailureNotifiersCoordinator(AbstractNotifiersCoordinator):
     Sends a notification of a failure report to the active platforms
     """
 
-    def notify_failure(self):
+    report: EventPublicationReport
+    platforms: List[AbstractPlatform]
+
+    def notify_failure(self) -> Optional[NotifierCoordinatorReport]:
         logger.info("Sending failure notifications")
         if self.report.status == PublicationStatus.FAILED:
-            Sender(self.report.get_failure_message(), self.platforms).send_to_all()
+            return Sender(
+                self.report.get_failure_message(),
+                self.report.publication,
+                self.platforms,
+            ).send_to_all()
 
 
 class PublicationFailureLoggerCoordinator(PublicationFailureNotifiersCoordinator):

mobilizon_reshare/storage/query/read.py

@@ -33,7 +33,7 @@ async def get_all_publishers() -> list[Publisher]:
 
 async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]:
     return (
-        await queryset.prefetch_related("publications__publisher")
+        await queryset.prefetch_related("publications__publisher", "publications__notifications")
         .order_by("begin_datetime")
         .distinct()
     )
@@ -46,6 +46,7 @@ async def prefetch_publication_relations(
         await queryset.prefetch_related(
             "publisher",
             "event",
+            "notifications",
             "event__publications",
             "event__publications__publisher",
         )

mobilizon_reshare/storage/query/write.py

@@ -9,11 +9,15 @@ from mobilizon_reshare.dataclasses.event import (
     get_mobilizon_events_without_publications,
 )
 from mobilizon_reshare.models.event import Event
+from mobilizon_reshare.models.notification import Notification
 from mobilizon_reshare.models.publication import Publication
 from mobilizon_reshare.models.publisher import Publisher
 from mobilizon_reshare.publishers.coordinators.event_publishing import (
     EventPublicationReport,
 )
+from mobilizon_reshare.publishers.coordinators.event_publishing.notify import (
+    NotifierCoordinatorReport,
+)
 from mobilizon_reshare.publishers.coordinators.event_publishing.publish import (
     PublisherCoordinatorReport,
 )
@@ -64,6 +68,24 @@ async def save_publication_report(
         await upsert_publication(publication_report, event)
 
 
+@atomic()
+async def save_notification_report(
+    coordinator_report: NotifierCoordinatorReport,
+) -> None:
+    """
+    Store a notification process outcome
+    """
+    for report in coordinator_report.reports:
+        publisher = await Publisher.filter(name=report.notification.publisher.name).first()
+
+        await Notification.create(
+            publication_id=report.notification.publication.id,
+            target_id=publisher.id,
+            status=report.status,
+            message=report.reason,
+        )
+
+
 @atomic()
 async def create_unpublished_events(
     events_from_mobilizon: Iterable[MobilizonEvent],

sample_settings/docker_web/settings.toml

@@ -2,8 +2,9 @@
 debug = false
 default = true
 local_state_dir = "/var/lib/mobilizon-reshare"
-#db_path = "@format {this.local_state_dir}/events.db"
+#db_url = "@format sqlite://{this.local_state_dir}/events.db"
 db_url = "@format postgres://mobilizon_reshare:mobilizon_reshare@db:5432/mobilizon_reshare"
+locale = "en-uk"
 
 [default.source.mobilizon]
 url="https://some-mobilizon.com/api"
@@ -28,6 +29,15 @@ class = "logging.StreamHandler"
 formatter = "standard"
 stream = "ext://sys.stderr"
 
+[default.logging.handlers.file]
+level = "INFO"
+class = "logging.handlers.RotatingFileHandler"
+formatter = "standard"
+filename = "@format {this.local_state_dir}/mobilizon_reshare.log"
+maxBytes = 52428800
+backupCount = 500
+encoding = "utf8"
+
 [default.logging.root]
-level = "DEBUG"
-handlers = ['console']
+level = "INFO"
+handlers = ['console', 'file']

tests/commands/conftest.py

@@ -8,7 +8,6 @@ import mobilizon_reshare.publishers
 import mobilizon_reshare.storage.query.read
 from mobilizon_reshare.models.publisher import Publisher
 import mobilizon_reshare.main.recap
-from mobilizon_reshare.publishers.coordinators.event_publishing import notify
 from tests import today
 from tests.conftest import event_1, event_0
 
@@ -138,15 +137,41 @@ async def mock_notifier_config(monkeypatch, publisher_class, mock_formatter_clas
         return mock_formatter_class
 
     monkeypatch.setattr(
-        notify, "get_notifier_class", _mock_notifier_class,
+        mobilizon_reshare.publishers.coordinators.event_publishing.notify,
+        "get_notifier_class",
+        _mock_notifier_class,
+    )
+    monkeypatch.setattr(
+        mobilizon_reshare.publishers.coordinators.event_publishing.notify,
+        "get_formatter_class",
+        _mock_format_class,
+    )
+    monkeypatch.setattr(
+        mobilizon_reshare.publishers.coordinators.event_publishing.notify,
+        "get_notifier_class",
+        _mock_notifier_class,
     )
     monkeypatch.setattr(
         mobilizon_reshare.publishers.platforms.platform_mapping,
         "get_formatter_class",
         _mock_format_class,
     )
+    monkeypatch.setattr(
+        mobilizon_reshare.publishers.coordinators.event_publishing.notify,
+        "get_formatter_class",
+        _mock_format_class,
+    )
 
-    monkeypatch.setattr(notify, "get_active_notifiers", _mock_active_notifier)
+    monkeypatch.setattr(
+        mobilizon_reshare.publishers.coordinators.event_publishing.notify,
+        "get_active_notifiers",
+        _mock_active_notifier,
+    )
+    monkeypatch.setattr(
+        mobilizon_reshare.config.notifiers,
+        "get_active_notifiers",
+        lambda s: [],
+    )
 
 
 @pytest.fixture

tests/commands/test_publish.py

@@ -1,13 +1,15 @@
 from logging import DEBUG
+from uuid import UUID
 
 import pytest
 
 from mobilizon_reshare.dataclasses import EventPublicationStatus
 from mobilizon_reshare.dataclasses import MobilizonEvent
 from mobilizon_reshare.main.publish import select_and_publish, publish_event
+from mobilizon_reshare.models.notification import NotificationStatus, Notification
 from mobilizon_reshare.models.event import Event
 from mobilizon_reshare.models.publication import PublicationStatus
-from mobilizon_reshare.storage.query.read import get_all_publications
+from mobilizon_reshare.storage.query.read import get_all_publications, get_event
 from tests.conftest import event_0, event_1
 
 one_unpublished_event_specification = {
@@ -112,3 +114,50 @@ async def test_publish_event(
         assert len(publications) == len(expected)
         assert all(p.status == PublicationStatus.COMPLETED for p in publications)
         assert {p.publisher.name for p in publications} == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "publisher_class", [pytest.lazy_fixture("mock_publisher_invalid_class")]
+)
+async def test_notify_publisher_failure(
+    caplog,
+    mock_publisher_config,
+    message_collector,
+    generate_models,
+    mock_notifier_config,
+    command_config,
+):
+    await generate_models(one_unpublished_event_specification)
+
+    with caplog.at_level(DEBUG):
+        # calling the publish command
+        result = await select_and_publish(command_config)
+
+        assert not result.successful
+        assert len(result.reports) == 1
+        assert result.reports[0].published_content is None
+
+        # since the db contains at least one event, this has to be picked and published
+        event_model = await get_event(UUID(int=0))
+        # it should create a publication for each publisher and a notification for each notifier
+        publications = event_model.publications
+        assert len(publications) == 1, publications
+        publication = publications[0]
+        notifications: list[Notification] = list(publications[0].notifications)
+        assert len(notifications) == 2, notifications
+
+        # all the publications for event should be saved as FAILED
+        for n in notifications:
+            assert n.status == NotificationStatus.COMPLETED
+            assert (
+                n.message
+                == f"Publication {publication.id} failed with status: FAILED.\nReason: credentials error"
+                "\nPublisher: mock\nEvent: event_0"
+            )
+
+        # the derived status for the event should be FAILED
+        assert (
+            MobilizonEvent.from_model(event_model).status
+            == EventPublicationStatus.FAILED
+        )

tests/commands/test_retry.py

@@ -122,16 +122,16 @@ async def test_retry_publication_missing(
 async def test_event_retry_failure(
     event_with_failed_publication,
     mock_publisher_config,
+    mock_notifier_config,
     failed_publication: Publication,
-    caplog,
 ):
 
-    with caplog.at_level(ERROR):
-        await retry_event(event_with_failed_publication.mobilizon_id)
-        assert (
-            f"Publication {failed_publication.id} failed with status: 0.\nReason: credentials error"
-            in caplog.text
-        )
+    report = await retry_event(event_with_failed_publication.mobilizon_id)
+    assert len(report.reports) == 1
+    assert (
+        f"Publication {failed_publication.id} failed with status: FAILED.\nReason: credentials error"
+        in report.reports[0].get_failure_message()
+    )
 
     p = await Publication.filter(id=failed_publication.id).first()
     assert p.status == PublicationStatus.FAILED, p.id
@@ -144,15 +144,17 @@ async def test_event_retry_failure(
 async def test_publication_retry_failure(
     event_with_failed_publication,
     mock_publisher_config,
+    mock_notifier_config,
     failed_publication: Publication,
     caplog,
 ):
 
     with caplog.at_level(ERROR):
-        await retry_publication(failed_publication.id)
+        report = await retry_publication(failed_publication.id)
+        assert len(report.reports) == 1
         assert (
-            f"Publication {failed_publication.id} failed with status: 0.\nReason: credentials error"
-            in caplog.text
+            f"Publication {failed_publication.id} failed with status: FAILED.\nReason: credentials error"
+            in report.reports[0].get_failure_message()
         )
     p = await Publication.filter(id=failed_publication.id).first()
     assert p.status == PublicationStatus.FAILED, p.id

tests/commands/test_start.py

@@ -186,7 +186,7 @@ async def test_start_publisher_failure(
 
         assert "Event to publish found" in caplog.text
         assert message_collector == [
-            f"Publication {p.id} failed with status: 0."
+            f"Publication {p.id} failed with status: FAILED."
             f"\nReason: credentials error\nPublisher: mock\nEvent: test event"
             for p in publications
             for _ in range(2)

tests/conftest.py

@@ -51,7 +51,7 @@ def generate_event_status(published):
 
 
 def generate_notification_status(published):
-    return NotificationStatus.COMPLETED if published else NotificationStatus.WAITING
+    return NotificationStatus.COMPLETED if published else NotificationStatus.FAILED
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -421,6 +421,12 @@ def mock_publisher_valid(message_collector, mock_publisher_class):
     return mock_publisher_class()
 
 
+@pytest.fixture
+def mock_zulip_publisher(message_collector, mock_zulip_publisher_class):
+
+    return mock_zulip_publisher_class()
+
+
 @pytest.fixture
 def mobilizon_url():
     return get_settings()["source"]["mobilizon"]["url"]

tests/models/test_notification.py

@@ -8,5 +8,5 @@ async def test_notification_create(notification_model_generator):
     notification_model = notification_model_generator()
     await notification_model.save()
     notification_db = await Notification.all().first()
-    assert notification_db.status == NotificationStatus.WAITING
+    assert notification_db.status == NotificationStatus.FAILED
     assert notification_db.message == "message_1"

tests/publishers/conftest.py

@@ -78,3 +78,21 @@ def mock_publisher_invalid_response(message_collector):
             pass
 
     return MockPublisher()
+
+
+@pytest.fixture
+def mock_zulip_publisher_invalid_response(message_collector):
+    class MockPublisher(AbstractPlatform):
+
+        name = "zulip"
+
+        def _send(self, message, event):
+            message_collector.append(message)
+
+        def _validate_response(self, response):
+            raise InvalidResponse("Invalid response")
+
+        def validate_credentials(self) -> None:
+            pass
+
+    return MockPublisher()

tests/publishers/test_coordinator.py

@@ -115,8 +115,12 @@ async def mock_publications(
 
 @pytest.mark.parametrize("num_publications", [2])
 @pytest.mark.asyncio
-async def test_publication_coordinator_run_success(mock_publications,):
-    coordinator = PublisherCoordinator(publications=mock_publications,)
+async def test_publication_coordinator_run_success(
+    mock_publications,
+):
+    coordinator = PublisherCoordinator(
+        publications=mock_publications,
+    )
     report = coordinator.run()
     assert len(report.reports) == 2
     assert report.successful, "\n".join(map(lambda rep: rep.reason, report.reports))
@@ -173,12 +177,12 @@ async def test_publication_coordinator_run_failure_response(
 
 @pytest.mark.asyncio
 async def test_notifier_coordinator_publication_failed(
-    mock_publisher_valid, failure_report
+    mock_zulip_publisher, failure_report
 ):
     mock_send = MagicMock()
-    mock_publisher_valid._send = mock_send
+    mock_zulip_publisher._send = mock_send
     coordinator = PublicationFailureNotifiersCoordinator(
-        failure_report, [mock_publisher_valid, mock_publisher_valid]
+        failure_report, [mock_zulip_publisher, mock_zulip_publisher]
     )
     coordinator.notify_failure()
 
@@ -188,18 +192,18 @@ async def test_notifier_coordinator_publication_failed(
 
 @pytest.mark.asyncio
 async def test_notifier_coordinator_error(
-    failure_report, mock_publisher_invalid_response, caplog
+    failure_report, mock_zulip_publisher_invalid_response, caplog
 ):
     mock_send = MagicMock()
-    mock_publisher_invalid_response._send = mock_send
+    mock_zulip_publisher_invalid_response._send = mock_send
 
     coordinator = PublicationFailureNotifiersCoordinator(
         failure_report,
-        [mock_publisher_invalid_response, mock_publisher_invalid_response],
+        [mock_zulip_publisher_invalid_response, mock_zulip_publisher_invalid_response],
     )
     with caplog.at_level(logging.CRITICAL):
         coordinator.notify_failure()
-        assert "Failed to send" in caplog.text
+        assert "Failed to notify failure of" in caplog.text
         assert failure_report.get_failure_message() in caplog.text
     # 4 = 2 reports * 2 notifiers
     assert mock_send.call_count == 2

tests/storage/test_query.py

@@ -9,10 +9,15 @@ from mobilizon_reshare.dataclasses.event import (
     get_mobilizon_events_with_status,
     get_mobilizon_events_without_publications,
 )
+from mobilizon_reshare.storage.query.read import (
+    get_all_events,
+    get_event,
+)
 from mobilizon_reshare.dataclasses.publication import build_publications_for_event
 from mobilizon_reshare.models.publication import PublicationStatus
 from mobilizon_reshare.storage.query.read import publications_with_status
 from tests import today
+from tests.commands.test_publish import one_unpublished_event_specification
 from tests.conftest import event_0, event_1, event_3
 from tests.storage import complete_specification
 from tests.storage import result_publication
@@ -153,6 +158,14 @@ async def test_events_without_publications(spec, expected_events, generate_model
     assert unpublished_events == expected_events
 
 
+@pytest.mark.asyncio
+async def test_get_all_events(generate_models):
+    await generate_models(one_unpublished_event_specification)
+
+    all_events = [await get_event(event_0.mobilizon_id)]
+    assert list(await get_all_events()) == all_events
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "mock_active_publishers, spec, event, n_publications",

tests/storage/test_read_query.py

@@ -1,16 +0,0 @@
-from uuid import UUID
-
-import pytest
-
-from mobilizon_reshare.dataclasses.event import get_all_mobilizon_events
-
-
-@pytest.mark.asyncio
-async def test_get_all_events(event_generator):
-    all_events = [
-        event_generator(mobilizon_id=UUID(int=i), published=False) for i in range(4)
-    ]
-    for e in all_events:
-        await e.to_model().save()
-
-    assert list(await get_all_mobilizon_events()) == all_events