mirror of
https://github.com/Tech-Workers-Coalition-Italia/mobilizon-reshare.git
synced 2025-02-06 04:13:27 +01:00
notify failure (#52)
* fixed visualization * simplified tests * split into files * refactored test expected publications * split update tests * expanded specifications and tests * added event_status window tests * fixed 'all' command * renamed everything * fixed uppercase * refactored main and publisher to add notifications * tested report successful * added tests to publisher coordinator * added more coordinator tests * test coordinator success
This commit is contained in:
Simone Robutti
GitHub
parent
2c8063cf4a
commit
2197e07213
@ -1,10 +1,11 @@
|
||||
import logging.config
|
||||
|
||||
|
||||
from mobilizon_reshare.event.event_selection_strategies import select_event_to_publish
|
||||
from mobilizon_reshare.mobilizon.events import get_unpublished_events
|
||||
from mobilizon_reshare.models.publication import PublicationStatus
|
||||
from mobilizon_reshare.publishers import get_active_publishers
|
||||
from mobilizon_reshare.publishers.coordinator import (
|
||||
PublicationFailureNotifiersCoordinator,
|
||||
)
|
||||
from mobilizon_reshare.publishers.coordinator import PublisherCoordinator
|
||||
from mobilizon_reshare.storage.query import (
|
||||
get_published_events,
|
||||
@ -23,7 +24,9 @@ async def main():
|
||||
:return:
|
||||
"""
|
||||
|
||||
active_publishers = get_active_publishers()
|
||||
# TODO: the logic to get published and unpublished events is probably redundant.
|
||||
# We need a simpler way to bring together events from mobilizon, unpublished events from the db
|
||||
# and published events from the DB
|
||||
|
||||
# Load past events
|
||||
published_events = list(await get_published_events())
|
||||
@ -31,27 +34,24 @@ async def main():
|
||||
# Pull unpublished events from Mobilizon
|
||||
unpublished_events = get_unpublished_events(published_events)
|
||||
# Store in the DB only the ones we didn't know about
|
||||
await create_unpublished_events(unpublished_events, active_publishers)
|
||||
await create_unpublished_events(unpublished_events)
|
||||
event = select_event_to_publish(
|
||||
published_events,
|
||||
# We must load unpublished events from DB since it contains
|
||||
# merged state between Mobilizon and previous WAITING events.
|
||||
list(await get_db_unpublished_events()),
|
||||
)
|
||||
if event:
|
||||
logger.debug(f"Event to publish found: {event.name}")
|
||||
result = PublisherCoordinator(
|
||||
event,
|
||||
[
|
||||
(pub.id, pub.publisher.name)
|
||||
for pub in await publications_with_status(
|
||||
status=PublicationStatus.WAITING,
|
||||
event_mobilizon_id=event.mobilizon_id,
|
||||
)
|
||||
],
|
||||
).run()
|
||||
await save_publication_report(result, event)
|
||||
|
||||
return 0 if result.successful else 1
|
||||
if event:
|
||||
|
||||
waiting_publications = await publications_with_status(
|
||||
status=PublicationStatus.WAITING, event_mobilizon_id=event.mobilizon_id,
|
||||
)
|
||||
logger.debug(f"Event to publish found: {event.name}")
|
||||
report = PublisherCoordinator(event, waiting_publications).run()
|
||||
await save_publication_report(report, waiting_publications)
|
||||
PublicationFailureNotifiersCoordinator(event, report).notify_failures()
|
||||
|
||||
return 0 if report.successful else 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
from typing import Iterator
|
||||
|
||||
from dynaconf import Validator
|
||||
|
||||
telegram_validators = [
|
||||
@ -18,7 +20,7 @@ notifier_name_to_validators = {
|
||||
notifier_names = notifier_name_to_validators.keys()
|
||||
|
||||
|
||||
def get_active_notifiers(settings):
|
||||
def get_active_notifiers(settings) -> Iterator[str]:
|
||||
return filter(
|
||||
lambda notifier_name: settings["notifier"][notifier_name]["active"],
|
||||
notifier_names,
|
||||
|
||||
@ -5,7 +5,7 @@ from tortoise.models import Model
|
||||
class Event(Model):
|
||||
id = fields.UUIDField(pk=True)
|
||||
name = fields.TextField()
|
||||
description = fields.TextField()
|
||||
description = fields.TextField(null=True)
|
||||
|
||||
mobilizon_id = fields.TextField()
|
||||
mobilizon_link = fields.TextField()
|
||||
|
||||
@ -1,6 +1,11 @@
|
||||
from mobilizon_reshare.config.config import get_settings
|
||||
import mobilizon_reshare.config.notifiers
|
||||
import mobilizon_reshare.config.publishers
|
||||
from mobilizon_reshare.config.config import get_settings
|
||||
|
||||
|
||||
def get_active_publishers():
|
||||
return mobilizon_reshare.config.publishers.get_active_publishers(get_settings())
|
||||
|
||||
|
||||
def get_active_notifiers():
|
||||
return mobilizon_reshare.config.notifiers.get_active_notifiers(get_settings())
|
||||
|
||||
@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
|
||||
|
||||
from dynaconf.utils.boxing import DynaBox
|
||||
from jinja2 import Environment, FileSystemLoader, Template
|
||||
from requests import Response
|
||||
|
||||
from mobilizon_reshare.config.config import get_settings
|
||||
from mobilizon_reshare.event.event import MobilizonEvent
|
||||
@ -29,9 +30,6 @@ class AbstractNotifier(ABC):
|
||||
# the second the name of its service (ie: 'facebook', 'telegram')
|
||||
_conf = tuple()
|
||||
|
||||
def __init__(self, message: str):
|
||||
self.message = message
|
||||
|
||||
def __repr__(self):
|
||||
return type(self).__name__
|
||||
|
||||
@ -56,6 +54,13 @@ class AbstractNotifier(ABC):
|
||||
f" (should be 2-tuple)"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def send(self, message):
|
||||
"""
|
||||
Sends a message to the target channel
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _log_debug(self, msg, *args, **kwargs):
|
||||
self.__log(logging.DEBUG, msg, *args, **kwargs)
|
||||
|
||||
@ -94,7 +99,7 @@ class AbstractNotifier(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def post(self) -> None:
|
||||
def publish(self) -> None:
|
||||
"""
|
||||
Publishes the actual post on social media.
|
||||
Should raise ``PublisherError`` (or one of its subclasses) if
|
||||
@ -135,7 +140,7 @@ class AbstractPublisher(AbstractNotifier):
|
||||
|
||||
def __init__(self, event: MobilizonEvent):
|
||||
self.event = event
|
||||
super().__init__(message=self.get_message_from_event())
|
||||
super().__init__()
|
||||
|
||||
def is_event_valid(self) -> bool:
|
||||
try:
|
||||
@ -172,3 +177,18 @@ class AbstractPublisher(AbstractNotifier):
|
||||
"""
|
||||
template_path = self.conf.msg_template_path or self.default_template_path
|
||||
return JINJA_ENV.get_template(template_path)
|
||||
|
||||
@abstractmethod
|
||||
def _send(self, message) -> Response:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _validate_response(self, response: Response) -> None:
|
||||
pass
|
||||
|
||||
def send(self, message):
|
||||
res = self._send(message)
|
||||
self._validate_response(res)
|
||||
|
||||
def publish(self) -> None:
|
||||
self.send(message=self.get_message_from_event())
|
||||
|
||||
@ -1,22 +1,41 @@
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from uuid import UUID
|
||||
|
||||
from mobilizon_reshare.event.event import MobilizonEvent
|
||||
from mobilizon_reshare.models.publication import Publication
|
||||
from mobilizon_reshare.models.publication import PublicationStatus
|
||||
from mobilizon_reshare.publishers import get_active_notifiers, get_active_publishers
|
||||
from mobilizon_reshare.publishers.abstract import AbstractPublisher
|
||||
from mobilizon_reshare.publishers.exceptions import PublisherError
|
||||
from mobilizon_reshare.publishers.telegram import TelegramPublisher
|
||||
|
||||
KEY2CLS = {"telegram": TelegramPublisher}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BuildPublisherMixin:
|
||||
@staticmethod
|
||||
def build_publishers(
|
||||
event: MobilizonEvent, publisher_names
|
||||
) -> dict[str, AbstractPublisher]:
|
||||
name_to_publisher_class = {"telegram": TelegramPublisher}
|
||||
|
||||
return {
|
||||
publisher_name: name_to_publisher_class[publisher_name](event)
|
||||
for publisher_name in publisher_names
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PublicationReport:
|
||||
status: PublicationStatus
|
||||
reason: str
|
||||
publication_id: UUID
|
||||
|
||||
|
||||
@dataclass
|
||||
class PublisherCoordinatorReport:
|
||||
publishers: dict[UUID, AbstractPublisher]
|
||||
reports: dict[UUID, PublicationReport] = field(default_factory={})
|
||||
|
||||
@property
|
||||
@ -25,47 +44,54 @@ class PublisherCoordinatorReport:
|
||||
r.status == PublicationStatus.COMPLETED for r in self.reports.values()
|
||||
)
|
||||
|
||||
def __iter__(self):
|
||||
return self.reports.items().__iter__()
|
||||
|
||||
|
||||
class PublisherCoordinator:
|
||||
def __init__(self, event: MobilizonEvent, publications: list[tuple[UUID, str]]):
|
||||
self.publications = tuple(
|
||||
(publication_id, KEY2CLS[publisher_name](event))
|
||||
for publication_id, publisher_name in publications
|
||||
)
|
||||
class PublisherCoordinator(BuildPublisherMixin):
|
||||
def __init__(self, event: MobilizonEvent, publications: dict[UUID, Publication]):
|
||||
publishers = self.build_publishers(event, get_active_publishers())
|
||||
self.publishers_by_publication_id = {
|
||||
publication_id: publishers[publication.publisher.name]
|
||||
for publication_id, publication in publications.items()
|
||||
}
|
||||
|
||||
def run(self) -> PublisherCoordinatorReport:
|
||||
errors = self._validate()
|
||||
if errors:
|
||||
return PublisherCoordinatorReport(reports=errors)
|
||||
return PublisherCoordinatorReport(
|
||||
reports=errors, publishers=self.publishers_by_publication_id
|
||||
)
|
||||
|
||||
return self._post()
|
||||
|
||||
def _make_successful_report(self):
|
||||
return {
|
||||
publication_id: PublicationReport(
|
||||
status=PublicationStatus.COMPLETED, reason="",
|
||||
status=PublicationStatus.COMPLETED,
|
||||
reason="",
|
||||
publication_id=publication_id,
|
||||
)
|
||||
for publication_id, _ in self.publications
|
||||
for publication_id in self.publishers_by_publication_id
|
||||
}
|
||||
|
||||
def _post(self):
|
||||
failed_publishers_reports = {}
|
||||
for publication_id, p in self.publications:
|
||||
for publication_id, p in self.publishers_by_publication_id.items():
|
||||
try:
|
||||
p.post()
|
||||
p.publish()
|
||||
except PublisherError as e:
|
||||
failed_publishers_reports[publication_id] = PublicationReport(
|
||||
status=PublicationStatus.FAILED, reason=repr(e),
|
||||
status=PublicationStatus.FAILED,
|
||||
reason=str(e),
|
||||
publication_id=publication_id,
|
||||
)
|
||||
|
||||
reports = failed_publishers_reports or self._make_successful_report()
|
||||
return PublisherCoordinatorReport(reports)
|
||||
return PublisherCoordinatorReport(
|
||||
publishers=self.publishers_by_publication_id, reports=reports
|
||||
)
|
||||
|
||||
def _validate(self):
|
||||
errors: dict[UUID, PublicationReport] = {}
|
||||
for publication_id, p in self.publications:
|
||||
for publication_id, p in self.publishers_by_publication_id.items():
|
||||
reason = []
|
||||
if not p.are_credentials_valid():
|
||||
reason.append("Invalid credentials")
|
||||
@ -76,7 +102,45 @@ class PublisherCoordinator:
|
||||
|
||||
if len(reason) > 0:
|
||||
errors[publication_id] = PublicationReport(
|
||||
status=PublicationStatus.FAILED, reason=", ".join(reason)
|
||||
status=PublicationStatus.FAILED,
|
||||
reason=", ".join(reason),
|
||||
publication_id=publication_id,
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
class AbstractNotifiersCoordinator(BuildPublisherMixin):
|
||||
def __init__(self, event: MobilizonEvent):
|
||||
self.event = event
|
||||
self.notifiers = self.build_publishers(event, get_active_notifiers())
|
||||
|
||||
def send_to_all(self, message):
|
||||
# TODO: failure to notify should fail safely and write to a dedicated log
|
||||
for notifier in self.notifiers.values():
|
||||
notifier.send(message)
|
||||
|
||||
|
||||
class PublicationFailureNotifiersCoordinator(AbstractNotifiersCoordinator):
|
||||
def __init__(
|
||||
self,
|
||||
event: MobilizonEvent,
|
||||
publisher_coordinator_report: PublisherCoordinatorReport,
|
||||
):
|
||||
self.report = publisher_coordinator_report
|
||||
super(PublicationFailureNotifiersCoordinator, self).__init__(event)
|
||||
|
||||
def build_failure_message(self, report: PublicationReport):
|
||||
return (
|
||||
f"Publication {report.publication_id} failed with status: {report.status}.\n"
|
||||
f"Reason: {report.reason}"
|
||||
)
|
||||
|
||||
def notify_failures(self):
|
||||
for publication_id, report in self.report.reports.items():
|
||||
|
||||
logger.info(
|
||||
f"Sending failure notifications for publication: {publication_id}"
|
||||
)
|
||||
if report.status == PublicationStatus.FAILED:
|
||||
self.send_to_all(self.build_failure_message(report))
|
||||
|
||||
@ -1,14 +1,15 @@
|
||||
import pkg_resources
|
||||
import requests
|
||||
from requests import Response
|
||||
|
||||
from .abstract import AbstractPublisher
|
||||
from .exceptions import (
|
||||
from mobilizon_reshare.formatting.description import html_to_markdown
|
||||
from mobilizon_reshare.publishers.abstract import AbstractPublisher
|
||||
from mobilizon_reshare.publishers.exceptions import (
|
||||
InvalidBot,
|
||||
InvalidCredentials,
|
||||
InvalidEvent,
|
||||
InvalidResponse,
|
||||
)
|
||||
from ..formatting.description import html_to_markdown
|
||||
|
||||
|
||||
class TelegramPublisher(AbstractPublisher):
|
||||
@ -21,18 +22,23 @@ class TelegramPublisher(AbstractPublisher):
|
||||
"mobilizon_reshare.publishers.templates", "telegram.tmpl.j2"
|
||||
)
|
||||
|
||||
def post(self) -> None:
|
||||
conf = self.conf
|
||||
res = requests.post(
|
||||
url=f"https://api.telegram.org/bot{conf.token}/sendMessage",
|
||||
def _escape_message(self, message: str) -> str:
|
||||
return (
|
||||
message.replace("-", "\\-")
|
||||
.replace(".", "\\.")
|
||||
.replace("(", "\\(")
|
||||
.replace(")", "\\)")
|
||||
)
|
||||
|
||||
def _send(self, message: str) -> Response:
|
||||
return requests.post(
|
||||
url=f"https://api.telegram.org/bot{self.conf.token}/sendMessage",
|
||||
json={
|
||||
"chat_id": conf.chat_id,
|
||||
"text": self.message,
|
||||
"chat_id": self.conf.chat_id,
|
||||
"text": self._escape_message(message),
|
||||
"parse_mode": "markdownv2",
|
||||
},
|
||||
)
|
||||
print(res.json())
|
||||
self._validate_response(res)
|
||||
|
||||
def validate_credentials(self):
|
||||
conf = self.conf
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import logging
|
||||
import sys
|
||||
from typing import Iterable, Optional, Union
|
||||
from typing import Iterable, Optional, Union, Dict
|
||||
from uuid import UUID
|
||||
|
||||
import arrow
|
||||
import sys
|
||||
from arrow import Arrow
|
||||
from tortoise.queryset import QuerySet
|
||||
from tortoise.transactions import atomic
|
||||
@ -12,6 +12,7 @@ from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus
|
||||
from mobilizon_reshare.models.event import Event
|
||||
from mobilizon_reshare.models.publication import Publication, PublicationStatus
|
||||
from mobilizon_reshare.models.publisher import Publisher
|
||||
from mobilizon_reshare.publishers import get_active_publishers
|
||||
from mobilizon_reshare.publishers.coordinator import PublisherCoordinatorReport
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -52,7 +53,7 @@ async def publications_with_status(
|
||||
event_mobilizon_id: Optional[UUID] = None,
|
||||
from_date: Optional[Arrow] = None,
|
||||
to_date: Optional[Arrow] = None,
|
||||
) -> Iterable[Publication]:
|
||||
) -> Dict[UUID, Publication]:
|
||||
query = Publication.filter(status=status)
|
||||
|
||||
if event_mobilizon_id:
|
||||
@ -62,7 +63,10 @@ async def publications_with_status(
|
||||
|
||||
query = _add_date_window(query, "timestamp", from_date, to_date)
|
||||
|
||||
return await query.prefetch_related("publisher").order_by("timestamp").distinct()
|
||||
publications_list = (
|
||||
await query.prefetch_related("publisher").order_by("timestamp").distinct()
|
||||
)
|
||||
return {pub.id: pub for pub in publications_list}
|
||||
|
||||
|
||||
async def events_with_status(
|
||||
@ -102,7 +106,17 @@ async def get_all_events(
|
||||
|
||||
|
||||
async def get_published_events() -> Iterable[MobilizonEvent]:
|
||||
return await events_with_status([EventPublicationStatus.COMPLETED])
|
||||
"""
|
||||
Retrieves events that are not waiting. Function could be renamed to something more fitting
|
||||
:return:
|
||||
"""
|
||||
return await events_with_status(
|
||||
[
|
||||
EventPublicationStatus.COMPLETED,
|
||||
EventPublicationStatus.PARTIAL,
|
||||
EventPublicationStatus.FAILED,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def get_unpublished_events() -> Iterable[MobilizonEvent]:
|
||||
@ -161,7 +175,6 @@ async def save_publication(
|
||||
@atomic(CONNECTION_NAME)
|
||||
async def create_unpublished_events(
|
||||
unpublished_mobilizon_events: Iterable[MobilizonEvent],
|
||||
active_publishers: Iterable[str],
|
||||
) -> None:
|
||||
# We store only new events, i.e. events whose mobilizon_id wasn't found in the DB.
|
||||
unpublished_event_models = set(
|
||||
@ -176,7 +189,7 @@ async def create_unpublished_events(
|
||||
|
||||
for event in unpublished_events:
|
||||
event_model = await save_event(event)
|
||||
for publisher in active_publishers:
|
||||
for publisher in get_active_publishers():
|
||||
await save_publication(
|
||||
publisher, event_model, status=PublicationStatus.WAITING
|
||||
)
|
||||
@ -184,13 +197,10 @@ async def create_unpublished_events(
|
||||
|
||||
@atomic(CONNECTION_NAME)
|
||||
async def save_publication_report(
|
||||
coordinator_report: PublisherCoordinatorReport, event: MobilizonEvent
|
||||
coordinator_report: PublisherCoordinatorReport,
|
||||
publications: Dict[UUID, Publication],
|
||||
) -> None:
|
||||
publications: dict[UUID, Publication] = {
|
||||
p.id: p for p in await get_mobilizon_event_publications(event)
|
||||
}
|
||||
|
||||
for publication_id, publication_report in coordinator_report:
|
||||
for publication_id, publication_report in coordinator_report.reports.items():
|
||||
|
||||
publications[publication_id].status = publication_report.status
|
||||
publications[publication_id].reason = publication_report.reason
|
||||
|
||||
89
tests/publishers/conftest.py
Normal file
89
tests/publishers/conftest.py
Normal file
@ -0,0 +1,89 @@
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from mobilizon_reshare.event.event import MobilizonEvent
|
||||
from mobilizon_reshare.publishers.abstract import AbstractPublisher
|
||||
from mobilizon_reshare.publishers.exceptions import PublisherError, InvalidResponse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_event():
|
||||
|
||||
now = datetime.now()
|
||||
return MobilizonEvent(
|
||||
**{
|
||||
"name": "TestName",
|
||||
"description": "TestDescr",
|
||||
"begin_datetime": now,
|
||||
"end_datetime": now + timedelta(hours=1),
|
||||
"mobilizon_link": "",
|
||||
"mobilizon_id": "",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_publisher_valid(event):
|
||||
class MockPublisher(AbstractPublisher):
|
||||
def validate_event(self) -> None:
|
||||
pass
|
||||
|
||||
def get_message_from_event(self) -> str:
|
||||
return self.event.description
|
||||
|
||||
def validate_credentials(self) -> None:
|
||||
pass
|
||||
|
||||
def validate_message(self) -> None:
|
||||
pass
|
||||
|
||||
def _send(self, message):
|
||||
pass
|
||||
|
||||
def _validate_response(self, response) -> None:
|
||||
pass
|
||||
|
||||
return MockPublisher(event)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_publisher_invalid(event):
|
||||
class MockPublisher(AbstractPublisher):
|
||||
def validate_event(self) -> None:
|
||||
raise PublisherError("Invalid event")
|
||||
|
||||
def get_message_from_event(self) -> str:
|
||||
return ""
|
||||
|
||||
def validate_credentials(self) -> None:
|
||||
raise PublisherError("Invalid credentials")
|
||||
|
||||
def validate_message(self) -> None:
|
||||
raise PublisherError("Invalid message")
|
||||
|
||||
def _send(self, message):
|
||||
pass
|
||||
|
||||
def _validate_response(self, response) -> None:
|
||||
pass
|
||||
|
||||
return MockPublisher(event)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_publisher_invalid_response(mock_publisher_invalid, event):
|
||||
class MockPublisher(type(mock_publisher_invalid)):
|
||||
def validate_event(self) -> None:
|
||||
pass
|
||||
|
||||
def validate_credentials(self) -> None:
|
||||
pass
|
||||
|
||||
def validate_message(self) -> None:
|
||||
pass
|
||||
|
||||
def _validate_response(self, response) -> None:
|
||||
raise InvalidResponse("Invalid response")
|
||||
|
||||
return MockPublisher(event)
|
||||
@ -1,89 +1,22 @@
|
||||
import pytest
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from mobilizon_reshare.event.event import MobilizonEvent
|
||||
from mobilizon_reshare.publishers.abstract import AbstractPublisher
|
||||
from mobilizon_reshare.publishers.exceptions import PublisherError
|
||||
def test_are_credentials_valid(test_event, mock_publisher_valid):
|
||||
assert mock_publisher_valid.are_credentials_valid()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_event():
|
||||
class TestMobilizonEvent(MobilizonEvent):
|
||||
pass
|
||||
|
||||
now = datetime.now()
|
||||
return TestMobilizonEvent(
|
||||
**{
|
||||
"name": "TestName",
|
||||
"description": "TestDescr",
|
||||
"begin_datetime": now,
|
||||
"end_datetime": now + timedelta(hours=1),
|
||||
"mobilizon_link": "",
|
||||
"mobilizon_id": "",
|
||||
}
|
||||
)
|
||||
def test_are_credentials_valid_false(mock_publisher_invalid):
|
||||
assert not mock_publisher_invalid.are_credentials_valid()
|
||||
|
||||
|
||||
def mock_publisher_valid(event):
|
||||
class MockPublisher(AbstractPublisher):
|
||||
def validate_event(self) -> None:
|
||||
pass
|
||||
|
||||
def get_message_from_event(self) -> str:
|
||||
return self.event.description
|
||||
|
||||
def validate_credentials(self) -> None:
|
||||
pass
|
||||
|
||||
def post(self) -> bool:
|
||||
return True
|
||||
|
||||
def validate_message(self) -> None:
|
||||
pass
|
||||
|
||||
return MockPublisher(event)
|
||||
def test_is_event_valid(mock_publisher_valid):
|
||||
assert mock_publisher_valid.is_event_valid()
|
||||
|
||||
|
||||
def mock_publisher_invalid(event):
|
||||
class MockPublisher(AbstractPublisher):
|
||||
def validate_event(self) -> None:
|
||||
raise PublisherError("Invalid event")
|
||||
|
||||
def get_message_from_event(self) -> str:
|
||||
return ""
|
||||
|
||||
def validate_credentials(self) -> None:
|
||||
raise PublisherError("Invalid credentials")
|
||||
|
||||
def post(self) -> bool:
|
||||
return False
|
||||
|
||||
def validate_message(self) -> None:
|
||||
raise PublisherError("Invalid message")
|
||||
|
||||
return MockPublisher(event)
|
||||
def test_is_event_valid_false(mock_publisher_invalid):
|
||||
assert not mock_publisher_invalid.is_event_valid()
|
||||
|
||||
|
||||
def test_are_credentials_valid(test_event):
|
||||
assert mock_publisher_valid(test_event).are_credentials_valid()
|
||||
def test_is_message_valid(mock_publisher_valid):
|
||||
assert mock_publisher_valid.is_message_valid()
|
||||
|
||||
|
||||
def test_are_credentials_valid_false(test_event):
|
||||
assert not mock_publisher_invalid(test_event).are_credentials_valid()
|
||||
|
||||
|
||||
def test_is_event_valid(test_event):
|
||||
assert mock_publisher_valid(test_event).is_event_valid()
|
||||
|
||||
|
||||
def test_is_event_valid_false(test_event):
|
||||
assert not mock_publisher_invalid(test_event).is_event_valid()
|
||||
|
||||
|
||||
def test_is_message_valid(test_event):
|
||||
assert mock_publisher_valid(test_event).is_message_valid()
|
||||
|
||||
|
||||
def test_is_message_valid_false(test_event):
|
||||
assert not mock_publisher_invalid(test_event).is_message_valid()
|
||||
def test_is_message_valid_false(mock_publisher_invalid):
|
||||
assert not mock_publisher_invalid.is_message_valid()
|
||||
|
||||
136
tests/publishers/test_coordinator.py
Normal file
136
tests/publishers/test_coordinator.py
Normal file
@ -0,0 +1,136 @@
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from asynctest import MagicMock
|
||||
|
||||
from mobilizon_reshare.event.event import MobilizonEvent
|
||||
from mobilizon_reshare.models.publication import PublicationStatus, Publication
|
||||
from mobilizon_reshare.models.publisher import Publisher
|
||||
from mobilizon_reshare.publishers.coordinator import (
|
||||
PublisherCoordinatorReport,
|
||||
PublicationReport,
|
||||
PublisherCoordinator,
|
||||
PublicationFailureNotifiersCoordinator,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"statuses, successful",
|
||||
[
|
||||
[[PublicationStatus.COMPLETED, PublicationStatus.COMPLETED], True],
|
||||
[[PublicationStatus.WAITING, PublicationStatus.COMPLETED], False],
|
||||
[[PublicationStatus.COMPLETED, PublicationStatus.FAILED], False],
|
||||
[[], True],
|
||||
[[PublicationStatus.COMPLETED], True],
|
||||
],
|
||||
)
|
||||
def test_publication_report_successful(statuses, successful):
|
||||
reports = {}
|
||||
for i, status in enumerate(statuses):
|
||||
reports[UUID(int=i)] = PublicationReport(
|
||||
reason=None, publication_id=None, status=status
|
||||
)
|
||||
assert PublisherCoordinatorReport(None, reports).successful == successful
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@pytest.mark.asyncio
|
||||
async def mock_publication(test_event: MobilizonEvent,):
|
||||
event = test_event.to_model()
|
||||
await event.save()
|
||||
publisher = Publisher(name="telegram")
|
||||
await publisher.save()
|
||||
publication = Publication(
|
||||
id=UUID(int=1),
|
||||
status=PublicationStatus.WAITING,
|
||||
event=event,
|
||||
publisher=publisher,
|
||||
timestamp=None,
|
||||
reason=None,
|
||||
)
|
||||
await publication.save()
|
||||
return publication
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_coordinator_run_success(
|
||||
test_event, mock_publication, mock_publisher_valid
|
||||
):
|
||||
coordinator = PublisherCoordinator(
|
||||
test_event, {UUID(int=1): mock_publication, UUID(int=2): mock_publication}
|
||||
)
|
||||
coordinator.publishers_by_publication_id = {
|
||||
UUID(int=1): mock_publisher_valid,
|
||||
UUID(int=2): mock_publisher_valid,
|
||||
}
|
||||
|
||||
report = coordinator.run()
|
||||
assert len(report.reports) == 2
|
||||
assert report.successful, "\n".join(
|
||||
map(lambda rep: rep.reason, report.reports.values())
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_coordinator_run_failure(
|
||||
test_event, mock_publication, mock_publisher_invalid
|
||||
):
|
||||
coordinator = PublisherCoordinator(test_event, {UUID(int=1): mock_publication})
|
||||
coordinator.publishers_by_publication_id = {
|
||||
UUID(int=1): mock_publisher_invalid,
|
||||
}
|
||||
|
||||
report = coordinator.run()
|
||||
assert len(report.reports) == 1
|
||||
assert not report.successful
|
||||
assert (
|
||||
list(report.reports.values())[0].reason
|
||||
== "Invalid credentials, Invalid event, Invalid message"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_coordinator_run_failure_response(
|
||||
test_event, mock_publication, mock_publisher_invalid_response
|
||||
):
|
||||
coordinator = PublisherCoordinator(test_event, {UUID(int=1): mock_publication})
|
||||
coordinator.publishers_by_publication_id = {
|
||||
UUID(int=1): mock_publisher_invalid_response,
|
||||
}
|
||||
|
||||
report = coordinator.run()
|
||||
assert len(report.reports) == 1
|
||||
assert not report.successful
|
||||
assert list(report.reports.values())[0].reason == "Invalid response"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notifier_coordinator_publication_failed(
|
||||
test_event, mock_publisher_valid
|
||||
):
|
||||
mock_send = MagicMock()
|
||||
mock_publisher_valid._send = mock_send
|
||||
report = PublisherCoordinatorReport(
|
||||
{UUID(int=1): mock_publisher_valid, UUID(int=2): mock_publisher_valid},
|
||||
{
|
||||
UUID(int=1): PublicationReport(
|
||||
status=PublicationStatus.FAILED,
|
||||
reason="some failure",
|
||||
publication_id=UUID(int=1),
|
||||
),
|
||||
UUID(int=2): PublicationReport(
|
||||
status=PublicationStatus.FAILED,
|
||||
reason="some failure",
|
||||
publication_id=UUID(int=2),
|
||||
),
|
||||
},
|
||||
)
|
||||
coordinator = PublicationFailureNotifiersCoordinator(test_event, report)
|
||||
coordinator.notifiers = {
|
||||
UUID(int=1): mock_publisher_valid,
|
||||
UUID(int=2): mock_publisher_valid,
|
||||
}
|
||||
coordinator.notify_failures()
|
||||
|
||||
# 4 = 2 reports * 2 notifiers
|
||||
assert mock_send.call_count == 4
|
||||
@ -1,9 +1,8 @@
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from mobilizon_reshare.models.publication import PublicationStatus
|
||||
from mobilizon_reshare.models.publication import Publication
|
||||
from mobilizon_reshare.models.publication import PublicationStatus
|
||||
|
||||
today = datetime(
|
||||
year=2021, month=6, day=6, hour=5, minute=0, tzinfo=timezone(timedelta(hours=2)),
|
||||
@ -26,6 +25,7 @@ complete_specification = {
|
||||
{"event_idx": 3, "publisher_idx": 1, "status": PublicationStatus.WAITING},
|
||||
{"event_idx": 3, "publisher_idx": 2, "status": PublicationStatus.WAITING},
|
||||
],
|
||||
"publisher": ["telegram", "twitter", "mastodon"],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -13,14 +13,13 @@ from tests.storage import today
|
||||
async def _generate_publishers(specification):
|
||||
|
||||
publishers = []
|
||||
for i in range(
|
||||
specification["publisher"] if "publisher" in specification.keys() else 3
|
||||
):
|
||||
for i, publisher_name in enumerate(specification["publisher"]):
|
||||
publisher = Publisher(
|
||||
id=UUID(int=i), name=f"publisher_{i}", account_ref=f"account_ref_{i}"
|
||||
id=UUID(int=i), name=publisher_name, account_ref=f"account_ref_{i}"
|
||||
)
|
||||
publishers.append(publisher)
|
||||
await publisher.save()
|
||||
|
||||
return publishers
|
||||
|
||||
|
||||
|
||||
@ -16,8 +16,8 @@ from mobilizon_reshare.storage.query import (
|
||||
get_publishers,
|
||||
publications_with_status,
|
||||
)
|
||||
from tests.storage import result_publication
|
||||
from tests.storage import complete_specification
|
||||
from tests.storage import result_publication
|
||||
from tests.storage import today
|
||||
|
||||
event_0 = MobilizonEvent(
|
||||
@ -28,9 +28,9 @@ event_0 = MobilizonEvent(
|
||||
thumbnail_link="thumblink_0",
|
||||
location="loc_0",
|
||||
publication_time={
|
||||
"publisher_0": arrow.get(today + timedelta(hours=0)),
|
||||
"publisher_1": arrow.get(today + timedelta(hours=1)),
|
||||
"publisher_2": arrow.get(today + timedelta(hours=2)),
|
||||
"telegram": arrow.get(today + timedelta(hours=0)),
|
||||
"twitter": arrow.get(today + timedelta(hours=1)),
|
||||
"mastodon": arrow.get(today + timedelta(hours=2)),
|
||||
},
|
||||
status=EventPublicationStatus.COMPLETED,
|
||||
begin_datetime=arrow.get(today + timedelta(days=0)),
|
||||
@ -43,8 +43,7 @@ async def test_get_published_events(generate_models):
|
||||
await generate_models(complete_specification)
|
||||
published_events = list(await get_published_events())
|
||||
|
||||
assert len(published_events) == 1
|
||||
assert published_events == [event_0]
|
||||
assert len(published_events) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@ -123,7 +122,6 @@ async def test_create_unpublished_events(
|
||||
expected_result, generate_models, event_generator,
|
||||
):
|
||||
await generate_models(complete_specification)
|
||||
|
||||
event_3 = event_generator(begin_date=arrow.get(today + timedelta(days=6)))
|
||||
event_4 = event_generator(
|
||||
begin_date=arrow.get(today + timedelta(days=12)), mobilizon_id="67890"
|
||||
@ -132,10 +130,7 @@ async def test_create_unpublished_events(
|
||||
|
||||
events_from_internet = [MobilizonEvent.from_model(models[0]), event_3, event_4]
|
||||
|
||||
await create_unpublished_events(
|
||||
unpublished_mobilizon_events=events_from_internet,
|
||||
active_publishers=["publisher_0", "publisher_1", "publisher_2"],
|
||||
)
|
||||
await create_unpublished_events(unpublished_mobilizon_events=events_from_internet,)
|
||||
unpublished_events = list(await get_unpublished_events())
|
||||
|
||||
assert len(unpublished_events) == 4
|
||||
@ -156,25 +151,22 @@ async def test_get_mobilizon_event_publications(generate_models):
|
||||
assert len(publications) == 3
|
||||
|
||||
assert publications[0].event.name == "event_0"
|
||||
assert publications[0].publisher.name == "publisher_0"
|
||||
assert publications[0].publisher.name == "telegram"
|
||||
assert publications[0].status == PublicationStatus.COMPLETED
|
||||
|
||||
assert publications[1].event.name == "event_0"
|
||||
assert publications[1].publisher.name == "publisher_1"
|
||||
assert publications[1].publisher.name == "twitter"
|
||||
assert publications[1].status == PublicationStatus.COMPLETED
|
||||
|
||||
assert publications[2].event.name == "event_0"
|
||||
assert publications[2].publisher.name == "publisher_2"
|
||||
assert publications[2].publisher.name == "mastodon"
|
||||
assert publications[2].status == PublicationStatus.COMPLETED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"name,expected_result",
|
||||
[
|
||||
[None, {"publisher_0", "publisher_1", "publisher_2"}],
|
||||
["publisher_0", {"publisher_0"}],
|
||||
],
|
||||
[[None, {"telegram", "twitter", "mastodon"}], ["telegram", {"telegram"}]],
|
||||
)
|
||||
async def test_get_publishers(
|
||||
name, expected_result, generate_models,
|
||||
@ -250,7 +242,7 @@ async def test_publications_with_status(
|
||||
to_date=to_date,
|
||||
)
|
||||
|
||||
assert publications == expected_result
|
||||
assert publications == {pub.id: pub for pub in expected_result}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
from datetime import timedelta
|
||||
from uuid import UUID
|
||||
from tests.storage import complete_specification
|
||||
|
||||
import arrow
|
||||
import pytest
|
||||
|
||||
@ -16,9 +16,11 @@ from mobilizon_reshare.storage.query import (
|
||||
update_publishers,
|
||||
save_publication_report,
|
||||
)
|
||||
from mobilizon_reshare.storage.query import publications_with_status
|
||||
from tests.storage import complete_specification
|
||||
from tests.storage import today
|
||||
|
||||
two_publishers_specification = {"publisher": 2}
|
||||
two_publishers_specification = {"publisher": ["telegram", "twitter"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@ -27,21 +29,21 @@ two_publishers_specification = {"publisher": 2}
|
||||
[
|
||||
[
|
||||
two_publishers_specification,
|
||||
["publisher_0", "publisher_1"],
|
||||
["telegram", "twitter"],
|
||||
{
|
||||
Publisher(id=UUID(int=0), name="publisher_0"),
|
||||
Publisher(id=UUID(int=1), name="publisher_1"),
|
||||
Publisher(id=UUID(int=0), name="telegram"),
|
||||
Publisher(id=UUID(int=1), name="twitter"),
|
||||
},
|
||||
],
|
||||
[
|
||||
{"publisher": 0},
|
||||
["publisher_0", "publisher_1"],
|
||||
{"publisher_0", "publisher_1"},
|
||||
{"publisher": ["telegram"]},
|
||||
["telegram", "twitter"],
|
||||
{"telegram", "twitter"},
|
||||
],
|
||||
[
|
||||
two_publishers_specification,
|
||||
["publisher_0", "publisher_2", "publisher_3"],
|
||||
{"publisher_0", "publisher_1", "publisher_2", "publisher_3"},
|
||||
["telegram", "mastodon", "facebook"],
|
||||
{"telegram", "twitter", "mastodon", "facebook"},
|
||||
],
|
||||
],
|
||||
)
|
||||
@ -68,12 +70,17 @@ async def test_update_publishers(
|
||||
PublisherCoordinatorReport(
|
||||
reports={
|
||||
UUID(int=3): PublicationReport(
|
||||
status=PublicationStatus.FAILED, reason="Invalid credentials"
|
||||
status=PublicationStatus.FAILED,
|
||||
reason="Invalid credentials",
|
||||
publication_id=UUID(int=3),
|
||||
),
|
||||
UUID(int=4): PublicationReport(
|
||||
status=PublicationStatus.COMPLETED, reason=""
|
||||
status=PublicationStatus.COMPLETED,
|
||||
reason="",
|
||||
publication_id=UUID(int=4),
|
||||
),
|
||||
}
|
||||
},
|
||||
publishers={},
|
||||
),
|
||||
MobilizonEvent(
|
||||
name="event_1",
|
||||
@ -103,7 +110,11 @@ async def test_save_publication_report(
|
||||
specification, report, event, expected_result, generate_models,
|
||||
):
|
||||
await generate_models(specification)
|
||||
await save_publication_report(report, event)
|
||||
|
||||
publications = await publications_with_status(
|
||||
status=PublicationStatus.WAITING, event_mobilizon_id=event.mobilizon_id,
|
||||
)
|
||||
await save_publication_report(report, publications)
|
||||
publication_ids = set(report.reports.keys())
|
||||
publications = {
|
||||
p_id: await Publication.filter(id=p_id).first() for p_id in publication_ids
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user