event recap (#69)

* added basic recap feature (no error handling)

* introduced abstractpublication

* extracted base reports

* added error report to recap

* added test

* added docs
This commit is contained in:
Simone Robutti 2021-10-16 01:25:45 +02:00 committed by GitHub
parent 5db7fb8597
commit 5e171216d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 305 additions and 107 deletions

View File

@ -5,6 +5,16 @@ to events and their promotion.
# Usage
## Installation
mobilizon-reshare is distributed through Pypi. Use
```pip install mobilizon-reshare```
to install the tool in your system or virtualenv.
This should install the command `mobilizon-reshare` in your system. Use it to access the CLI and discover the available
commands and their description.
## Scheduling and temporal logic
@ -53,6 +63,15 @@ Currently only one strategy is supported: `next_event`. The semantic of the stra
event in chronological order that hasn't been published yet and publish it only if at least
`break_between_events_in_minutes` minutes have passed.
## Recap
In addition to the event publication feature, `mobilizon-reshare` allows you to do periodical recap of your events.
In the current version, the two features are handled separately and triggered by different CLI commands (respectively
`mobilizon-reshare start` and `mobilizon-reshare recap`).
The recap command, when executed, will retrieve the list of already published events and summarize in a single message
to publish on all the active publishers. At the moment it doesn't support any decision logic and will always publish
when triggered.
## Core Concepts

View File

@ -7,7 +7,9 @@ from click import pass_context
from mobilizon_reshare.cli import safe_execution
from mobilizon_reshare.cli.format import format_event
from mobilizon_reshare.cli.inspect_event import inspect_events
from mobilizon_reshare.cli.main import main
from mobilizon_reshare.cli.publish_event.main import main as start_main
from mobilizon_reshare.cli.publish_recap.main import main as recap_main
from mobilizon_reshare.event.event import EventPublicationStatus
settings_file_option = click.option("--settings-file", type=click.Path(exists=True))
@ -33,7 +35,13 @@ def mobilizon_reshare():
@mobilizon_reshare.command()
@settings_file_option
def start(settings_file):
safe_execution(main, settings_file=settings_file)
safe_execution(start_main, settings_file=settings_file)
@mobilizon_reshare.command()
@settings_file_option
def recap(settings_file):
safe_execution(recap_main, settings_file=settings_file)
@mobilizon_reshare.command()
@ -54,12 +62,7 @@ def inspect(ctx, target, begin, end, settings_file):
"all": None,
}
safe_execution(
functools.partial(
inspect_events,
target_to_status[target],
frm=begin,
to=end,
),
functools.partial(inspect_events, target_to_status[target], frm=begin, to=end,),
settings_file,
)
@ -70,8 +73,7 @@ def inspect(ctx, target, begin, end, settings_file):
@click.argument("publisher", type=str)
def format(settings_file, event_id, publisher):
safe_execution(
functools.partial(format_event, event_id, publisher),
settings_file,
functools.partial(format_event, event_id, publisher), settings_file,
)

View File

@ -60,7 +60,7 @@ async def main():
reports = PublisherCoordinator(waiting_publications).run()
await save_publication_report(reports, waiting_publications_models)
for _, report in reports.reports.items():
for report in reports.reports:
PublicationFailureNotifiersCoordinator(report).notify_failure()
return 0 if reports.successful else 1

View File

@ -0,0 +1,50 @@
import logging.config
from typing import List
from arrow import now
from mobilizon_reshare.event.event import EventPublicationStatus, MobilizonEvent
from mobilizon_reshare.publishers import get_active_publishers
from mobilizon_reshare.publishers.abstract import RecapPublication
from mobilizon_reshare.publishers.coordinator import (
RecapCoordinator,
PublicationFailureNotifiersCoordinator,
)
from mobilizon_reshare.publishers.platforms.platform_mapping import (
get_publisher_class,
get_formatter_class,
)
from mobilizon_reshare.storage.query import events_with_status
logger = logging.getLogger(__name__)
async def select_events_to_recap() -> List[MobilizonEvent]:
return list(
await events_with_status(
status=[EventPublicationStatus.COMPLETED], from_date=now()
)
)
async def main():
# I want to recap only the events that have been succesfully published and that haven't happened yet
events_to_recap = await select_events_to_recap()
if events_to_recap:
recap_publications = [
RecapPublication(
get_publisher_class(publisher)(),
get_formatter_class(publisher)(),
events_to_recap,
)
for publisher in get_active_publishers()
]
reports = RecapCoordinator(recap_publications).run()
for report in reports.reports:
if report.status == EventPublicationStatus.FAILED:
PublicationFailureNotifiersCoordinator(report).notify_failure()
return 0 if reports.successful else 1
else:
return 0

View File

@ -10,7 +10,6 @@ telegram_validators = [
zulip_validators = [
Validator("publisher.zulip.chat_id", must_exist=True),
Validator("publisher.zulip.subject", must_exist=True),
Validator("publisher.zulip.msg_template_path", must_exist=True, default=None),
Validator("publisher.zulip.bot_token", must_exist=True),
Validator("publisher.zulip.bot_email", must_exist=True),
]

View File

@ -4,6 +4,7 @@ from dynaconf import Validator
telegram_validators = [
Validator("publisher.telegram.chat_id", must_exist=True),
Validator("publisher.telegram.msg_template_path", must_exist=True, default=None),
Validator("publisher.telegram.recap_template_path", must_exist=True, default=None),
Validator("publisher.telegram.token", must_exist=True),
Validator("publisher.telegram.username", must_exist=True),
]
@ -11,6 +12,7 @@ zulip_validators = [
Validator("publisher.zulip.chat_id", must_exist=True),
Validator("publisher.zulip.subject", must_exist=True),
Validator("publisher.zulip.msg_template_path", must_exist=True, default=None),
Validator("publisher.zulip.recap_template_path", must_exist=True, default=None),
Validator("publisher.zulip.bot_token", must_exist=True),
Validator("publisher.zulip.bot_email", must_exist=True),
]

View File

@ -2,6 +2,7 @@ import inspect
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import List
from uuid import UUID
from dynaconf.utils.boxing import DynaBox
@ -129,7 +130,7 @@ class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin):
"""
Allows publishers to preprocess events before feeding them to the template
"""
pass # pragma: no cover
return event
def get_message_from_event(self, event) -> str:
"""
@ -168,14 +169,28 @@ class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin):
return False
return True
def get_recap_fragment_template(self) -> Template:
template_path = (
self.conf.recap_template_path or self.default_recap_template_path
)
return JINJA_ENV.get_template(template_path)
def get_recap_fragment(self, event: MobilizonEvent) -> str:
event = self._preprocess_event(event)
return event.format(self.get_recap_fragment_template())
@dataclass
class EventPublication:
event: MobilizonEvent
id: UUID
class BasePublication:
publisher: AbstractPlatform
formatter: AbstractEventFormatter
@dataclass
class EventPublication(BasePublication):
event: MobilizonEvent
id: UUID
@classmethod
def from_orm(cls, model: PublicationModel, event: MobilizonEvent):
# imported here to avoid circular dependencies
@ -186,4 +201,9 @@ class EventPublication:
publisher = get_publisher_class(model.publisher.name)()
formatter = get_formatter_class(model.publisher.name)()
return cls(event, model.id, publisher, formatter)
return cls(publisher, formatter, event, model.id,)
@dataclass
class RecapPublication(BasePublication):
events: List[MobilizonEvent]

View File

@ -1,11 +1,15 @@
import logging
from dataclasses import dataclass, field
from typing import List
from dataclasses import dataclass
from typing import List, Optional
from uuid import UUID
from mobilizon_reshare.models.publication import PublicationStatus
from mobilizon_reshare.publishers import get_active_notifiers
from mobilizon_reshare.publishers.abstract import EventPublication
from mobilizon_reshare.publishers.abstract import (
EventPublication,
AbstractPlatform,
RecapPublication,
)
from mobilizon_reshare.publishers.exceptions import PublisherError
from mobilizon_reshare.publishers.platforms.platform_mapping import get_notifier_class
@ -13,22 +17,43 @@ logger = logging.getLogger(__name__)
@dataclass
class PublicationReport:
class BasePublicationReport:
status: PublicationStatus
reason: str
publication_id: UUID
reason: Optional[str]
def get_failure_message(self):
return (
f"Publication failed with status: {self.status}.\n" f"Reason: {self.reason}"
)
@dataclass
class PublisherCoordinatorReport:
publications: List[EventPublication]
reports: dict[UUID, PublicationReport] = field(default_factory={})
class EventPublicationReport(BasePublicationReport):
publication_id: UUID
def get_failure_message(self):
return (
f"Publication {self.publication_id } failed with status: {self.status}.\n"
f"Reason: {self.reason}"
)
@dataclass
class BaseCoordinatorReport:
reports: List[BasePublicationReport]
@property
def successful(self):
return all(
r.status == PublicationStatus.COMPLETED for r in self.reports.values()
)
return all(r.status == PublicationStatus.COMPLETED for r in self.reports)
@dataclass
class PublisherCoordinatorReport(BaseCoordinatorReport):
reports: List[EventPublicationReport]
publications: List[EventPublication]
class PublisherCoordinator:
@ -45,18 +70,18 @@ class PublisherCoordinator:
return self._post()
def _make_successful_report(self, failed_ids):
return {
publication.id: PublicationReport(
return [
EventPublicationReport(
status=PublicationStatus.COMPLETED,
reason="",
publication_id=publication.id,
)
for publication in self.publications
if publication.id not in failed_ids
}
]
def _post(self):
failed_publishers_reports = {}
reports = []
for publication in self.publications:
@ -65,22 +90,28 @@ class PublisherCoordinator:
publication.event
)
publication.publisher.send(message)
reports.append(
EventPublicationReport(
status=PublicationStatus.COMPLETED,
publication_id=publication.id,
reason=None,
)
)
except PublisherError as e:
failed_publishers_reports[publication.id] = PublicationReport(
status=PublicationStatus.FAILED,
reason=str(e),
publication_id=publication.id,
reports.append(
EventPublicationReport(
status=PublicationStatus.FAILED,
reason=str(e),
publication_id=publication.id,
)
)
reports = failed_publishers_reports | self._make_successful_report(
failed_publishers_reports.keys()
)
return PublisherCoordinatorReport(
publications=self.publications, reports=reports
)
def _validate(self):
errors: dict[UUID, PublicationReport] = {}
errors = []
for publication in self.publications:
@ -93,45 +124,75 @@ class PublisherCoordinator:
reason.append("Invalid message")
if len(reason) > 0:
errors[publication.id] = PublicationReport(
status=PublicationStatus.FAILED,
reason=", ".join(reason),
publication_id=publication.id,
errors.append(
EventPublicationReport(
status=PublicationStatus.FAILED,
reason=", ".join(reason),
publication_id=publication.id,
)
)
return errors
class AbstractNotifiersCoordinator:
def __init__(self, message: str, notifiers=None):
class AbstractCoordinator:
def __init__(self, message: str, platforms: List[AbstractPlatform] = None):
self.message = message
self.notifiers = notifiers or [
get_notifier_class(notifier)() for notifier in get_active_notifiers()
]
self.platforms = platforms
def send_to_all(self):
# TODO: failure to notify should fail safely and write to a dedicated log
for notifier in self.notifiers:
notifier.send(self.message)
# TODO: failure to send should fail safely and write to a dedicated log
for platform in self.platforms:
platform.send(self.message)
class AbstractNotifiersCoordinator(AbstractCoordinator):
def __init__(self, message: str, notifiers: List[AbstractPlatform] = None):
platforms = notifiers or [
get_notifier_class(notifier)() for notifier in get_active_notifiers()
]
super(AbstractNotifiersCoordinator, self).__init__(message, platforms)
class PublicationFailureNotifiersCoordinator(AbstractNotifiersCoordinator):
def __init__(self, report: PublicationReport, notifiers=None):
def __init__(self, report: EventPublicationReport, platforms=None):
self.report = report
super(PublicationFailureNotifiersCoordinator, self).__init__(
message=self.build_failure_message(), notifiers=notifiers
)
def build_failure_message(self):
report = self.report
return (
f"Publication {report.publication_id} failed with status: {report.status}.\n"
f"Reason: {report.reason}"
message=report.get_failure_message(), notifiers=platforms
)
def notify_failure(self):
logger.info(
f"Sending failure notifications for publication: {self.report.publication_id}"
)
logger.info("Sending failure notifications")
if self.report.status == PublicationStatus.FAILED:
self.send_to_all()
class RecapCoordinator:
def __init__(self, recap_publications: List[RecapPublication]):
self.recap_publications = recap_publications
def run(self):
reports = []
for recap_publication in self.recap_publications:
try:
fragments = []
for event in recap_publication.events:
fragments.append(
recap_publication.formatter.get_recap_fragment(event)
)
message = "\n\n".join(fragments)
recap_publication.publisher.send(message)
reports.append(
BasePublicationReport(
status=PublicationStatus.COMPLETED, reason=None,
)
)
except PublisherError as e:
reports.append(
BasePublicationReport(
status=PublicationStatus.FAILED, reason=str(e),
)
)
return BaseCoordinatorReport(reports=reports)

View File

@ -22,6 +22,10 @@ class TelegramFormatter(AbstractEventFormatter):
"mobilizon_reshare.publishers.templates", "telegram.tmpl.j2"
)
default_recap_template_path = pkg_resources.resource_filename(
"mobilizon_reshare.publishers.templates", "telegram_recap.tmpl.j2"
)
_conf = ("publisher", "telegram")
@staticmethod
@ -40,9 +44,6 @@ class TelegramFormatter(AbstractEventFormatter):
if not (description and description.strip()):
self._log_error("No description was found", raise_error=InvalidEvent)
def get_message_from_event(self, event: MobilizonEvent) -> str:
return super(TelegramFormatter, self).get_message_from_event(event)
def validate_message(self, message: str) -> None:
if len(message) >= 4096:
raise PublisherError("Message is too long")

View File

@ -26,6 +26,10 @@ class ZulipFormatter(AbstractEventFormatter):
"mobilizon_reshare.publishers.templates", "zulip.tmpl.j2"
)
default_recap_template_path = pkg_resources.resource_filename(
"mobilizon_reshare.publishers.templates", "zulip_recap.tmpl.j2"
)
def validate_event(self, event: MobilizonEvent) -> None:
text = event.description
if not (text and text.strip()):
@ -47,9 +51,7 @@ class ZulipPlatform(AbstractPlatform):
"""
_conf = ("publisher", "zulip")
default_template_path = pkg_resources.resource_filename(
"mobilizon_reshare.publishers.templates", "zulip.tmpl.j2"
)
api_uri = "https://zulip.twc-italia.org/api/v1/"
def _send_private(self, message: str) -> Response:

View File

@ -0,0 +1,4 @@
*{{ name }}*
🕒 {{ begin_datetime.format('DD MMMM, HH:mm') }} - {{ end_datetime.format('DD MMMM, HH:mm') }}
{% if location %}📍 {{ location }}{% endif %}

View File

@ -0,0 +1,8 @@
# {{ name }}
🕒 {{ begin_datetime.format('DD MMMM, HH:mm') }} - {{ end_datetime.format('DD MMMM, HH:mm') }}
{% if location %}
📍 {{ location }}
{% endif %}

View File

@ -94,8 +94,7 @@ async def events_with_status(
async def get_all_events(
from_date: Optional[Arrow] = None,
to_date: Optional[Arrow] = None,
from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
) -> Iterable[MobilizonEvent]:
return map(
@ -154,9 +153,7 @@ async def create_publisher(name: str, account_ref: Optional[str] = None) -> None
@atomic(CONNECTION_NAME)
async def update_publishers(
names: Iterable[str],
) -> None:
async def update_publishers(names: Iterable[str],) -> None:
names = set(names)
known_publisher_names = set(p.name for p in await get_publishers())
for name in names.difference(known_publisher_names):
@ -171,9 +168,7 @@ async def save_publication(
publisher = await get_publishers(publisher_name)
await Publication.create(
status=status,
event_id=event_model.id,
publisher_id=publisher.id,
status=status, event_id=event_model.id, publisher_id=publisher.id,
)
@ -205,8 +200,8 @@ async def save_publication_report(
coordinator_report: PublisherCoordinatorReport,
publications: Dict[UUID, Publication],
) -> None:
for publication_id, publication_report in coordinator_report.reports.items():
for publication_report in coordinator_report.reports:
publication_id = publication_report.publication_id
publications[publication_id].status = publication_report.status
publications[publication_id].reason = publication_report.reason
publications[publication_id].timestamp = arrow.now().datetime

View File

@ -42,6 +42,9 @@ def mock_formatter_valid():
def _send(self, message):
pass
def get_recap_fragment(self, event):
return event.name
return MockFormatter()

View File

@ -9,12 +9,13 @@ from mobilizon_reshare.models.publication import (
Publication as PublicationModel,
)
from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers.abstract import EventPublication
from mobilizon_reshare.publishers.abstract import EventPublication, RecapPublication
from mobilizon_reshare.publishers.coordinator import (
PublisherCoordinatorReport,
PublicationReport,
EventPublicationReport,
PublisherCoordinator,
PublicationFailureNotifiersCoordinator,
RecapCoordinator,
)
@ -29,12 +30,36 @@ from mobilizon_reshare.publishers.coordinator import (
],
)
def test_publication_report_successful(statuses, successful):
reports = {}
reports = []
for i, status in enumerate(statuses):
reports[UUID(int=i)] = PublicationReport(
reason=None, publication_id=None, status=status
reports.append(
EventPublicationReport(reason=None, publication_id=None, status=status)
)
assert PublisherCoordinatorReport(None, reports).successful == successful
assert (
PublisherCoordinatorReport(publications=[], reports=reports).successful
== successful
)
@pytest.fixture
@pytest.mark.asyncio
async def mock_recap_publications(
num_publications: int,
test_event: MobilizonEvent,
mock_publisher_valid,
mock_formatter_valid,
):
result = []
for _ in range(num_publications):
result.append(
RecapPublication(
publisher=mock_publisher_valid,
formatter=mock_formatter_valid,
events=[test_event, test_event],
)
)
return result
@pytest.fixture
@ -69,18 +94,16 @@ async def mock_publications(
@pytest.mark.parametrize("num_publications", [2])
@pytest.mark.asyncio
async def test_coordinator_run_success(mock_publications,):
async def test_publication_coordinator_run_success(mock_publications,):
coordinator = PublisherCoordinator(publications=mock_publications,)
report = coordinator.run()
assert len(report.reports) == 2
assert report.successful, "\n".join(
map(lambda rep: rep.reason, report.reports.values())
)
assert report.successful, "\n".join(map(lambda rep: rep.reason, report.reports))
@pytest.mark.parametrize("num_publications", [1])
@pytest.mark.asyncio
async def test_coordinator_run_failure(
async def test_publication_coordinator_run_failure(
mock_publications, mock_publisher_invalid, mock_formatter_invalid
):
for pub in mock_publications:
@ -92,14 +115,14 @@ async def test_coordinator_run_failure(
assert len(report.reports) == 1
assert not report.successful
assert (
list(report.reports.values())[0].reason
list(report.reports)[0].reason
== "Invalid credentials, Invalid event, Invalid message"
)
@pytest.mark.parametrize("num_publications", [1])
@pytest.mark.asyncio
async def test_coordinator_run_failure_response(
async def test_publication_coordinator_run_failure_response(
mock_publications, mock_publisher_invalid_response
):
@ -109,14 +132,14 @@ async def test_coordinator_run_failure_response(
report = coordinator.run()
assert len(report.reports) == 1
assert not report.successful
assert list(report.reports.values())[0].reason == "Invalid response"
assert list(report.reports)[0].reason == "Invalid response"
@pytest.mark.asyncio
async def test_notifier_coordinator_publication_failed(mock_publisher_valid):
mock_send = MagicMock()
mock_publisher_valid._send = mock_send
report = PublicationReport(
report = EventPublicationReport(
status=PublicationStatus.FAILED,
reason="some failure",
publication_id=UUID(int=1),
@ -128,3 +151,12 @@ async def test_notifier_coordinator_publication_failed(mock_publisher_valid):
# 4 = 2 reports * 2 notifiers
assert mock_send.call_count == 2
@pytest.mark.parametrize("num_publications", [2])
@pytest.mark.asyncio
async def test_recap_coordinator_run_success(mock_recap_publications,):
coordinator = RecapCoordinator(recap_publications=mock_recap_publications)
report = coordinator.run()
assert len(report.reports) == 2
assert report.successful, "\n".join(map(lambda rep: rep.reason, report.reports))

View File

@ -120,7 +120,7 @@ async def test_zulip_publisher(mocked_valid_response, setup_db, event):
)
).run()
assert list(report.reports.values())[0].status == PublicationStatus.COMPLETED
assert report.reports[0].status == PublicationStatus.COMPLETED
@pytest.mark.asyncio
@ -138,8 +138,8 @@ async def test_zulip_publishr_failure_invalid_credentials(
)
)
).run()
assert list(report.reports.values())[0].status == PublicationStatus.FAILED
assert list(report.reports.values())[0].reason == "Invalid credentials"
assert report.reports[0].status == PublicationStatus.FAILED
assert report.reports[0].reason == "Invalid credentials"
@pytest.mark.asyncio
@ -157,8 +157,8 @@ async def test_zulip_publisher_failure_client_error(
)
)
).run()
assert list(report.reports.values())[0].status == PublicationStatus.FAILED
assert list(report.reports.values())[0].reason == "400 Error - Invalid request"
assert report.reports[0].status == PublicationStatus.FAILED
assert report.reports[0].reason == "400 Error - Invalid request"
def test_event_validation(event):

View File

@ -9,7 +9,7 @@ from mobilizon_reshare.models.publication import PublicationStatus, Publication
from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers.coordinator import (
PublisherCoordinatorReport,
PublicationReport,
EventPublicationReport,
)
from mobilizon_reshare.storage.query import (
get_publishers,
@ -69,18 +69,18 @@ async def test_update_publishers(
complete_specification,
PublisherCoordinatorReport(
publications=[],
reports={
UUID(int=3): PublicationReport(
reports=[
EventPublicationReport(
status=PublicationStatus.FAILED,
reason="Invalid credentials",
publication_id=UUID(int=3),
),
UUID(int=4): PublicationReport(
EventPublicationReport(
status=PublicationStatus.COMPLETED,
reason="",
publication_id=UUID(int=4),
),
},
],
),
MobilizonEvent(
name="event_1",
@ -115,7 +115,7 @@ async def test_save_publication_report(
status=PublicationStatus.WAITING, event_mobilizon_id=event.mobilizon_id,
)
await save_publication_report(report, publications)
publication_ids = set(report.reports.keys())
publication_ids = set(publications.keys())
publications = {
p_id: await Publication.filter(id=p_id).first() for p_id in publication_ids
}