From 41317f062daaeb844c842ec77078444bdfb4d401 Mon Sep 17 00:00:00 2001 From: tinoilcotechino <70446383+tinoilcotechino@users.noreply.github.com> Date: Thu, 5 Aug 2021 00:29:50 +0200 Subject: [PATCH] Publication report (#46) * publishers: coordinator: Change `PublisherReport` to `PublicationReport`. * publishers: AbstractNotifier: Add `AbstractNotifier.get_name`. * models: Publication: Add `reason`. * storage: query: Add `get_mobilizon_event_publications`. * tests: query: Refactor models generation. * storage: query: Add `save_publication_report`. * Track publication ids during the publishing process. This patch changes the PublisherCoordinator to keep track of the different Publications it's performing. This also enables multiple publications for the same publisher. * tests: storage: Add some tests. Co-authored-by: Giacomo Leidi --- mobilizon_bots/cli/cli.py | 2 +- .../cli/{inspect.py => inspect_event.py} | 0 mobilizon_bots/cli/main.py | 42 +- mobilizon_bots/event/event.py | 6 +- mobilizon_bots/mobilizon/events.py | 4 +- mobilizon_bots/models/publication.py | 3 +- mobilizon_bots/publishers/coordinator.py | 93 +-- mobilizon_bots/storage/db.py | 8 +- mobilizon_bots/storage/query.py | 120 +++- tests/storage/test_query.py | 622 +++++++++++++++--- 10 files changed, 697 insertions(+), 203 deletions(-) rename mobilizon_bots/cli/{inspect.py => inspect_event.py} (100%) diff --git a/mobilizon_bots/cli/cli.py b/mobilizon_bots/cli/cli.py index 2af14dc..f3a0b88 100644 --- a/mobilizon_bots/cli/cli.py +++ b/mobilizon_bots/cli/cli.py @@ -5,7 +5,7 @@ from arrow import Arrow from click import pass_context, pass_obj from mobilizon_bots.cli import safe_execution -from mobilizon_bots.cli.inspect import inspect_events +from mobilizon_bots.cli.inspect_event import inspect_events from mobilizon_bots.cli.main import main from mobilizon_bots.event.event import EventPublicationStatus diff --git a/mobilizon_bots/cli/inspect.py b/mobilizon_bots/cli/inspect_event.py similarity index 100% rename from mobilizon_bots/cli/inspect.py rename to mobilizon_bots/cli/inspect_event.py diff --git a/mobilizon_bots/cli/main.py b/mobilizon_bots/cli/main.py index e1d52e6..19bf4af 100644 --- a/mobilizon_bots/cli/main.py +++ b/mobilizon_bots/cli/main.py @@ -1,20 +1,22 @@ import logging.config + from mobilizon_bots.event.event_selection_strategies import select_event_to_publish from mobilizon_bots.mobilizon.events import get_unpublished_events +from mobilizon_bots.models.publication import PublicationStatus from mobilizon_bots.publishers import get_active_publishers from mobilizon_bots.publishers.coordinator import PublisherCoordinator -from mobilizon_bots.storage.db import tear_down -from mobilizon_bots.storage.query import get_published_events, create_unpublished_events +from mobilizon_bots.storage.query import ( + get_published_events, + get_unpublished_events as get_db_unpublished_events, + create_unpublished_events, + save_publication_report, + publications_with_status, +) logger = logging.getLogger(__name__) -async def graceful_exit(code): - await tear_down() - exit(code) - - async def main(): """ STUB @@ -30,14 +32,26 @@ async def main(): unpublished_events = get_unpublished_events(published_events) # Store in the DB only the ones we didn't know about await create_unpublished_events(unpublished_events, active_publishers) - event = select_event_to_publish(published_events, unpublished_events) + event = select_event_to_publish( + published_events, + # We must load unpublished events from DB since it contains + # merged state between Mobilizon and previous WAITING events. + list(await get_db_unpublished_events()), + ) if event: logger.debug(f"Event to publish found: {event.name}") - result = PublisherCoordinator(event).run() + result = PublisherCoordinator( + event, + [ + (pub.id, pub.publisher.name) + for pub in await publications_with_status( + status=PublicationStatus.WAITING, + event_mobilizon_id=event.mobilizon_id, + ) + ], + ).run() + await save_publication_report(result, event) - logger.debug("Closing") - - await graceful_exit(0 if result.successful else 1) + return 0 if result.successful else 1 else: - logger.debug("Closing") - await graceful_exit(0) + return 0 diff --git a/mobilizon_bots/event/event.py b/mobilizon_bots/event/event.py index 5eb866b..97b7d35 100644 --- a/mobilizon_bots/event/event.py +++ b/mobilizon_bots/event/event.py @@ -85,10 +85,10 @@ class MobilizonEvent: description=event.description, begin_datetime=arrow.get( tortoise.timezone.localtime(value=event.begin_datetime, timezone=tz) - ), + ).to('local'), end_datetime=arrow.get( tortoise.timezone.localtime(value=event.end_datetime, timezone=tz) - ), + ).to('local'), mobilizon_link=event.mobilizon_link, mobilizon_id=event.mobilizon_id, thumbnail_link=event.thumbnail_link, @@ -96,7 +96,7 @@ class MobilizonEvent: publication_time={ pub.publisher.name: arrow.get( tortoise.timezone.localtime(value=pub.timestamp, timezone=tz) - ) + ).to('local') for pub in event.publications } if publication_status != PublicationStatus.WAITING diff --git a/mobilizon_bots/mobilizon/events.py b/mobilizon_bots/mobilizon/events.py index 5dfc629..3d7b8ee 100644 --- a/mobilizon_bots/mobilizon/events.py +++ b/mobilizon_bots/mobilizon/events.py @@ -7,7 +7,7 @@ import arrow import requests from mobilizon_bots.config.config import get_settings -from mobilizon_bots.event.event import MobilizonEvent, PublicationStatus +from mobilizon_bots.event.event import MobilizonEvent, EventPublicationStatus logger = logging.getLogger(__name__) @@ -39,7 +39,7 @@ def parse_event(data): thumbnail_link=parse_picture(data), location=parse_location(data), publication_time=None, - status=PublicationStatus.WAITING, + status=EventPublicationStatus.WAITING, ) diff --git a/mobilizon_bots/models/publication.py b/mobilizon_bots/models/publication.py index 9e7262e..603e334 100644 --- a/mobilizon_bots/models/publication.py +++ b/mobilizon_bots/models/publication.py @@ -15,8 +15,9 @@ class Publication(Model): status = fields.IntEnumField(PublicationStatus) # When a Publication's status is WAITING - # we don't need a timestamp + # we don't need a timestamp nor a reason timestamp = fields.DatetimeField(null=True) + reason = fields.TextField(null=True) event = fields.ForeignKeyField("models.Event", related_name="publications") publisher = fields.ForeignKeyField("models.Publisher", related_name="publications") diff --git a/mobilizon_bots/publishers/coordinator.py b/mobilizon_bots/publishers/coordinator.py index 135e730..114bb92 100644 --- a/mobilizon_bots/publishers/coordinator.py +++ b/mobilizon_bots/publishers/coordinator.py @@ -1,104 +1,83 @@ from dataclasses import dataclass, field -from enum import IntEnum -from typing import List +from uuid import UUID from mobilizon_bots.event.event import MobilizonEvent -from mobilizon_bots.publishers import get_active_publishers -from mobilizon_bots.publishers.abstract import AbstractPublisher +from mobilizon_bots.models.publication import PublicationStatus from mobilizon_bots.publishers.exceptions import PublisherError from mobilizon_bots.publishers.telegram import TelegramPublisher KEY2CLS = {"telegram": TelegramPublisher} -class PublisherStatus(IntEnum): - WAITING = 1 - FAILED = 2 - COMPLETED = 3 - - @dataclass -class PublisherReport: - status: PublisherStatus +class PublicationReport: + status: PublicationStatus reason: str - publisher: AbstractPublisher @dataclass class PublisherCoordinatorReport: - reports: List[PublisherReport] = field(default_factory=[]) + reports: dict[UUID, PublicationReport] = field(default_factory={}) @property def successful(self): - return all(r.status == PublisherStatus.COMPLETED for r in self.reports) + return all( + r.status == PublicationStatus.COMPLETED for r in self.reports.values() + ) def __iter__(self): - return self.reports.__iter__() + return self.reports.items().__iter__() class PublisherCoordinator: - def __init__(self, event: MobilizonEvent): - self.publishers = tuple(KEY2CLS[pn](event) for pn in get_active_publishers()) + def __init__(self, event: MobilizonEvent, publications: list[tuple[UUID, str]]): + self.publications = tuple( + (publication_id, KEY2CLS[publisher_name](event)) for publication_id, publisher_name in publications + ) def run(self) -> PublisherCoordinatorReport: - invalid_credentials, invalid_event, invalid_msg = self._validate() - errors = invalid_credentials + invalid_event + invalid_msg + errors = self._validate() if errors: return PublisherCoordinatorReport(reports=errors) return self._post() def _make_successful_report(self): - return [ - PublisherReport( - status=PublisherStatus.COMPLETED, + return { + publication_id: PublicationReport( + status=PublicationStatus.COMPLETED, reason="", - publisher=p, ) - for p in self.publishers - ] + for publication_id, _ in self.publications + } def _post(self): - failed_publishers_reports = [] - for p in self.publishers: + failed_publishers_reports = {} + for publication_id, p in self.publications: try: p.post() except PublisherError as e: - failed_publishers_reports.append( - PublisherReport( - status=PublisherStatus.FAILED, - reason=repr(e), - publisher=p, - ) + failed_publishers_reports[publication_id] = PublicationReport( + status=PublicationStatus.FAILED, + reason=repr(e), ) reports = failed_publishers_reports or self._make_successful_report() return PublisherCoordinatorReport(reports) def _validate(self): - invalid_credentials, invalid_event, invalid_msg = [], [], [] - for p in self.publishers: + errors: dict[UUID, PublicationReport] = {} + for publication_id, p in self.publications: + reason = [] if not p.are_credentials_valid(): - invalid_credentials.append( - PublisherReport( - status=PublisherStatus.FAILED, - reason="Invalid credentials", - publisher=p, - ) - ) + reason.append("Invalid credentials") if not p.is_event_valid(): - invalid_event.append( - PublisherReport( - status=PublisherStatus.FAILED, - reason="Invalid event", - publisher=p, - ) - ) + reason.append("Invalid event") if not p.is_message_valid(): - invalid_msg.append( - PublisherReport( - status=PublisherStatus.FAILED, - reason="Invalid message", - publisher=p, - ) + reason.append("Invalid message") + + if len(reason) > 0: + errors[publication_id] = PublicationReport( + status=PublicationStatus.FAILED, reason=", ".join(reason) ) - return invalid_credentials, invalid_event, invalid_msg + + return errors diff --git a/mobilizon_bots/storage/db.py b/mobilizon_bots/storage/db.py index 73defb2..0e5954b 100644 --- a/mobilizon_bots/storage/db.py +++ b/mobilizon_bots/storage/db.py @@ -6,7 +6,7 @@ from pathlib import Path from tortoise import Tortoise from mobilizon_bots.config.publishers import publisher_names -from mobilizon_bots.storage.query import create_publisher +from mobilizon_bots.storage.query import update_publishers logger = logging.getLogger(__name__) @@ -35,13 +35,11 @@ class MobilizonBotsDB: ) if not self.is_init: await Tortoise.generate_schemas() - for name in publisher_names: - logging.info(f"Creating {name} publisher") - # TODO: Deal with account_ref - await create_publisher(name) self.is_init = True logger.info(f"Succesfully initialized database at {self.path}") + await update_publishers(publisher_names) + @atexit.register def gracefully_tear_down(): diff --git a/mobilizon_bots/storage/query.py b/mobilizon_bots/storage/query.py index fef5c35..2ab40a6 100644 --- a/mobilizon_bots/storage/query.py +++ b/mobilizon_bots/storage/query.py @@ -1,6 +1,9 @@ -from typing import Iterable, Optional, List - +import logging import sys +from typing import Iterable, Optional, Union +from uuid import UUID + +import arrow from arrow import Arrow from tortoise.queryset import QuerySet from tortoise.transactions import atomic @@ -11,14 +14,17 @@ from mobilizon_bots.models.publication import Publication, PublicationStatus from mobilizon_bots.models.publisher import Publisher from mobilizon_bots.publishers.coordinator import PublisherCoordinatorReport +logger = logging.getLogger(__name__) + # This is due to Tortoise community fixtures to # set up and tear down a DB instance for Pytest. # See: https://github.com/tortoise/tortoise-orm/issues/419#issuecomment-696991745 # and: https://docs.pytest.org/en/stable/example/simple.html + CONNECTION_NAME = "models" if "pytest" in sys.modules else None -async def prefetch_event_relations(queryset: QuerySet[Event]) -> List[Event]: +async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]: return ( await queryset.prefetch_related("publications__publisher") .order_by("begin_datetime") @@ -27,18 +33,40 @@ async def prefetch_event_relations(queryset: QuerySet[Event]) -> List[Event]: def _add_date_window( - query, from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, + query, + field_name: str, + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, ): if from_date: - query = query.filter(end_datetime__gt=from_date.datetime) + query = query.filter(**{f"{field_name}__gt": from_date.to("utc").datetime}) if to_date: - query = query.filter(end_datetime__lt=to_date.datetime) + query = query.filter(**{f"{field_name}__lt": to_date.to("utc").datetime}) return query +@atomic(CONNECTION_NAME) +async def publications_with_status( + status: PublicationStatus, + event_mobilizon_id: Optional[UUID] = None, + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, +) -> Iterable[Publication]: + query = Publication.filter(status=status) + + if event_mobilizon_id: + query = query.prefetch_related("event").filter( + event__mobilizon_id=event_mobilizon_id + ) + + query = _add_date_window(query, "timestamp", from_date, to_date) + + return await query.prefetch_related("publisher").order_by("timestamp").distinct() + + async def events_with_status( - status: List[EventPublicationStatus], + status: list[EventPublicationStatus], from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, ) -> Iterable[MobilizonEvent]: @@ -49,21 +77,27 @@ async def events_with_status( return event_status in status query = Event.all() - _add_date_window(query, from_date, to_date) + return map( MobilizonEvent.from_model, - filter(_filter_event_with_status, await prefetch_event_relations(query),), + filter( + _filter_event_with_status, + await prefetch_event_relations( + _add_date_window(query, "begin_datetime", from_date, to_date) + ), + ), ) async def get_all_events( - from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, ) -> Iterable[MobilizonEvent]: return map( MobilizonEvent.from_model, await prefetch_event_relations( - _add_date_window(Event.all(), from_date, to_date) + _add_date_window(Event.all(), "begin_datetime", from_date, to_date) ), ) @@ -76,18 +110,56 @@ async def get_unpublished_events() -> Iterable[MobilizonEvent]: return await events_with_status([EventPublicationStatus.WAITING]) -async def save_event(event): +async def get_mobilizon_event_publications( + event: MobilizonEvent, +) -> Iterable[Publication]: + models = await prefetch_event_relations( + Event.filter(mobilizon_id=event.mobilizon_id) + ) + return models[0].publications + + +async def get_publishers( + name: Optional[str] = None, +) -> Union[Publisher, Iterable[Publisher]]: + if name: + return await Publisher.filter(name=name).first() + else: + return await Publisher.all() + + +async def save_event(event: MobilizonEvent) -> Event: event_model = event.to_model() await event_model.save() return event_model -async def save_publication(publisher_name, event_model, status: PublicationStatus): +async def create_publisher(name: str, account_ref: Optional[str] = None) -> None: + await Publisher.create(name=name, account_ref=account_ref) - publisher = await Publisher.filter(name=publisher_name).first() + +@atomic(CONNECTION_NAME) +async def update_publishers( + names: Iterable[str], +) -> None: + names = set(names) + known_publisher_names = set(p.name for p in await get_publishers()) + for name in names.difference(known_publisher_names): + logging.info(f"Creating {name} publisher") + await create_publisher(name) + + +@atomic(CONNECTION_NAME) +async def save_publication( + publisher_name: str, event_model: Event, status: PublicationStatus +) -> None: + + publisher = await get_publishers(publisher_name) await Publication.create( - status=status, event_id=event_model.id, publisher_id=publisher.id, + status=status, + event_id=event_model.id, + publisher_id=publisher.id, ) @@ -115,10 +187,18 @@ async def create_unpublished_events( ) -async def create_publisher(name: str, account_ref: Optional[str] = None) -> None: - await Publisher.create(name=name, account_ref=account_ref) +@atomic(CONNECTION_NAME) +async def save_publication_report( + coordinator_report: PublisherCoordinatorReport, event: MobilizonEvent +) -> None: + publications: dict[UUID, Publication] = { + p.id: p for p in await get_mobilizon_event_publications(event) + } + for publication_id, publication_report in coordinator_report: -async def save_publication_report(publication_report: PublisherCoordinatorReport): - for publisher_report in publication_report: - pass + publications[publication_id].status = publication_report.status + publications[publication_id].reason = publication_report.reason + publications[publication_id].timestamp = arrow.now().datetime + + await publications[publication_id].save() diff --git a/tests/storage/test_query.py b/tests/storage/test_query.py index a322838..131b4c7 100644 --- a/tests/storage/test_query.py +++ b/tests/storage/test_query.py @@ -1,143 +1,565 @@ from datetime import datetime, timedelta, timezone +from typing import Union +from uuid import UUID import arrow import pytest -from mobilizon_bots.event.event import MobilizonEvent -from mobilizon_bots.models.publication import PublicationStatus +from mobilizon_bots.event.event import MobilizonEvent, EventPublicationStatus +from mobilizon_bots.models.event import Event +from mobilizon_bots.models.publication import PublicationStatus, Publication +from mobilizon_bots.models.publisher import Publisher +from mobilizon_bots.publishers.coordinator import ( + PublisherCoordinatorReport, + PublicationReport, +) from mobilizon_bots.storage.query import ( get_published_events, get_unpublished_events, create_unpublished_events, + get_mobilizon_event_publications, + prefetch_event_relations, + get_publishers, + publications_with_status, + update_publishers, + save_publication_report, ) +today = datetime( + year=2021, + month=6, + day=6, + hour=5, + minute=0, + tzinfo=timezone(timedelta(hours=2)), +) + +two_publishers_specification = {"publisher": 2} + +complete_specification = { + "event": 4, + "publications": [ + {"event_idx": 0, "publisher_idx": 0}, + { + "event_idx": 0, + "publisher_idx": 1, + "status": PublicationStatus.COMPLETED, + }, + { + "event_idx": 1, + "publisher_idx": 0, + "status": PublicationStatus.WAITING, + }, + { + "event_idx": 1, + "publisher_idx": 1, + }, + { + "event_idx": 2, + "publisher_idx": 2, + "status": PublicationStatus.FAILED, + }, + { + "event_idx": 2, + "publisher_idx": 1, + "status": PublicationStatus.COMPLETED, + }, + { + "event_idx": 3, + "publisher_idx": 2, + "status": PublicationStatus.COMPLETED, + }, + ], +} + + @pytest.fixture(scope="module") -def setup(): - async def _setup( - publisher_model_generator, publication_model_generator, event_model_generator - ): - today = datetime( - year=2021, - month=6, - day=6, - hour=5, - minute=0, - tzinfo=timezone(timedelta(hours=2)), - ) - publisher_1 = publisher_model_generator() - publisher_2 = publisher_model_generator(idx=2) - await publisher_1.save() - await publisher_2.save() +def generate_models(): + async def _generate_models(specification: dict[str, Union[int, list]]): + publishers = [] + for i in range( + specification["publisher"] if "publisher" in specification.keys() else 3 + ): + publisher = Publisher( + id=UUID(int=i), name=f"publisher_{i}", account_ref=f"account_ref_{i}" + ) + publishers.append(publisher) + await publisher.save() - event_1 = event_model_generator(begin_date=today) - event_2 = event_model_generator(idx=2, begin_date=today + timedelta(days=2)) - event_3 = event_model_generator(idx=3, begin_date=today + timedelta(days=-2)) - event_4 = event_model_generator(idx=4, begin_date=today + timedelta(days=-4)) - await event_1.save() - await event_2.save() - await event_3.save() - await event_4.save() + events = [] + if "event" in specification.keys(): + for i in range(specification["event"]): + begin_date = today + timedelta(days=i) + event = Event( + id=UUID(int=i), + name=f"event_{i}", + description=f"desc_{i}", + mobilizon_id=f"mobid_{i}", + mobilizon_link=f"moblink_{i}", + thumbnail_link=f"thumblink_{i}", + location=f"loc_{i}", + begin_datetime=begin_date, + end_datetime=begin_date + timedelta(hours=2), + ) + events.append(event) + await event.save() - publication_1 = publication_model_generator( - event_id=event_1.id, publisher_id=publisher_1.id - ) - publication_2 = publication_model_generator( - event_id=event_1.id, - publisher_id=publisher_2.id, - status=PublicationStatus.COMPLETED, - ) - publication_3 = publication_model_generator( - event_id=event_2.id, - publisher_id=publisher_1.id, - status=PublicationStatus.FAILED, - ) - publication_4 = publication_model_generator( - event_id=event_3.id, - publisher_id=publisher_2.id, - status=PublicationStatus.WAITING, - ) - publication_5 = publication_model_generator( - event_id=event_4.id, - publisher_id=publisher_2.id, - status=PublicationStatus.COMPLETED, - ) - await publication_1.save() - await publication_2.save() - await publication_3.save() - await publication_4.save() - await publication_5.save() - return ( - [event_1, event_2, event_3, event_4], - [publication_1, publication_2, publication_3, publication_4, publication_5], - [publisher_1, publisher_2], - today, - ) + if "publications" in specification.keys(): + for i in range(len(specification["publications"])): + await Publication.create( + id=UUID(int=i), + status=specification["publications"][i].get( + "status", PublicationStatus.WAITING + ), + timestamp=specification["publications"][i].get( + "timestamp", today + timedelta(hours=i) + ), + event_id=events[specification["publications"][i]["event_idx"]].id, + publisher_id=publishers[ + specification["publications"][i]["publisher_idx"] + ].id, + ) - return _setup + return _generate_models @pytest.mark.asyncio -async def test_get_published_events( - publisher_model_generator, publication_model_generator, event_model_generator, setup -): - events, publications, publishers, today = await setup( - publisher_model_generator, publication_model_generator, event_model_generator - ) - +@pytest.mark.parametrize( + "specification,expected_result", + [ + [ + complete_specification, + [ + MobilizonEvent( + name="event_3", + description="desc_3", + mobilizon_id="mobid_3", + mobilizon_link="moblink_3", + thumbnail_link="thumblink_3", + location="loc_3", + publication_time={ + "publisher_2": arrow.get(today + timedelta(hours=6)), + }, + status=EventPublicationStatus.COMPLETED, + begin_datetime=arrow.get(today + timedelta(days=3)), + end_datetime=arrow.get( + today + timedelta(days=3) + timedelta(hours=2) + ), + ) + ], + ] + ], +) +async def test_get_published_events(specification, expected_result, generate_models): + await generate_models(specification) published_events = list(await get_published_events()) - assert len(published_events) == 1 - assert published_events[0].mobilizon_id == events[3].mobilizon_id - - assert published_events[0].begin_datetime == arrow.get(today + timedelta(days=-4)) + assert len(published_events) == len(expected_result) + assert published_events == expected_result @pytest.mark.asyncio -async def test_get_unpublished_events( - publisher_model_generator, publication_model_generator, event_model_generator, setup -): - events, publications, publishers, today = await setup( - publisher_model_generator, publication_model_generator, event_model_generator - ) - +@pytest.mark.parametrize( + "specification,expected_result", + [ + [ + complete_specification, + [ + MobilizonEvent( + name="event_1", + description="desc_1", + mobilizon_id="mobid_1", + mobilizon_link="moblink_1", + thumbnail_link="thumblink_1", + location="loc_1", + status=EventPublicationStatus.WAITING, + begin_datetime=arrow.get(today + timedelta(days=1)), + end_datetime=arrow.get( + today + timedelta(days=1) + timedelta(hours=2) + ), + ), + ], + ] + ], +) +async def test_get_unpublished_events(specification, expected_result, generate_models): + await generate_models(specification) unpublished_events = list(await get_unpublished_events()) - assert len(unpublished_events) == 1 - assert unpublished_events[0].mobilizon_id == events[2].mobilizon_id - assert unpublished_events[0].begin_datetime == events[2].begin_datetime + assert len(unpublished_events) == len(expected_result) + assert unpublished_events == expected_result @pytest.mark.asyncio +@pytest.mark.parametrize( + "specification,expected_result", + [ + [ + complete_specification, + [ + Event( + name="event_1", + description="desc_1", + mobilizon_id="mobid_1", + mobilizon_link="moblink_1", + thumbnail_link="thumblink_1", + location="loc_1", + begin_datetime=today + timedelta(days=1), + end_datetime=today + timedelta(days=1) + timedelta(hours=2), + ), + Event( + name="test event", + description="description of the event", + mobilizon_id="12345", + mobilizon_link="http://some_link.com/123", + thumbnail_link="http://some_link.com/123.jpg", + location="location", + begin_datetime=today + timedelta(days=6), + end_datetime=today + timedelta(days=6) + timedelta(hours=2), + ), + Event( + name="test event", + description="description of the event", + mobilizon_id="67890", + mobilizon_link="http://some_link.com/123", + thumbnail_link="http://some_link.com/123.jpg", + location="location", + begin_datetime=today + timedelta(days=12), + end_datetime=today + timedelta(days=12) + timedelta(hours=2), + ), + ], + ] + ], +) async def test_create_unpublished_events( - publisher_model_generator, - publication_model_generator, - event_model_generator, + specification, + expected_result, + generate_models, event_generator, - setup, ): - events, publications, publishers, today = await setup( - publisher_model_generator, publication_model_generator, event_model_generator - ) + await generate_models(specification) - event_4 = event_generator(begin_date=arrow.get(today + timedelta(days=6))) - event_5 = event_generator( + event_3 = event_generator(begin_date=arrow.get(today + timedelta(days=6))) + event_4 = event_generator( begin_date=arrow.get(today + timedelta(days=12)), mobilizon_id="67890" ) + models = await prefetch_event_relations(Event.filter(name="event_1")) - await events[0].fetch_related("publications") - await events[0].fetch_related("publications__publisher") - events_from_internet = [MobilizonEvent.from_model(events[0]), event_4, event_5] + events_from_internet = [MobilizonEvent.from_model(models[0]), event_3, event_4] await create_unpublished_events( unpublished_mobilizon_events=events_from_internet, - active_publishers=["publisher_1", "publisher_2"], + active_publishers=["publisher_0", "publisher_1", "publisher_2"], ) unpublished_events = list(await get_unpublished_events()) - assert len(unpublished_events) == 4 - assert unpublished_events[0].mobilizon_id == "mobid_3" - assert unpublished_events[1].mobilizon_id == "mobid_1" - assert unpublished_events[2].mobilizon_id == "12345" - assert unpublished_events[3].mobilizon_id == "67890" + assert len(unpublished_events) == 3 + assert unpublished_events[0].mobilizon_id == unpublished_events[0].mobilizon_id + assert unpublished_events[1].mobilizon_id == unpublished_events[1].mobilizon_id + assert unpublished_events[2].mobilizon_id == unpublished_events[2].mobilizon_id + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "specification", + [ + complete_specification, + ], +) +async def test_get_mobilizon_event_publications(specification, generate_models): + await generate_models(specification) + + models = await prefetch_event_relations(Event.filter(name="event_0")) + mobilizon_event = MobilizonEvent.from_model(models[0]) + + publications = list(await get_mobilizon_event_publications(mobilizon_event)) + for pub in publications: + await pub.fetch_related("event") + await pub.fetch_related("publisher") + + assert len(publications) == 2 + + assert publications[0].event.name == "event_0" + assert publications[0].publisher.name == "publisher_0" + assert publications[0].status == PublicationStatus.WAITING + + assert publications[1].event.name == "event_0" + assert publications[1].publisher.name == "publisher_1" + assert publications[1].status == PublicationStatus.COMPLETED + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "specification,name,expected_result", + [ + [ + complete_specification, + None, + { + "publisher_0", + "publisher_1", + "publisher_2", + }, + ], + [ + complete_specification, + "publisher_0", + {"publisher_0"}, + ], + ], +) +async def test_get_publishers( + specification, + name, + expected_result, + generate_models, +): + await generate_models(specification) + result = await get_publishers(name) + + if type(result) == list: + publishers = set(p.name for p in result) + else: + publishers = {result.name} + + assert len(publishers) == len(expected_result) + assert publishers == expected_result + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "specification,status,mobilizon_id,from_date,to_date,expected_result", + [ + [ + complete_specification, + PublicationStatus.WAITING, + None, + None, + None, + [ + Publication( + id=UUID(int=0), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=0), + event_id=UUID(int=0), + publisher_id=UUID(int=0), + ), + Publication( + id=UUID(int=2), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=2), + event_id=UUID(int=0), + publisher_id=UUID(int=1), + ), + Publication( + id=UUID(int=3), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=3), + event_id=UUID(int=1), + publisher_id=UUID(int=1), + ), + ], + ], + [ + complete_specification, + PublicationStatus.WAITING, + "mobid_1", + None, + None, + [ + Publication( + id=UUID(int=2), + status=PublicationStatus.COMPLETED, + timestamp=today + timedelta(hours=2), + event_id=UUID(int=1), + publisher_id=UUID(int=1), + ), + Publication( + id=UUID(int=3), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=5), + event_id=UUID(int=1), + publisher_id=UUID(int=1), + ), + ], + ], + [ + complete_specification, + PublicationStatus.WAITING, + None, + arrow.get(today + timedelta(hours=-1)), + arrow.get(today + timedelta(hours=1)), + [ + Publication( + id=UUID(int=0), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=0), + event_id=UUID(int=0), + publisher_id=UUID(int=0), + ), + ], + ], + [ + complete_specification, + PublicationStatus.WAITING, + None, + arrow.get(today + timedelta(hours=1)), + None, + [ + Publication( + id=UUID(int=2), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=2), + event_id=UUID(int=0), + publisher_id=UUID(int=1), + ), + Publication( + id=UUID(int=3), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=5), + event_id=UUID(int=1), + publisher_id=UUID(int=1), + ), + ], + ], + [ + complete_specification, + PublicationStatus.WAITING, + None, + None, + arrow.get(today + timedelta(hours=1)), + [ + Publication( + id=UUID(int=0), + status=PublicationStatus.WAITING, + timestamp=today + timedelta(hours=0), + event_id=UUID(int=0), + publisher_id=UUID(int=0), + ), + ], + ], + ], +) +async def test_publications_with_status( + specification, + status, + mobilizon_id, + from_date, + to_date, + expected_result, + generate_models, +): + await generate_models(specification) + publications = await publications_with_status( + status=status, + event_mobilizon_id=mobilizon_id, + from_date=from_date, + to_date=to_date, + ) + + assert len(publications) == len(expected_result) + assert publications == expected_result + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "specification,names,expected_result", + [ + [ + two_publishers_specification, + ["publisher_0", "publisher_1"], + { + Publisher(id=UUID(int=0), name="publisher_0"), + Publisher(id=UUID(int=1), name="publisher_1"), + }, + ], + [ + {"publisher": 0}, + ["publisher_0", "publisher_1"], + {"publisher_0", "publisher_1"}, + ], + [ + two_publishers_specification, + ["publisher_0", "publisher_2", "publisher_3"], + {"publisher_0", "publisher_1", "publisher_2", "publisher_3"}, + ], + ], +) +async def test_update_publishers( + specification, + names, + expected_result, + generate_models, +): + await generate_models(specification) + await update_publishers(names) + if type(list(expected_result)[0]) == Publisher: + publishers = set(await get_publishers()) + else: + publishers = set(p.name for p in await get_publishers()) + + assert len(publishers) == len(expected_result) + assert publishers == expected_result + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "specification,report,event,expected_result", + [ + [ + complete_specification, + PublisherCoordinatorReport( + reports={ + UUID(int=2): PublicationReport( + status=PublicationStatus.FAILED, reason="Invalid credentials" + ), + UUID(int=3): PublicationReport( + status=PublicationStatus.COMPLETED, reason="" + ), + } + ), + MobilizonEvent( + name="event_1", + description="desc_1", + mobilizon_id="mobid_1", + mobilizon_link="moblink_1", + thumbnail_link="thumblink_1", + location="loc_1", + status=EventPublicationStatus.WAITING, + begin_datetime=arrow.get(today + timedelta(days=1)), + end_datetime=arrow.get(today + timedelta(days=1) + timedelta(hours=2)), + ), + { + UUID(int=2): Publication( + id=UUID(int=2), + status=PublicationStatus.FAILED, + reason="Invalid credentials", + ), + UUID(int=3): Publication( + id=UUID(int=0), status=PublicationStatus.COMPLETED, reason="" + ), + }, + ], + ], +) +async def test_save_publication_report( + specification, + report, + event, + expected_result, + generate_models, +): + await generate_models(specification) + await save_publication_report(report, event) + publication_ids = set(report.reports.keys()) + publications = { + p_id: await Publication.filter(id=p_id).first() for p_id in publication_ids + } + + assert len(publications) == len(expected_result) + for i in publication_ids: + assert publications[i].status == expected_result[i].status + assert publications[i].reason == expected_result[i].reason + assert publications[i].timestamp