From 4dc1e4080aa4d6ba27e6939e54906d978415f587 Mon Sep 17 00:00:00 2001 From: Simone Robutti Date: Thu, 11 Nov 2021 15:18:04 +0100 Subject: [PATCH] fix publisher deactivation (#93) * filtering publications with inactive publishers * filtering publications with inactive publishers * WIP: Generate publications at runtime. TODO: - change `MobilizonEvent.compute_status`'s contract and break everything - while we're at it we should remove `PublicationStatus.WAITING` - test `storage.query.create_publications_for_publishers` * cli: inspect_events: Unnest if-then-else. * publishers: abstract: Remove `EventPublication.make`. * fixed tests * split query.py file * added tests for get_unpublished_events * added tests * more tests * added start test * main: start: Remove filter_publications_with_inactive_publishers. Co-authored-by: Giacomo Leidi --- .../cli/commands/inspect/inspect_event.py | 29 ++- mobilizon_reshare/config/publishers.py | 8 +- mobilizon_reshare/event/event.py | 16 +- .../event/event_selection_strategies.py | 33 ++- mobilizon_reshare/main/recap.py | 2 +- mobilizon_reshare/main/start.py | 43 ++-- mobilizon_reshare/models/event.py | 26 ++ mobilizon_reshare/models/publication.py | 6 +- mobilizon_reshare/publishers/abstract.py | 6 +- mobilizon_reshare/publishers/coordinator.py | 13 +- mobilizon_reshare/storage/db.py | 4 +- mobilizon_reshare/storage/query/__init__.py | 3 + .../storage/query/model_creation.py | 14 + .../storage/{query.py => query/read_query.py} | 231 +++++++---------- mobilizon_reshare/storage/query/save_query.py | 53 ++++ tests/commands/test_start.py | 106 ++++++++ tests/conftest.py | 96 ++++++- tests/event/test_strategies.py | 16 +- tests/mobilizon/conftest.py | 25 +- tests/models/test_event.py | 123 +++------ tests/models/test_publication.py | 30 --- tests/publishers/conftest.py | 51 ---- tests/publishers/test_coordinator.py | 3 +- tests/publishers/test_zulip.py | 57 ++-- tests/storage/__init__.py | 15 +- tests/storage/conftest.py | 5 +- tests/storage/test_query.py | 243 +++++------------- tests/storage/test_read_query.py | 92 +++++++ tests/storage/test_update.py | 27 +- 29 files changed, 709 insertions(+), 667 deletions(-) create mode 100644 mobilizon_reshare/storage/query/__init__.py create mode 100644 mobilizon_reshare/storage/query/model_creation.py rename mobilizon_reshare/storage/{query.py => query/read_query.py} (50%) create mode 100644 mobilizon_reshare/storage/query/save_query.py create mode 100644 tests/commands/test_start.py delete mode 100644 tests/models/test_publication.py create mode 100644 tests/storage/test_read_query.py diff --git a/mobilizon_reshare/cli/commands/inspect/inspect_event.py b/mobilizon_reshare/cli/commands/inspect/inspect_event.py index d994310..b1328b8 100644 --- a/mobilizon_reshare/cli/commands/inspect/inspect_event.py +++ b/mobilizon_reshare/cli/commands/inspect/inspect_event.py @@ -5,9 +5,13 @@ from arrow import Arrow from mobilizon_reshare.event.event import EventPublicationStatus from mobilizon_reshare.event.event import MobilizonEvent -from mobilizon_reshare.storage.query import get_all_events -from mobilizon_reshare.storage.query import events_with_status - +from mobilizon_reshare.event.event_selection_strategies import select_unpublished_events +from mobilizon_reshare.storage.query.read_query import ( + get_published_events, + events_with_status, + get_all_events, + events_without_publications, +) status_to_color = { EventPublicationStatus.COMPLETED: "green", @@ -28,15 +32,22 @@ def pretty(event: MobilizonEvent): ) +async def inspect_unpublished_events(frm: Arrow = None, to: Arrow = None): + return select_unpublished_events( + list(await get_published_events(from_date=frm, to_date=to)), + list(await events_without_publications(from_date=frm, to_date=to)), + ) + + async def inspect_events( status: EventPublicationStatus = None, frm: Arrow = None, to: Arrow = None ): - - events = ( - await events_with_status([status], from_date=frm, to_date=to) - if status - else await get_all_events(from_date=frm, to_date=to) - ) + if status is None: + events = await get_all_events(from_date=frm, to_date=to) + elif status == EventPublicationStatus.WAITING: + events = await inspect_unpublished_events(frm=frm, to=to) + else: + events = await events_with_status([status], from_date=frm, to_date=to) if events: show_events(events) diff --git a/mobilizon_reshare/config/publishers.py b/mobilizon_reshare/config/publishers.py index 1f66dfc..a6b7958 100644 --- a/mobilizon_reshare/config/publishers.py +++ b/mobilizon_reshare/config/publishers.py @@ -57,9 +57,11 @@ publisher_names = publisher_name_to_validators.keys() def get_active_publishers(settings): - return filter( - lambda publisher_name: settings["publisher"][publisher_name]["active"], - publisher_names, + return list( + filter( + lambda publisher_name: settings["publisher"][publisher_name]["active"], + publisher_names, + ) ) diff --git a/mobilizon_reshare/event/event.py b/mobilizon_reshare/event/event.py index 1072e60..1fe8929 100644 --- a/mobilizon_reshare/event/event.py +++ b/mobilizon_reshare/event/event.py @@ -36,6 +36,8 @@ class MobilizonEvent: def __post_init__(self): assert self.begin_datetime.tzinfo == self.end_datetime.tzinfo assert self.begin_datetime < self.end_datetime + if self.publication_time is None: + self.publication_time = {} if self.publication_time: assert self.status in [ EventPublicationStatus.COMPLETED, @@ -63,15 +65,16 @@ class MobilizonEvent: @staticmethod def compute_status(publications: list[Publication]) -> EventPublicationStatus: + if not publications: + return EventPublicationStatus.WAITING + unique_statuses: Set[PublicationStatus] = set( pub.status for pub in publications ) - if PublicationStatus.FAILED in unique_statuses: - return EventPublicationStatus.FAILED - elif unique_statuses == { + if unique_statuses == { PublicationStatus.COMPLETED, - PublicationStatus.WAITING, + PublicationStatus.FAILED, }: return EventPublicationStatus.PARTIAL elif len(unique_statuses) == 1: @@ -100,8 +103,7 @@ class MobilizonEvent: tortoise.timezone.localtime(value=pub.timestamp, timezone=tz) ).to("local") for pub in event.publications - } - if publication_status != PublicationStatus.WAITING - else None, + if publication_status != EventPublicationStatus.WAITING + }, status=publication_status, ) diff --git a/mobilizon_reshare/event/event_selection_strategies.py b/mobilizon_reshare/event/event_selection_strategies.py index e46ef3e..bac1ce8 100644 --- a/mobilizon_reshare/event/event_selection_strategies.py +++ b/mobilizon_reshare/event/event_selection_strategies.py @@ -18,8 +18,13 @@ class EventSelectionStrategy(ABC): ) -> Optional[MobilizonEvent]: if not self.is_in_publishing_window(): + logger.info("Outside of publishing window, no event will be published.") + return None + selected = self._select(published_events, unpublished_events) + if selected: + return selected[0] + else: return None - return self._select(published_events, unpublished_events) def is_in_publishing_window(self) -> bool: settings = get_settings() @@ -36,7 +41,7 @@ class EventSelectionStrategy(ABC): self, published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent], - ) -> Optional[MobilizonEvent]: + ) -> Optional[List[MobilizonEvent]]: pass @@ -45,21 +50,19 @@ class SelectNextEventStrategy(EventSelectionStrategy): self, published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent], - ) -> Optional[MobilizonEvent]: + ) -> Optional[List[MobilizonEvent]]: # if there are no unpublished events, there's nothing I can do if not unpublished_events: logger.debug("No event to publish.") - return None - - first_unpublished_event = unpublished_events[0] + return [] # if there's no published event (first execution) I return the next in queue if not published_events: logger.debug( "First Execution with an available event. Picking next event in the queue." ) - return first_unpublished_event + return unpublished_events last_published_event = published_events[-1] now = arrow.now() @@ -84,14 +87,26 @@ class SelectNextEventStrategy(EventSelectionStrategy): logger.debug( "Last event was published recently. No event is going to be published." ) - return None + return [] - return first_unpublished_event + return unpublished_events STRATEGY_NAME_TO_STRATEGY_CLASS = {"next_event": SelectNextEventStrategy} +def select_unpublished_events( + published_events: List[MobilizonEvent], + unpublished_events: List[MobilizonEvent], +): + + strategy = STRATEGY_NAME_TO_STRATEGY_CLASS[ + get_settings()["selection"]["strategy"] + ]() + + return strategy._select(published_events, unpublished_events) + + def select_event_to_publish( published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent], diff --git a/mobilizon_reshare/main/recap.py b/mobilizon_reshare/main/recap.py index 7edc559..5a2dacc 100644 --- a/mobilizon_reshare/main/recap.py +++ b/mobilizon_reshare/main/recap.py @@ -14,7 +14,7 @@ from mobilizon_reshare.publishers.platforms.platform_mapping import ( get_publisher_class, get_formatter_class, ) -from mobilizon_reshare.storage.query import events_with_status +from mobilizon_reshare.storage.query.read_query import events_with_status async def select_events_to_recap() -> List[MobilizonEvent]: diff --git a/mobilizon_reshare/main/start.py b/mobilizon_reshare/main/start.py index 0e56c01..0be1a76 100644 --- a/mobilizon_reshare/main/start.py +++ b/mobilizon_reshare/main/start.py @@ -1,26 +1,30 @@ -import logging -from functools import partial +import logging.config from mobilizon_reshare.event.event_selection_strategies import select_event_to_publish from mobilizon_reshare.mobilizon.events import get_unpublished_events -from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.publishers.abstract import EventPublication from mobilizon_reshare.publishers.coordinator import ( PublicationFailureNotifiersCoordinator, ) from mobilizon_reshare.publishers.coordinator import PublisherCoordinator -from mobilizon_reshare.storage.query import ( - get_published_events, - get_unpublished_events as get_db_unpublished_events, +from mobilizon_reshare.storage.query.model_creation import ( + create_event_publication_models, +) +from mobilizon_reshare.storage.query.read_query import get_published_events +from mobilizon_reshare.storage.query.save_query import ( create_unpublished_events, save_publication_report, - publications_with_status, ) logger = logging.getLogger(__name__) async def start(): + """ + STUB + :return: + """ + # TODO: the logic to get published and unpublished events is probably redundant. # We need a simpler way to bring together events from mobilizon, unpublished events from the db # and published events from the DB @@ -31,29 +35,24 @@ async def start(): # Pull unpublished events from Mobilizon unpublished_events = get_unpublished_events(published_events) # Store in the DB only the ones we didn't know about - await create_unpublished_events(unpublished_events) + db_unpublished_events = await create_unpublished_events(unpublished_events) event = select_event_to_publish( published_events, # We must load unpublished events from DB since it contains # merged state between Mobilizon and previous WAITING events. - list(await get_db_unpublished_events()), + db_unpublished_events, ) if event: logger.debug(f"Event to publish found: {event.name}") - waiting_publications_models = await publications_with_status( - status=PublicationStatus.WAITING, event_mobilizon_id=event.mobilizon_id, - ) - waiting_publications = list( - map( - partial(EventPublication.from_orm, event=event), - waiting_publications_models.values(), - ) - ) + models = await create_event_publication_models(event) + publications = list(EventPublication.from_orm(m, event) for m in models) + reports = PublisherCoordinator(publications).run() - reports = PublisherCoordinator(waiting_publications).run() - - await save_publication_report(reports, waiting_publications_models) + await save_publication_report(reports, models) for report in reports.reports: - PublicationFailureNotifiersCoordinator(report).notify_failure() + if not report.succesful: + PublicationFailureNotifiersCoordinator(report).notify_failure() + else: + logger.debug("No event to publish found") diff --git a/mobilizon_reshare/models/event.py b/mobilizon_reshare/models/event.py index c5a16b8..892bda8 100644 --- a/mobilizon_reshare/models/event.py +++ b/mobilizon_reshare/models/event.py @@ -1,6 +1,10 @@ from tortoise import fields from tortoise.models import Model +from mobilizon_reshare.models.publication import PublicationStatus, Publication +from mobilizon_reshare.models.publisher import Publisher +from mobilizon_reshare.publishers import get_active_publishers + class Event(Model): id = fields.UUIDField(pk=True) @@ -26,3 +30,25 @@ class Event(Model): class Meta: table = "event" + + async def build_unsaved_publication_models(self): + result = [] + publishers = get_active_publishers() + for publisher in publishers: + result.append( + await self.build_publication_by_publisher_name( + publisher, PublicationStatus.UNSAVED + ) + ) + return result + + async def build_publication_by_publisher_name( + self, publisher_name: str, status: PublicationStatus + ) -> Publication: + publisher = await Publisher.filter(name=publisher_name).first() + return Publication( + status=status, + event_id=self.id, + publisher_id=publisher.id, + publisher=publisher, + ) diff --git a/mobilizon_reshare/models/publication.py b/mobilizon_reshare/models/publication.py index 603e334..49b323f 100644 --- a/mobilizon_reshare/models/publication.py +++ b/mobilizon_reshare/models/publication.py @@ -5,9 +5,9 @@ from tortoise.models import Model class PublicationStatus(IntEnum): - WAITING = 1 - FAILED = 2 - COMPLETED = 3 + UNSAVED = 0 + FAILED = 1 + COMPLETED = 2 class Publication(Model): diff --git a/mobilizon_reshare/publishers/abstract.py b/mobilizon_reshare/publishers/abstract.py index 16032d8..0e03c39 100644 --- a/mobilizon_reshare/publishers/abstract.py +++ b/mobilizon_reshare/publishers/abstract.py @@ -2,7 +2,7 @@ import inspect import logging from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List +from typing import List, Optional from uuid import UUID from dynaconf.utils.boxing import DynaBox @@ -10,7 +10,9 @@ from jinja2 import Environment, FileSystemLoader, Template from mobilizon_reshare.config.config import get_settings from mobilizon_reshare.event.event import MobilizonEvent -from mobilizon_reshare.models.publication import Publication as PublicationModel +from mobilizon_reshare.models.publication import ( + Publication as PublicationModel, +) from .exceptions import PublisherError, InvalidAttribute JINJA_ENV = Environment(loader=FileSystemLoader("/")) diff --git a/mobilizon_reshare/publishers/coordinator.py b/mobilizon_reshare/publishers/coordinator.py index 2976fd3..eba980b 100644 --- a/mobilizon_reshare/publishers/coordinator.py +++ b/mobilizon_reshare/publishers/coordinator.py @@ -20,6 +20,10 @@ class BasePublicationReport: status: PublicationStatus reason: Optional[str] + @property + def succesful(self): + return self.status == PublicationStatus.COMPLETED + def get_failure_message(self): return ( @@ -72,15 +76,6 @@ class PublisherCoordinator: return self._post() - def _make_successful_report(self, failed_ids): - return [ - EventPublicationReport( - status=PublicationStatus.COMPLETED, reason="", publication=publication, - ) - for publication in self.publications - if publication.id not in failed_ids - ] - def _post(self): reports = [] diff --git a/mobilizon_reshare/storage/db.py b/mobilizon_reshare/storage/db.py index 2348779..7f0b289 100644 --- a/mobilizon_reshare/storage/db.py +++ b/mobilizon_reshare/storage/db.py @@ -6,7 +6,7 @@ from pathlib import Path from tortoise import Tortoise from mobilizon_reshare.config.publishers import publisher_names -from mobilizon_reshare.storage.query import update_publishers +from mobilizon_reshare.storage.query.save_query import update_publishers logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class MoReDB: if not self.is_init: await Tortoise.generate_schemas() self.is_init = True - logger.info(f"Succesfully initialized database at {self.path}") + logger.info(f"Successfully initialized database at {self.path}") await update_publishers(publisher_names) diff --git a/mobilizon_reshare/storage/query/__init__.py b/mobilizon_reshare/storage/query/__init__.py new file mode 100644 index 0000000..8261f5b --- /dev/null +++ b/mobilizon_reshare/storage/query/__init__.py @@ -0,0 +1,3 @@ +import sys + +CONNECTION_NAME = "models" if "pytest" in sys.modules else None diff --git a/mobilizon_reshare/storage/query/model_creation.py b/mobilizon_reshare/storage/query/model_creation.py new file mode 100644 index 0000000..7bc16d5 --- /dev/null +++ b/mobilizon_reshare/storage/query/model_creation.py @@ -0,0 +1,14 @@ +from tortoise.transactions import atomic + +from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.models.event import Event +from mobilizon_reshare.models.publication import Publication +from mobilizon_reshare.storage.query import CONNECTION_NAME +from mobilizon_reshare.storage.query.read_query import prefetch_event_relations + + +@atomic(CONNECTION_NAME) +async def create_event_publication_models(event: MobilizonEvent) -> list[Publication]: + return await ( + await prefetch_event_relations(Event.filter(mobilizon_id=event.mobilizon_id)) + )[0].build_unsaved_publication_models() diff --git a/mobilizon_reshare/storage/query.py b/mobilizon_reshare/storage/query/read_query.py similarity index 50% rename from mobilizon_reshare/storage/query.py rename to mobilizon_reshare/storage/query/read_query.py index 4383023..62b89b8 100644 --- a/mobilizon_reshare/storage/query.py +++ b/mobilizon_reshare/storage/query/read_query.py @@ -1,9 +1,6 @@ -import logging -from typing import Iterable, Optional, Union, Dict +from typing import Iterable, Optional, Dict, List from uuid import UUID -import arrow -import sys from arrow import Arrow from tortoise.queryset import QuerySet from tortoise.transactions import atomic @@ -11,62 +8,34 @@ from tortoise.transactions import atomic from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import Publication, PublicationStatus -from mobilizon_reshare.models.publisher import Publisher -from mobilizon_reshare.publishers import get_active_publishers -from mobilizon_reshare.publishers.coordinator import PublisherCoordinatorReport - -logger = logging.getLogger(__name__) - -# This is due to Tortoise community fixtures to -# set up and tear down a DB instance for Pytest. -# See: https://github.com/tortoise/tortoise-orm/issues/419#issuecomment-696991745 -# and: https://docs.pytest.org/en/stable/example/simple.html - -CONNECTION_NAME = "models" if "pytest" in sys.modules else None +from mobilizon_reshare.storage.query import CONNECTION_NAME -async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]: - return ( - await queryset.prefetch_related("publications__publisher") - .order_by("begin_datetime") - .distinct() +async def get_mobilizon_event_publications( + event: MobilizonEvent, +) -> Iterable[Publication]: + models = await prefetch_event_relations( + Event.filter(mobilizon_id=event.mobilizon_id) ) + return models[0].publications -def _add_date_window( - query, - field_name: str, - from_date: Optional[Arrow] = None, - to_date: Optional[Arrow] = None, -): - - if from_date: - query = query.filter(**{f"{field_name}__gt": from_date.to("utc").datetime}) - if to_date: - query = query.filter(**{f"{field_name}__lt": to_date.to("utc").datetime}) - return query - - -@atomic(CONNECTION_NAME) -async def publications_with_status( - status: PublicationStatus, - event_mobilizon_id: Optional[UUID] = None, - from_date: Optional[Arrow] = None, - to_date: Optional[Arrow] = None, -) -> Dict[UUID, Publication]: - query = Publication.filter(status=status) - - if event_mobilizon_id: - query = query.prefetch_related("event").filter( - event__mobilizon_id=event_mobilizon_id - ) - - query = _add_date_window(query, "timestamp", from_date, to_date) - - publications_list = ( - await query.prefetch_related("publisher").order_by("timestamp").distinct() +async def get_published_events( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None +) -> Iterable[MobilizonEvent]: + """ + Retrieves events that are not waiting. Function could be renamed to something more fitting + :return: + """ + return await events_with_status( + [ + EventPublicationStatus.COMPLETED, + EventPublicationStatus.PARTIAL, + EventPublicationStatus.FAILED, + ], + from_date=from_date, + to_date=to_date, ) - return {pub.id: pub for pub in publications_list} async def events_with_status( @@ -96,7 +65,6 @@ async def events_with_status( async def get_all_events( from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, ) -> Iterable[MobilizonEvent]: - return map( MobilizonEvent.from_model, await prefetch_event_relations( @@ -105,105 +73,78 @@ async def get_all_events( ) -async def get_published_events() -> Iterable[MobilizonEvent]: - """ - Retrieves events that are not waiting. Function could be renamed to something more fitting - :return: - """ - return await events_with_status( - [ - EventPublicationStatus.COMPLETED, - EventPublicationStatus.PARTIAL, - EventPublicationStatus.FAILED, - ] +async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]: + return ( + await queryset.prefetch_related("publications__publisher") + .order_by("begin_datetime") + .distinct() ) -async def get_unpublished_events() -> Iterable[MobilizonEvent]: - return await events_with_status([EventPublicationStatus.WAITING]) - - -async def get_mobilizon_event_publications( - event: MobilizonEvent, -) -> Iterable[Publication]: - models = await prefetch_event_relations( - Event.filter(mobilizon_id=event.mobilizon_id) - ) - return models[0].publications - - -async def get_publishers( - name: Optional[str] = None, -) -> Union[Publisher, Iterable[Publisher]]: - if name: - return await Publisher.filter(name=name).first() - else: - return await Publisher.all() - - -async def save_event(event: MobilizonEvent) -> Event: - - event_model = event.to_model() - await event_model.save() - return event_model - - -async def create_publisher(name: str, account_ref: Optional[str] = None) -> None: - await Publisher.create(name=name, account_ref=account_ref) +def _add_date_window( + query, + field_name: str, + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, +): + if from_date: + query = query.filter(**{f"{field_name}__gt": from_date.to("utc").datetime}) + if to_date: + query = query.filter(**{f"{field_name}__lt": to_date.to("utc").datetime}) + return query @atomic(CONNECTION_NAME) -async def update_publishers(names: Iterable[str],) -> None: - names = set(names) - known_publisher_names = set(p.name for p in await get_publishers()) - for name in names.difference(known_publisher_names): - logging.info(f"Creating {name} publisher") - await create_publisher(name) +async def publications_with_status( + status: PublicationStatus, + event_mobilizon_id: Optional[UUID] = None, + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, +) -> Dict[UUID, Publication]: + query = Publication.filter(status=status) - -@atomic(CONNECTION_NAME) -async def save_publication( - publisher_name: str, event_model: Event, status: PublicationStatus -) -> None: - - publisher = await get_publishers(publisher_name) - await Publication.create( - status=status, event_id=event_model.id, publisher_id=publisher.id, - ) - - -@atomic(CONNECTION_NAME) -async def create_unpublished_events( - unpublished_mobilizon_events: Iterable[MobilizonEvent], -) -> None: - # We store only new events, i.e. events whose mobilizon_id wasn't found in the DB. - unpublished_event_models: set[UUID] = set( - map(lambda event: event.mobilizon_id, await get_unpublished_events()) - ) - unpublished_events = list( - filter( - lambda event: event.mobilizon_id not in unpublished_event_models, - unpublished_mobilizon_events, + if event_mobilizon_id: + query = query.prefetch_related("event").filter( + event__mobilizon_id=event_mobilizon_id ) + + query = _add_date_window(query, "timestamp", from_date, to_date) + + publications_list = ( + await query.prefetch_related("publisher").order_by("timestamp").distinct() ) - - for event in unpublished_events: - event_model = await save_event(event) - for publisher in get_active_publishers(): - await save_publication( - publisher, event_model, status=PublicationStatus.WAITING - ) + return {pub.id: pub for pub in publications_list} -@atomic(CONNECTION_NAME) -async def save_publication_report( - coordinator_report: PublisherCoordinatorReport, - publications: Dict[UUID, Publication], -) -> None: - for publication_report in coordinator_report.reports: - publication_id = publication_report.publication.id - publications[publication_id].status = publication_report.status - publications[publication_id].reason = publication_report.reason - publications[publication_id].timestamp = arrow.now().datetime +async def events_without_publications( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, +) -> List[MobilizonEvent]: + query = Event.filter(publications__id=None) + events = await prefetch_event_relations( + _add_date_window(query, "begin_datetime", from_date, to_date) + ) + return list(map(MobilizonEvent.from_model, events)) - await publications[publication_id].save() + +def _remove_duplicated_events(events: List[MobilizonEvent]): + """Remove duplicates based on mobilizon_id""" + result = [] + seen_ids = set() + for event in events: + if event.mobilizon_id not in seen_ids: + result.append(event) + seen_ids.add(event.mobilizon_id) + return result + + +async def get_unpublished_events( + unpublished_mobilizon_events: Iterable[MobilizonEvent], +) -> List[MobilizonEvent]: + """ + Returns all the unpublished events, removing duplicates that are present both in the DB and in the mobilizon query + """ + db_unpublished_events = await events_without_publications() + all_unpublished_events = list(unpublished_mobilizon_events) + list( + db_unpublished_events + ) + return _remove_duplicated_events(all_unpublished_events) diff --git a/mobilizon_reshare/storage/query/save_query.py b/mobilizon_reshare/storage/query/save_query.py new file mode 100644 index 0000000..ebaecd8 --- /dev/null +++ b/mobilizon_reshare/storage/query/save_query.py @@ -0,0 +1,53 @@ +import logging +from typing import List, Iterable, Optional + +import arrow +from tortoise.transactions import atomic + +from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.models.publication import Publication +from mobilizon_reshare.models.publisher import Publisher +from mobilizon_reshare.publishers.coordinator import PublisherCoordinatorReport +from mobilizon_reshare.storage.query import CONNECTION_NAME +from mobilizon_reshare.storage.query.read_query import get_unpublished_events + + +@atomic(CONNECTION_NAME) +async def save_publication_report( + coordinator_report: PublisherCoordinatorReport, + publication_models: List[Publication], +) -> None: + publication_models = {m.id: m for m in publication_models} + for publication_report in coordinator_report.reports: + publication_id = publication_report.publication.id + publication_models[publication_id].status = publication_report.status + publication_models[publication_id].reason = publication_report.reason + publication_models[publication_id].timestamp = arrow.now().datetime + + await publication_models[publication_id].save() + + +@atomic(CONNECTION_NAME) +async def create_unpublished_events( + unpublished_mobilizon_events: Iterable[MobilizonEvent], +) -> List[MobilizonEvent]: + # We store only new events, i.e. events whose mobilizon_id wasn't found in the DB. + + unpublished_events = await get_unpublished_events(unpublished_mobilizon_events) + for event in unpublished_events: + await event.to_model().save() + + return unpublished_events + + +async def create_publisher(name: str, account_ref: Optional[str] = None) -> None: + await Publisher.create(name=name, account_ref=account_ref) + + +@atomic(CONNECTION_NAME) +async def update_publishers(names: Iterable[str],) -> None: + names = set(names) + known_publisher_names = set(p.name for p in await Publisher.all()) + for name in names.difference(known_publisher_names): + logging.info(f"Creating {name} publisher") + await create_publisher(name) diff --git a/tests/commands/test_start.py b/tests/commands/test_start.py new file mode 100644 index 0000000..b349e07 --- /dev/null +++ b/tests/commands/test_start.py @@ -0,0 +1,106 @@ +from logging import DEBUG + +import pytest + +import mobilizon_reshare.publishers.platforms.platform_mapping +from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus +from mobilizon_reshare.main.start import start +from mobilizon_reshare.models import event +from mobilizon_reshare.models.event import Event +from mobilizon_reshare.models.publication import PublicationStatus +from mobilizon_reshare.models.publisher import Publisher + +simple_event_element = { + "beginsOn": "2021-05-23T12:15:00Z", + "description": "Some description", + "endsOn": "2021-05-23T15:15:00Z", + "onlineAddress": None, + "options": {"showEndTime": True, "showStartTime": True}, + "physicalAddress": None, + "picture": None, + "title": "test event", + "url": "https://some_mobilizon/events/1e2e5943-4a5c-497a-b65d-90457b715d7b", + "uuid": "1e2e5943-4a5c-497a-b65d-90457b715d7b", +} +simple_event_response = { + "data": {"group": {"organizedEvents": {"elements": [simple_event_element]}}} +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mobilizon_answer", [{"data": {"group": {"organizedEvents": {"elements": []}}}}], +) +async def test_start_no_event(mock_mobilizon_success_answer, mobilizon_answer, caplog): + + with caplog.at_level(DEBUG): + assert await start() is None + assert "No event to publish found" in caplog.text + + +@pytest.fixture +async def mock_publisher_config( + monkeypatch, mock_publisher_class, mock_formatter_class +): + p = Publisher(name="test") + await p.save() + + def _mock_active_pub(): + return ["test"] + + def _mock_pub_class(name): + return mock_publisher_class + + def _mock_format_class(name): + return mock_formatter_class + + monkeypatch.setattr(event, "get_active_publishers", _mock_active_pub) + monkeypatch.setattr( + mobilizon_reshare.publishers.platforms.platform_mapping, + "get_publisher_class", + _mock_pub_class, + ) + monkeypatch.setattr( + mobilizon_reshare.publishers.platforms.platform_mapping, + "get_formatter_class", + _mock_format_class, + ) + return p + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mobilizon_answer", [simple_event_response], +) +@pytest.mark.parametrize("publication_window", [(0, 24)]) +async def test_start_new_event( + mock_mobilizon_success_answer, + mobilizon_answer, + caplog, + mock_publisher_config, + mock_publication_window, + message_collector, +): + + with caplog.at_level(DEBUG): + assert await start() is None + + assert "Event to publish found" in caplog.text + assert message_collector == ["test event|Some description"] + + all_events = ( + await Event.all() + .prefetch_related("publications") + .prefetch_related("publications__publisher") + ) + + assert len(all_events) == 1, all_events + + publications = all_events[0].publications + assert len(publications) == 1, publications + + assert publications[0].status == PublicationStatus.COMPLETED + assert ( + MobilizonEvent.from_model(all_events[0]).status + == EventPublicationStatus.COMPLETED + ) diff --git a/tests/conftest.py b/tests/conftest.py index cd3257a..df217de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,25 @@ import importlib.resources import os +from collections import UserList from datetime import datetime, timedelta, timezone from uuid import UUID import arrow import pytest +import responses from tortoise.contrib.test import finalizer, initializer import mobilizon_reshare +from mobilizon_reshare.config.config import get_settings from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.notification import Notification, NotificationStatus from mobilizon_reshare.models.publication import Publication, PublicationStatus from mobilizon_reshare.models.publisher import Publisher +from mobilizon_reshare.publishers.abstract import ( + AbstractPlatform, + AbstractEventFormatter, +) def generate_publication_status(published): @@ -145,9 +152,7 @@ def event_model_generator(): @pytest.fixture() def publisher_model_generator(): - def _publisher_model_generator( - idx=1, - ): + def _publisher_model_generator(idx=1,): return Publisher(name=f"publisher_{idx}", account_ref=f"account_ref_{idx}") return _publisher_model_generator @@ -156,7 +161,7 @@ def publisher_model_generator(): @pytest.fixture() def publication_model_generator(): def _publication_model_generator( - status=PublicationStatus.WAITING, + status=PublicationStatus.COMPLETED, publication_time=datetime(year=2021, month=1, day=1, hour=11, minute=30), event_id=None, publisher_id=None, @@ -184,3 +189,86 @@ def notification_model_generator(): ) return _notification_model_generator + + +@pytest.fixture() +def message_collector(): + class MessageCollector(UserList): + def collect_message(self, message): + self.append(message) + + return MessageCollector() + + +@pytest.fixture +def mock_publisher_class(message_collector): + class MockPublisher(AbstractPlatform): + name = "mock" + + def _send(self, message): + message_collector.append(message) + + def _validate_response(self, response): + pass + + def validate_credentials(self) -> None: + pass + + return MockPublisher + + +@pytest.fixture +def mock_publisher_valid(message_collector, mock_publisher_class): + + return mock_publisher_class() + + +@pytest.fixture +def mobilizon_url(): + return get_settings()["source"]["mobilizon"]["url"] + + +@responses.activate +@pytest.fixture +def mock_mobilizon_success_answer(mobilizon_answer, mobilizon_url): + with responses.RequestsMock() as rsps: + + rsps.add( + responses.POST, mobilizon_url, json=mobilizon_answer, status=200, + ) + yield + + +@pytest.fixture +def mock_publication_window(publication_window): + begin, end = publication_window + get_settings().update( + {"publishing.window.begin": begin, "publishing.window.end": end} + ) + + +@pytest.fixture +def mock_formatter_class(): + class MockFormatter(AbstractEventFormatter): + def validate_event(self, event) -> None: + pass + + def get_message_from_event(self, event) -> str: + return f"{event.name}|{event.description}" + + def validate_message(self, event) -> None: + pass + + def get_recap_fragment(self, event): + return event.name + + def get_recap_header(self): + return "Upcoming" + + return MockFormatter + + +@pytest.fixture +def mock_formatter_valid(mock_formatter_class): + + return mock_formatter_class() diff --git a/tests/event/test_strategies.py b/tests/event/test_strategies.py index e5b1c88..e922e38 100644 --- a/tests/event/test_strategies.py +++ b/tests/event/test_strategies.py @@ -26,14 +26,6 @@ def set_strategy(strategy_name): get_settings().update({"selection.strategy": strategy_name}) -@pytest.fixture -def mock_publication_window(publication_window): - begin, end = publication_window - get_settings().update( - {"publishing.window.begin": begin, "publishing.window.end": end} - ) - - @pytest.mark.parametrize("current_hour", [15]) def test_window_no_event(mock_arrow_now): selected_event = SelectNextEventStrategy().select([], []) @@ -109,9 +101,7 @@ def test_window_simple_event_found( @pytest.mark.parametrize("current_hour", [15]) @pytest.mark.parametrize("strategy_name", ["next_event"]) def test_window_simple_no_published_events( - event_generator, - set_strategy, - mock_arrow_now, + event_generator, set_strategy, mock_arrow_now, ): "Testing that if no event is published, the function takes the first available unpublished event" unpublished_events = [ @@ -132,9 +122,7 @@ def test_window_simple_no_published_events( @pytest.mark.parametrize("current_hour", [15]) @pytest.mark.parametrize("strategy_name", ["next_event"]) def test_window_simple_event_too_recent( - event_generator, - set_strategy, - mock_arrow_now, + event_generator, set_strategy, mock_arrow_now, ): "Testing that if an event has been published too recently, no event is selected for publication" unpublished_events = [ diff --git a/tests/mobilizon/conftest.py b/tests/mobilizon/conftest.py index 7a245c6..2b8dd82 100644 --- a/tests/mobilizon/conftest.py +++ b/tests/mobilizon/conftest.py @@ -1,27 +1,6 @@ import pytest import responses -from mobilizon_reshare.config.config import get_settings - - -@pytest.fixture -def mobilizon_url(): - return get_settings()["source"]["mobilizon"]["url"] - - -@responses.activate -@pytest.fixture -def mock_mobilizon_success_answer(mobilizon_answer, mobilizon_url): - with responses.RequestsMock() as rsps: - - rsps.add( - responses.POST, - mobilizon_url, - json=mobilizon_answer, - status=200, - ) - yield - @responses.activate @pytest.fixture @@ -29,8 +8,6 @@ def mock_mobilizon_failure_answer(mobilizon_url): with responses.RequestsMock() as rsps: rsps.add( - responses.POST, - mobilizon_url, - status=500, + responses.POST, mobilizon_url, status=500, ) yield diff --git a/tests/models/test_event.py b/tests/models/test_event.py index 7df74ca..7b956e2 100644 --- a/tests/models/test_event.py +++ b/tests/models/test_event.py @@ -5,10 +5,10 @@ import arrow import pytest import tortoise.timezone +from mobilizon_reshare.event.event import EventPublicationStatus from mobilizon_reshare.event.event import MobilizonEvent from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.event.event import EventPublicationStatus @pytest.mark.asyncio @@ -119,7 +119,7 @@ async def test_mobilizon_event_from_model( await publisher_model_2.save() publication = publication_model_generator( - status=PublicationStatus.WAITING, + status=PublicationStatus.FAILED, event_id=event_model.id, publisher_id=publisher_model.id, ) @@ -153,94 +153,43 @@ async def test_mobilizon_event_from_model( assert event.status == EventPublicationStatus.PARTIAL -@pytest.mark.asyncio -async def test_mobilizon_event_compute_status_failed( - event_model_generator, publication_model_generator, publisher_model_generator -): - event_model = event_model_generator() - await event_model.save() - - publisher_model = publisher_model_generator() - await publisher_model.save() - publisher_model_2 = publisher_model_generator(idx=2) - await publisher_model_2.save() - - publication = publication_model_generator( - status=PublicationStatus.FAILED, - event_id=event_model.id, - publisher_id=publisher_model.id, - ) - await publication.save() - publication_2 = publication_model_generator( - status=PublicationStatus.COMPLETED, - event_id=event_model.id, - publisher_id=publisher_model_2.id, - ) - await publication_2.save() - - assert ( - MobilizonEvent.compute_status([publication, publication_2]) - == PublicationStatus.FAILED - ) - - +@pytest.mark.parametrize( + "statuses, expected_result", + [ + ( + [PublicationStatus.FAILED, PublicationStatus.COMPLETED], + EventPublicationStatus.PARTIAL, + ), + ( + [PublicationStatus.FAILED, PublicationStatus.FAILED], + EventPublicationStatus.FAILED, + ), + ( + [PublicationStatus.COMPLETED, PublicationStatus.COMPLETED], + EventPublicationStatus.COMPLETED, + ), + ([PublicationStatus.FAILED], EventPublicationStatus.FAILED), + ([], EventPublicationStatus.WAITING), + ], +) @pytest.mark.asyncio async def test_mobilizon_event_compute_status_partial( - event_model_generator, publication_model_generator, publisher_model_generator + event_model_generator, + publication_model_generator, + publisher_model_generator, + statuses, + expected_result, ): event_model = event_model_generator() await event_model.save() + publications = [] + for status in statuses: + publisher_model = publisher_model_generator() + await publisher_model.save() - publisher_model = publisher_model_generator() - await publisher_model.save() - publisher_model_2 = publisher_model_generator(idx=2) - await publisher_model_2.save() - - publication = publication_model_generator( - status=PublicationStatus.WAITING, - event_id=event_model.id, - publisher_id=publisher_model.id, - ) - await publication.save() - publication_2 = publication_model_generator( - status=PublicationStatus.COMPLETED, - event_id=event_model.id, - publisher_id=publisher_model_2.id, - ) - await publication_2.save() - - assert ( - MobilizonEvent.compute_status([publication, publication_2]) - == EventPublicationStatus.PARTIAL - ) - - -@pytest.mark.asyncio -async def test_mobilizon_event_compute_status_waiting( - event_model_generator, publication_model_generator, publisher_model_generator -): - event_model = event_model_generator() - await event_model.save() - - publisher_model = publisher_model_generator() - await publisher_model.save() - publisher_model_2 = publisher_model_generator(idx=2) - await publisher_model_2.save() - - publication = publication_model_generator( - status=PublicationStatus.WAITING, - event_id=event_model.id, - publisher_id=publisher_model.id, - ) - await publication.save() - publication_2 = publication_model_generator( - status=PublicationStatus.WAITING, - event_id=event_model.id, - publisher_id=publisher_model_2.id, - ) - await publication_2.save() - - assert ( - MobilizonEvent.compute_status([publication, publication_2]) - == EventPublicationStatus.WAITING - ) + publication = publication_model_generator( + status=status, event_id=event_model.id, publisher_id=publisher_model.id, + ) + await publication.save() + publications.append(publication) + assert MobilizonEvent.compute_status(publications) == expected_result diff --git a/tests/models/test_publication.py b/tests/models/test_publication.py deleted file mode 100644 index da1ab3a..0000000 --- a/tests/models/test_publication.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from datetime import datetime -from tortoise import timezone - -from mobilizon_reshare.models.publication import Publication, PublicationStatus - - -@pytest.mark.asyncio -async def test_publication_create( - publication_model_generator, event_model_generator, publisher_model_generator -): - event = event_model_generator() - await event.save() - publisher = publisher_model_generator() - await publisher.save() - publication_model = await publication_model_generator( - event_id=event.id, publisher_id=publisher.id - ) - await publication_model.save() - publication_db = await Publication.all().first() - assert publication_db.status == PublicationStatus.WAITING - assert publication_db.timestamp == datetime( - year=2021, - month=1, - day=1, - hour=11, - minute=30, - tzinfo=timezone.get_default_timezone(), - ) diff --git a/tests/publishers/conftest.py b/tests/publishers/conftest.py index 70ce70a..7b7dd1d 100644 --- a/tests/publishers/conftest.py +++ b/tests/publishers/conftest.py @@ -1,4 +1,3 @@ -from collections import UserList from datetime import timedelta from uuid import UUID @@ -28,39 +27,6 @@ def test_event(): ) -@pytest.fixture -def mock_formatter_valid(): - class MockFormatter(AbstractEventFormatter): - def validate_event(self, event) -> None: - pass - - def get_message_from_event(self, event) -> str: - return event.description - - def validate_message(self, event) -> None: - pass - - def _send(self, message): - pass - - def get_recap_fragment(self, event): - return event.name - - def get_recap_header(self): - return "Upcoming" - - return MockFormatter() - - -@pytest.fixture() -def message_collector(): - class MessageCollector(UserList): - def collect_message(self, message): - self.append(message) - - return MessageCollector() - - @pytest.fixture def mock_formatter_invalid(): class MockFormatter(AbstractEventFormatter): @@ -76,23 +42,6 @@ def mock_formatter_invalid(): return MockFormatter() -@pytest.fixture -def mock_publisher_valid(message_collector): - class MockPublisher(AbstractPlatform): - name = "mock" - - def _send(self, message): - message_collector.append(message) - - def _validate_response(self, response): - pass - - def validate_credentials(self) -> None: - pass - - return MockPublisher() - - @pytest.fixture def mock_publisher_invalid(message_collector): class MockPublisher(AbstractPlatform): diff --git a/tests/publishers/test_coordinator.py b/tests/publishers/test_coordinator.py index 19f7145..d0f06d4 100644 --- a/tests/publishers/test_coordinator.py +++ b/tests/publishers/test_coordinator.py @@ -35,7 +35,6 @@ def failure_report(mock_publisher_invalid): "statuses, successful", [ [[PublicationStatus.COMPLETED, PublicationStatus.COMPLETED], True], - [[PublicationStatus.WAITING, PublicationStatus.COMPLETED], False], [[PublicationStatus.COMPLETED, PublicationStatus.FAILED], False], [[], True], [[PublicationStatus.COMPLETED], True], @@ -90,7 +89,7 @@ async def mock_publications( await publisher.save() publication = PublicationModel( id=UUID(int=i + 1), - status=PublicationStatus.WAITING, + status=PublicationStatus.UNSAVED, event=event, publisher=publisher, timestamp=None, diff --git a/tests/publishers/test_zulip.py b/tests/publishers/test_zulip.py index a92a920..0656a55 100644 --- a/tests/publishers/test_zulip.py +++ b/tests/publishers/test_zulip.py @@ -6,6 +6,7 @@ import responses from mobilizon_reshare.config.config import get_settings from mobilizon_reshare.models.publication import PublicationStatus +from mobilizon_reshare.models.publisher import Publisher from mobilizon_reshare.publishers import get_active_publishers from mobilizon_reshare.publishers.abstract import EventPublication from mobilizon_reshare.publishers.coordinator import PublisherCoordinator @@ -16,10 +17,9 @@ from mobilizon_reshare.publishers.exceptions import ( InvalidMessage, ) from mobilizon_reshare.publishers.platforms.zulip import ZulipFormatter, ZulipPublisher -from mobilizon_reshare.storage.query import ( - get_publishers, - update_publishers, - publications_with_status, +from mobilizon_reshare.storage.query.save_query import update_publishers +from mobilizon_reshare.storage.query.model_creation import ( + create_event_publication_models, ) api_uri = "https://zulip.twc-italia.org/api/v1/" @@ -98,7 +98,7 @@ async def setup_db(event_model_generator, publication_model_generator): ] = "giacomotest2-bot@zulip.twc-italia.org" await update_publishers(["zulip"]) - publisher = await get_publishers(name="zulip") + publisher = await Publisher.filter(name="zulip").first() event = event_model_generator() await event.save() publication = publication_model_generator( @@ -107,18 +107,21 @@ async def setup_db(event_model_generator, publication_model_generator): await publication.save() +@pytest.fixture @pytest.mark.asyncio -async def test_zulip_publisher(mocked_valid_response, setup_db, event): - publication_models = await publications_with_status( - status=PublicationStatus.WAITING - ) +async def publication_models(event): + await event.to_model().save() + publication_models = await create_event_publication_models(event) + return publication_models + + +@pytest.mark.asyncio +async def test_zulip_publisher( + mocked_valid_response, setup_db, event, publication_models +): + report = PublisherCoordinator( - list( - map( - partial(EventPublication.from_orm, event=event), - publication_models.values(), - ) - ) + list(map(partial(EventPublication.from_orm, event=event), publication_models,)) ).run() assert report.reports[0].status == PublicationStatus.COMPLETED @@ -126,18 +129,10 @@ async def test_zulip_publisher(mocked_valid_response, setup_db, event): @pytest.mark.asyncio async def test_zulip_publishr_failure_invalid_credentials( - mocked_credential_error_response, setup_db, event + mocked_credential_error_response, setup_db, event, publication_models ): - publication_models = await publications_with_status( - status=PublicationStatus.WAITING - ) report = PublisherCoordinator( - list( - map( - partial(EventPublication.from_orm, event=event), - publication_models.values(), - ) - ) + list(map(partial(EventPublication.from_orm, event=event), publication_models)) ).run() assert report.reports[0].status == PublicationStatus.FAILED assert report.reports[0].reason == "403 Error - Your credentials are not valid!" @@ -145,18 +140,10 @@ async def test_zulip_publishr_failure_invalid_credentials( @pytest.mark.asyncio async def test_zulip_publisher_failure_client_error( - mocked_client_error_response, setup_db, event + mocked_client_error_response, setup_db, event, publication_models ): - publication_models = await publications_with_status( - status=PublicationStatus.WAITING - ) report = PublisherCoordinator( - list( - map( - partial(EventPublication.from_orm, event=event), - publication_models.values(), - ) - ) + list(map(partial(EventPublication.from_orm, event=event), publication_models)) ).run() assert report.reports[0].status == PublicationStatus.FAILED assert report.reports[0].reason == "400 Error - Invalid request" diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index ceea1fe..eebaab4 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -5,12 +5,7 @@ from mobilizon_reshare.models.publication import Publication from mobilizon_reshare.models.publication import PublicationStatus today = datetime( - year=2021, - month=6, - day=6, - hour=5, - minute=0, - tzinfo=timezone(timedelta(hours=2)), + year=2021, month=6, day=6, hour=5, minute=0, tzinfo=timezone(timedelta(hours=2)), ) @@ -20,15 +15,9 @@ complete_specification = { {"event_idx": 0, "publisher_idx": 0, "status": PublicationStatus.COMPLETED}, {"event_idx": 0, "publisher_idx": 1, "status": PublicationStatus.COMPLETED}, {"event_idx": 0, "publisher_idx": 2, "status": PublicationStatus.COMPLETED}, - {"event_idx": 1, "publisher_idx": 0, "status": PublicationStatus.WAITING}, - {"event_idx": 1, "publisher_idx": 1, "status": PublicationStatus.WAITING}, + {"event_idx": 1, "publisher_idx": 0, "status": PublicationStatus.FAILED}, {"event_idx": 1, "publisher_idx": 2, "status": PublicationStatus.COMPLETED}, - {"event_idx": 2, "publisher_idx": 0, "status": PublicationStatus.FAILED}, {"event_idx": 2, "publisher_idx": 1, "status": PublicationStatus.COMPLETED}, - {"event_idx": 2, "publisher_idx": 2, "status": PublicationStatus.WAITING}, - {"event_idx": 3, "publisher_idx": 0, "status": PublicationStatus.WAITING}, - {"event_idx": 3, "publisher_idx": 1, "status": PublicationStatus.WAITING}, - {"event_idx": 3, "publisher_idx": 2, "status": PublicationStatus.WAITING}, ], "publisher": ["telegram", "twitter", "mastodon", "zulip"], } diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 42c6d2e..84e5afc 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -46,9 +46,8 @@ async def _generate_events(specification): async def _generate_publications(events, publishers, specification): if "publications" in specification.keys(): - for i in range(len(specification["publications"])): - publication = specification["publications"][i] - status = publication.get("status", PublicationStatus.WAITING) + for i, publication in enumerate(specification["publications"]): + status = publication.get("status", PublicationStatus.COMPLETED) timestamp = publication.get("timestamp", today + timedelta(hours=i)) await Publication.create( id=UUID(int=i), diff --git a/tests/storage/test_query.py b/tests/storage/test_query.py index 8844b35..4813a01 100644 --- a/tests/storage/test_query.py +++ b/tests/storage/test_query.py @@ -7,15 +7,13 @@ import pytest from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.storage.query import events_with_status -from mobilizon_reshare.storage.query import ( - get_published_events, - get_unpublished_events, - create_unpublished_events, +from mobilizon_reshare.storage.query.read_query import ( get_mobilizon_event_publications, + get_published_events, + events_with_status, prefetch_event_relations, - get_publishers, publications_with_status, + events_without_publications, ) from tests.storage import complete_specification from tests.storage import result_publication @@ -24,16 +22,12 @@ from tests.storage import today event_0 = MobilizonEvent( name="event_0", description="desc_0", - mobilizon_id="mobid_0", + mobilizon_id=UUID(int=0), mobilizon_link="moblink_0", thumbnail_link="thumblink_0", location="loc_0", - publication_time={ - "telegram": arrow.get(today + timedelta(hours=0)), - "twitter": arrow.get(today + timedelta(hours=1)), - "mastodon": arrow.get(today + timedelta(hours=2)), - }, - status=EventPublicationStatus.COMPLETED, + publication_time={}, + status=EventPublicationStatus.WAITING, begin_datetime=arrow.get(today + timedelta(days=0)), end_datetime=arrow.get(today + timedelta(days=0) + timedelta(hours=2)), ) @@ -47,100 +41,6 @@ async def test_get_published_events(generate_models): assert len(published_events) == 3 -@pytest.mark.asyncio -@pytest.mark.parametrize( - "specification,expected_result", - [ - [ - complete_specification, - [ - MobilizonEvent( - name="event_3", - description="desc_3", - mobilizon_id=UUID(int=3), - mobilizon_link="moblink_3", - thumbnail_link="thumblink_3", - location="loc_3", - status=EventPublicationStatus.WAITING, - begin_datetime=arrow.get(today + timedelta(days=3)), - end_datetime=arrow.get( - today + timedelta(days=3) + timedelta(hours=2) - ), - ), - ], - ] - ], -) -async def test_get_unpublished_events(specification, expected_result, generate_models): - await generate_models(specification) - unpublished_events = list(await get_unpublished_events()) - - assert len(unpublished_events) == len(expected_result) - assert unpublished_events == expected_result - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "expected_result", - [ - [ - [ - Event( - name="event_1", - description="desc_1", - mobilizon_id=UUID(int=101112), - mobilizon_link="moblink_1", - thumbnail_link="thumblink_1", - location="loc_1", - begin_datetime=today + timedelta(days=1), - end_datetime=today + timedelta(days=1) + timedelta(hours=2), - ), - Event( - name="test event", - description="description of the event", - mobilizon_id=UUID(int=12345), - mobilizon_link="http://some_link.com/123", - thumbnail_link="http://some_link.com/123.jpg", - location="location", - begin_datetime=today + timedelta(days=6), - end_datetime=today + timedelta(days=6) + timedelta(hours=2), - ), - Event( - name="test event", - description="description of the event", - mobilizon_id=UUID(int=67890), - mobilizon_link="http://some_link.com/123", - thumbnail_link="http://some_link.com/123.jpg", - location="location", - begin_datetime=today + timedelta(days=12), - end_datetime=today + timedelta(days=12) + timedelta(hours=2), - ), - ], - ] - ], -) -async def test_create_unpublished_events( - expected_result, - generate_models, - event_generator, -): - await generate_models(complete_specification) - event_3 = event_generator(begin_date=arrow.get(today + timedelta(days=6))) - event_4 = event_generator( - begin_date=arrow.get(today + timedelta(days=12)), mobilizon_id=UUID(int=67890) - ) - models = await prefetch_event_relations(Event.filter(name="event_1")) - - events_from_internet = [MobilizonEvent.from_model(models[0]), event_3, event_4] - - await create_unpublished_events( - unpublished_mobilizon_events=events_from_internet, - ) - unpublished_events = list(await get_unpublished_events()) - - assert len(unpublished_events) == 4 - - @pytest.mark.asyncio async def test_get_mobilizon_event_publications(generate_models): await generate_models(complete_specification) @@ -168,66 +68,16 @@ async def test_get_mobilizon_event_publications(generate_models): assert publications[2].status == PublicationStatus.COMPLETED -@pytest.mark.asyncio -@pytest.mark.parametrize( - "name,expected_result", - [[None, {"telegram", "twitter", "mastodon", "zulip"}], ["telegram", {"telegram"}]], -) -async def test_get_publishers( - name, - expected_result, - generate_models, -): - await generate_models(complete_specification) - result = await get_publishers(name) - - if type(result) == list: - publishers = set(p.name for p in result) - else: - publishers = {result.name} - - assert len(publishers) == len(expected_result) - assert publishers == expected_result - - @pytest.mark.asyncio @pytest.mark.parametrize( "status,mobilizon_id,from_date,to_date,expected_result", [ - [ - PublicationStatus.WAITING, - None, - None, - None, - [ - result_publication[3], - result_publication[4], - result_publication[8], - result_publication[9], - result_publication[10], - result_publication[11], - ], - ], - [ - PublicationStatus.WAITING, - UUID(int=1), - None, - None, - [result_publication[3], result_publication[4]], - ], - [ - PublicationStatus.WAITING, - None, - arrow.get(today), - arrow.get(today + timedelta(hours=6)), - [result_publication[3], result_publication[4]], - ], [ PublicationStatus.COMPLETED, None, arrow.get(today + timedelta(hours=1)), None, - [result_publication[2], result_publication[5], result_publication[7]], + [result_publication[2], result_publication[4], result_publication[5]], ], [ PublicationStatus.COMPLETED, @@ -236,15 +86,17 @@ async def test_get_publishers( arrow.get(today + timedelta(hours=2)), [result_publication[0], result_publication[1]], ], + [ + PublicationStatus.FAILED, + None, + None, + arrow.get(today + timedelta(hours=5)), + [result_publication[3]], + ], ], ) async def test_publications_with_status( - status, - mobilizon_id, - from_date, - to_date, - expected_result, - generate_models, + status, mobilizon_id, from_date, to_date, expected_result, generate_models, ): await generate_models(complete_specification) publications = await publications_with_status( @@ -260,12 +112,7 @@ async def test_publications_with_status( @pytest.mark.asyncio @pytest.mark.parametrize( "status, expected_events_count", - [ - (EventPublicationStatus.COMPLETED, 1), - (EventPublicationStatus.FAILED, 1), - (EventPublicationStatus.PARTIAL, 1), - (EventPublicationStatus.WAITING, 1), - ], + [(EventPublicationStatus.COMPLETED, 2), (EventPublicationStatus.PARTIAL, 1)], ) async def test_event_with_status(generate_models, status, expected_events_count): await generate_models(complete_specification) @@ -280,13 +127,13 @@ async def test_event_with_status(generate_models, status, expected_events_count) [ ( EventPublicationStatus.COMPLETED, - 1, + 2, arrow.get(today + timedelta(hours=-1)), None, ), ( EventPublicationStatus.COMPLETED, - 0, + 1, arrow.get(today + timedelta(hours=1)), None, ), @@ -313,3 +160,55 @@ async def test_event_with_status_window( ) assert len(result) == expected_events_count + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "spec, expected_events", + [ + ( + {"event": 2, "publications": [], "publisher": ["zulip"]}, + [ + event_0, + MobilizonEvent( + name="event_1", + description="desc_1", + mobilizon_id=UUID(int=1), + mobilizon_link="moblink_1", + thumbnail_link="thumblink_1", + location="loc_1", + status=EventPublicationStatus.WAITING, + publication_time={}, + begin_datetime=arrow.get(today + timedelta(days=1)), + end_datetime=arrow.get( + today + timedelta(days=1) + timedelta(hours=2) + ), + ), + ], + ), + ( + complete_specification, + [ + MobilizonEvent( + name="event_3", + description="desc_3", + mobilizon_id=UUID(int=3), + mobilizon_link="moblink_3", + thumbnail_link="thumblink_3", + location="loc_3", + status=EventPublicationStatus.WAITING, + publication_time={}, + begin_datetime=arrow.get(today + timedelta(days=3)), + end_datetime=arrow.get( + today + timedelta(days=3) + timedelta(hours=2) + ), + ), + ], + ), + ], +) +async def test_events_without_publications(spec, expected_events, generate_models): + await generate_models(spec) + unpublished_events = list(await events_without_publications()) + assert len(unpublished_events) == len(expected_events) + assert unpublished_events == expected_events diff --git a/tests/storage/test_read_query.py b/tests/storage/test_read_query.py new file mode 100644 index 0000000..7d785b5 --- /dev/null +++ b/tests/storage/test_read_query.py @@ -0,0 +1,92 @@ +from uuid import UUID + +import pytest + +from mobilizon_reshare.storage.query.read_query import ( + get_unpublished_events, + get_all_events, +) + + +@pytest.mark.parametrize( + "spec, expected_output_len", + [ + [{"event": 2, "publisher": [], "publications": []}, 2], + [{"event": 0, "publisher": [], "publications": []}, 0], + [ + { + "event": 2, + "publisher": ["zulip"], + "publications": [{"event_idx": 0, "publisher_idx": 0}], + }, + 1, + ], + ], +) +@pytest.mark.asyncio +async def test_get_unpublished_events_db_only( + spec, generate_models, expected_output_len, event_generator +): + """Testing that with no events on Mobilizon, I retrieve all the DB unpublished events """ + await generate_models(spec) + unpublished_events = await get_unpublished_events([]) + assert len(unpublished_events) == expected_output_len + + +@pytest.mark.parametrize("num_mobilizon_events", [0, 2]) +@pytest.mark.asyncio +async def test_get_unpublished_events_mobilizon_only_no_publications( + event_generator, num_mobilizon_events +): + """Testing that when there are no events present in the DB, all the mobilizon events are returned""" + mobilizon_events = [ + event_generator(mobilizon_id=UUID(int=i), published=False) + for i in range(num_mobilizon_events) + ] + unpublished_events = await get_unpublished_events(mobilizon_events) + assert unpublished_events == mobilizon_events + + +@pytest.mark.asyncio +async def test_get_unpublished_events_no_overlap(event_generator): + "Testing that all the events are returned when there's no overlap" + all_events = [ + event_generator(mobilizon_id=UUID(int=i), published=False) for i in range(4) + ] + db_events = all_events[:1] + mobilizon_events = all_events[1:] + for e in db_events: + await e.to_model().save() + + unpublished_events = await get_unpublished_events(mobilizon_events) + assert sorted(all_events, key=lambda x: x.mobilizon_id) == sorted( + unpublished_events, key=lambda x: x.mobilizon_id + ) + + +@pytest.mark.asyncio +async def test_get_unpublished_events_overlap(event_generator): + """Testing that there are no duplicates when an event from mobilizon is already present in the db + and that no event is lost""" + + all_events = [ + event_generator(mobilizon_id=UUID(int=i), published=False) for i in range(4) + ] + db_events = all_events[:2] + mobilizon_events = all_events[1:] + for e in db_events: + await e.to_model().save() + + unpublished_events = await get_unpublished_events(mobilizon_events) + assert len(unpublished_events) == 4 + + +@pytest.mark.asyncio +async def test_get_all_events(event_generator): + all_events = [ + event_generator(mobilizon_id=UUID(int=i), published=False) for i in range(4) + ] + for e in all_events: + await e.to_model().save() + + assert list(await get_all_events()) == all_events diff --git a/tests/storage/test_update.py b/tests/storage/test_update.py index be6c856..d10ec56 100644 --- a/tests/storage/test_update.py +++ b/tests/storage/test_update.py @@ -12,12 +12,11 @@ from mobilizon_reshare.publishers.coordinator import ( PublisherCoordinatorReport, EventPublicationReport, ) -from mobilizon_reshare.storage.query import ( - get_publishers, - update_publishers, +from mobilizon_reshare.storage.query.read_query import publications_with_status +from mobilizon_reshare.storage.query.save_query import ( save_publication_report, + update_publishers, ) -from mobilizon_reshare.storage.query import publications_with_status from tests.storage import complete_specification from tests.storage import today @@ -54,9 +53,9 @@ async def test_update_publishers( await generate_models(specification) await update_publishers(names) if type(list(expected_result)[0]) == Publisher: - publishers = set(await get_publishers()) + publishers = set(await Publisher.all()) else: - publishers = set(p.name for p in await get_publishers()) + publishers = set(p.name for p in await Publisher.all()) assert len(publishers) == len(expected_result) assert publishers == expected_result @@ -71,13 +70,6 @@ async def test_update_publishers( PublisherCoordinatorReport( publications=[], reports=[ - EventPublicationReport( - status=PublicationStatus.FAILED, - reason="Invalid credentials", - publication=EventPublication( - id=UUID(int=3), formatter=None, event=None, publisher=None - ), - ), EventPublicationReport( status=PublicationStatus.COMPLETED, reason="", @@ -99,11 +91,6 @@ async def test_update_publishers( end_datetime=arrow.get(today + timedelta(days=1) + timedelta(hours=2)), ), { - UUID(int=3): Publication( - id=UUID(int=3), - status=PublicationStatus.FAILED, - reason="Invalid credentials", - ), UUID(int=4): Publication( id=UUID(int=4), status=PublicationStatus.COMPLETED, reason="" ), @@ -117,9 +104,9 @@ async def test_save_publication_report( await generate_models(specification) publications = await publications_with_status( - status=PublicationStatus.WAITING, event_mobilizon_id=event.mobilizon_id, + status=PublicationStatus.COMPLETED, event_mobilizon_id=event.mobilizon_id, ) - await save_publication_report(report, publications) + await save_publication_report(report, list(publications.values())) publication_ids = set(publications.keys()) publications = { p_id: await Publication.filter(id=p_id).first() for p_id in publication_ids