From 370e00d18715a66318e76d8501771c74918002dc Mon Sep 17 00:00:00 2001 From: Simone Robutti Date: Sun, 11 Dec 2022 14:15:04 +0100 Subject: [PATCH] decouple dataclasses from models (#181) * fixed parsing bug * implemented events and publications endpoints split endpoints by entity removed credentials * add pagination (#179) * added pagination * integrated pagination with tortoise * added test for publications * removed converter file * moved publications to dataclasses module * implemented import pattern on dataclasses to prevent circular imports * removed redundant fetch * removed unused query * split build_publications * split failed_publications * removed redundant query functions * split publication retrieve * split all read functions * removed redundant write function * fixed lock --- mobilizon_reshare/cli/cli.py | 10 +- .../cli/commands/format/format.py | 2 +- .../cli/commands/list/list_event.py | 34 ++-- mobilizon_reshare/dataclasses/__init__.py | 9 + .../{event => dataclasses}/event.py | 106 +++++++---- .../dataclasses/event_publication_status.py | 27 +++ mobilizon_reshare/dataclasses/publication.py | 72 +++++++ .../event/event_selection_strategies.py | 2 +- mobilizon_reshare/main/publish.py | 30 +-- mobilizon_reshare/main/pull.py | 2 +- mobilizon_reshare/main/recap.py | 17 +- mobilizon_reshare/main/retry.py | 17 +- mobilizon_reshare/mobilizon/events.py | 4 +- mobilizon_reshare/models/event.py | 17 ++ mobilizon_reshare/publishers/abstract.py | 48 +---- .../coordinators/event_publishing/__init__.py | 7 +- .../coordinators/event_publishing/publish.py | 7 +- .../coordinators/recap_publishing/recap.py | 2 +- .../publishers/platforms/facebook.py | 2 +- .../publishers/platforms/mastodon.py | 2 +- .../publishers/platforms/telegram.py | 2 +- .../publishers/platforms/twitter.py | 2 +- .../publishers/platforms/zulip.py | 2 +- mobilizon_reshare/storage/query/read.py | 147 ++------------- mobilizon_reshare/storage/query/write.py | 37 ++-- poetry.lock | 76 ++++---- tests/commands/test_list.py | 6 +- tests/commands/test_publish.py | 3 +- tests/commands/test_pull.py | 25 ++- tests/commands/test_start.py | 7 +- tests/conftest.py | 2 +- tests/formatting/test_output_format.py | 12 +- tests/mobilizon/test_events.py | 2 +- tests/models/test_event.py | 6 +- tests/publishers/conftest.py | 2 +- tests/publishers/test_coordinator.py | 13 +- tests/publishers/test_zulip.py | 5 +- tests/storage/test_query.py | 178 +++++++----------- tests/storage/test_read_query.py | 2 +- tests/storage/test_update.py | 6 +- 40 files changed, 466 insertions(+), 486 deletions(-) create mode 100644 mobilizon_reshare/dataclasses/__init__.py rename mobilizon_reshare/{event => dataclasses}/event.py (52%) create mode 100644 mobilizon_reshare/dataclasses/event_publication_status.py create mode 100644 mobilizon_reshare/dataclasses/publication.py diff --git a/mobilizon_reshare/cli/cli.py b/mobilizon_reshare/cli/cli.py index 8777ca0..0aff8b6 100644 --- a/mobilizon_reshare/cli/cli.py +++ b/mobilizon_reshare/cli/cli.py @@ -19,7 +19,7 @@ from mobilizon_reshare.cli.commands.start.main import start_command as start_mai from mobilizon_reshare.config.command import CommandConfig from mobilizon_reshare.config.config import current_version, get_settings from mobilizon_reshare.config.publishers import publisher_names -from mobilizon_reshare.event.event import EventPublicationStatus +from mobilizon_reshare.dataclasses.event import _EventPublicationStatus from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.publishers import get_active_publishers @@ -49,10 +49,10 @@ def print_platforms(ctx, param, value): status_name_to_enum = { "event": { - "waiting": EventPublicationStatus.WAITING, - "completed": EventPublicationStatus.COMPLETED, - "failed": EventPublicationStatus.FAILED, - "partial": EventPublicationStatus.PARTIAL, + "waiting": _EventPublicationStatus.WAITING, + "completed": _EventPublicationStatus.COMPLETED, + "failed": _EventPublicationStatus.FAILED, + "partial": _EventPublicationStatus.PARTIAL, "all": None, }, "publication": { diff --git a/mobilizon_reshare/cli/commands/format/format.py b/mobilizon_reshare/cli/commands/format/format.py index 8d69bbe..e45c4dc 100644 --- a/mobilizon_reshare/cli/commands/format/format.py +++ b/mobilizon_reshare/cli/commands/format/format.py @@ -1,6 +1,6 @@ import click -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.models.event import Event from mobilizon_reshare.publishers.platforms.platform_mapping import get_formatter_class diff --git a/mobilizon_reshare/cli/commands/list/list_event.py b/mobilizon_reshare/cli/commands/list/list_event.py index 3c1df56..84d1551 100644 --- a/mobilizon_reshare/cli/commands/list/list_event.py +++ b/mobilizon_reshare/cli/commands/list/list_event.py @@ -4,21 +4,21 @@ from typing import Iterable, Optional import click from arrow import Arrow -from mobilizon_reshare.event.event import EventPublicationStatus -from mobilizon_reshare.event.event import MobilizonEvent -from mobilizon_reshare.event.event_selection_strategies import select_unpublished_events -from mobilizon_reshare.storage.query.read import ( - get_published_events, - events_with_status, +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.event import ( + _EventPublicationStatus, get_all_mobilizon_events, - events_without_publications, + get_published_events, + get_mobilizon_events_with_status, + get_mobilizon_events_without_publications, ) +from mobilizon_reshare.event.event_selection_strategies import select_unpublished_events status_to_color = { - EventPublicationStatus.COMPLETED: "green", - EventPublicationStatus.FAILED: "red", - EventPublicationStatus.PARTIAL: "yellow", - EventPublicationStatus.WAITING: "white", + _EventPublicationStatus.COMPLETED: "green", + _EventPublicationStatus.FAILED: "red", + _EventPublicationStatus.PARTIAL: "yellow", + _EventPublicationStatus.WAITING: "white", } @@ -38,12 +38,14 @@ def pretty(event: MobilizonEvent): async def list_unpublished_events(frm: Arrow = None, to: Arrow = None): return select_unpublished_events( list(await get_published_events(from_date=frm, to_date=to)), - list(await events_without_publications(from_date=frm, to_date=to)), + list( + await get_mobilizon_events_without_publications(from_date=frm, to_date=to) + ), ) async def list_events( - status: Optional[EventPublicationStatus] = None, + status: Optional[_EventPublicationStatus] = None, frm: Optional[datetime] = None, to: Optional[datetime] = None, ): @@ -52,10 +54,12 @@ async def list_events( to = Arrow.fromdatetime(to) if to else None if status is None: events = await get_all_mobilizon_events(from_date=frm, to_date=to) - elif status == EventPublicationStatus.WAITING: + elif status == _EventPublicationStatus.WAITING: events = await list_unpublished_events(frm=frm, to=to) else: - events = await events_with_status([status], from_date=frm, to_date=to) + events = await get_mobilizon_events_with_status( + [status], from_date=frm, to_date=to + ) events = list(events) if events: show_events(events) diff --git a/mobilizon_reshare/dataclasses/__init__.py b/mobilizon_reshare/dataclasses/__init__.py new file mode 100644 index 0000000..57b9974 --- /dev/null +++ b/mobilizon_reshare/dataclasses/__init__.py @@ -0,0 +1,9 @@ +from mobilizon_reshare.dataclasses.event import _MobilizonEvent +from mobilizon_reshare.dataclasses.event_publication_status import ( + _EventPublicationStatus, +) +from mobilizon_reshare.dataclasses.publication import _EventPublication + +EventPublication = _EventPublication +MobilizonEvent = _MobilizonEvent +EventPublicationStatus = _EventPublicationStatus diff --git a/mobilizon_reshare/event/event.py b/mobilizon_reshare/dataclasses/event.py similarity index 52% rename from mobilizon_reshare/event/event.py rename to mobilizon_reshare/dataclasses/event.py index 8e294ad..ec13bc7 100644 --- a/mobilizon_reshare/event/event.py +++ b/mobilizon_reshare/dataclasses/event.py @@ -1,25 +1,26 @@ from dataclasses import dataclass, asdict -from enum import IntEnum -from typing import Optional +from typing import Optional, Iterable from uuid import UUID import arrow +from arrow import Arrow from jinja2 import Template from mobilizon_reshare.config.config import get_settings +from mobilizon_reshare.dataclasses.event_publication_status import ( + _EventPublicationStatus, + _compute_event_status, +) from mobilizon_reshare.models.event import Event -from mobilizon_reshare.models.publication import PublicationStatus, Publication - - -class EventPublicationStatus(IntEnum): - WAITING = 1 - FAILED = 2 - COMPLETED = 3 - PARTIAL = 4 +from mobilizon_reshare.storage.query.read import ( + get_all_events, + get_event, + get_events_without_publications, +) @dataclass -class MobilizonEvent: +class _MobilizonEvent: """Class representing an event retrieved from Mobilizon.""" name: str @@ -32,7 +33,7 @@ class MobilizonEvent: thumbnail_link: Optional[str] = None location: Optional[str] = None publication_time: Optional[dict[str, arrow.Arrow]] = None - status: EventPublicationStatus = EventPublicationStatus.WAITING + status: _EventPublicationStatus = _EventPublicationStatus.WAITING def __post_init__(self): assert self.begin_datetime.tzinfo == self.end_datetime.tzinfo @@ -41,9 +42,9 @@ class MobilizonEvent: self.publication_time = {} if self.publication_time: assert self.status in [ - EventPublicationStatus.COMPLETED, - EventPublicationStatus.PARTIAL, - EventPublicationStatus.FAILED, + _EventPublicationStatus.COMPLETED, + _EventPublicationStatus.PARTIAL, + _EventPublicationStatus.FAILED, ] def _fill_template(self, pattern: Template) -> str: @@ -55,11 +56,11 @@ class MobilizonEvent: @classmethod def from_model(cls, event: Event): - publication_status = cls._compute_event_status(list(event.publications)) + publication_status = _compute_event_status(list(event.publications)) publication_time = {} for pub in event.publications: - if publication_status != EventPublicationStatus.WAITING: + if publication_status != _EventPublicationStatus.WAITING: assert pub.timestamp is not None publication_time[pub.publisher.name] = arrow.get(pub.timestamp).to( "local" @@ -99,23 +100,58 @@ class MobilizonEvent: kwargs.update({"id": db_id}) return Event(**kwargs) - @staticmethod - def _compute_event_status( - publications: list[Publication], - ) -> EventPublicationStatus: - if not publications: - return EventPublicationStatus.WAITING + @classmethod + async def retrieve(cls, mobilizon_id): + return cls.from_model(await get_event(mobilizon_id)) - unique_statuses: set[PublicationStatus] = set( - pub.status for pub in publications + +async def get_all_mobilizon_events( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, +) -> list[_MobilizonEvent]: + return [_MobilizonEvent.from_model(event) for event in await get_all_events()] + + +async def get_published_events( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None +) -> Iterable[_MobilizonEvent]: + """ + Retrieves events that are not waiting. Function could be renamed to something more fitting + :return: + """ + return await get_mobilizon_events_with_status( + [ + _EventPublicationStatus.COMPLETED, + _EventPublicationStatus.PARTIAL, + _EventPublicationStatus.FAILED, + ], + from_date=from_date, + to_date=to_date, + ) + + +async def get_mobilizon_events_with_status( + status: list[_EventPublicationStatus], + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, +) -> Iterable[_MobilizonEvent]: + def _filter_event_with_status(event: Event) -> bool: + # This computes the status client-side instead of running in the DB. It shouldn't pose a performance problem + # in the short term, but should be moved to the query if possible. + event_status = _compute_event_status(list(event.publications)) + return event_status in status + + return map( + _MobilizonEvent.from_model, + filter(_filter_event_with_status, await get_all_events(from_date, to_date)), + ) + + +async def get_mobilizon_events_without_publications( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, +) -> list[_MobilizonEvent]: + return [ + _MobilizonEvent.from_model(event) + for event in await get_events_without_publications( + from_date=from_date, to_date=to_date ) - - if unique_statuses == { - PublicationStatus.COMPLETED, - PublicationStatus.FAILED, - }: - return EventPublicationStatus.PARTIAL - elif len(unique_statuses) == 1: - return EventPublicationStatus[unique_statuses.pop().name] - - raise ValueError(f"Illegal combination of PublicationStatus: {unique_statuses}") + ] diff --git a/mobilizon_reshare/dataclasses/event_publication_status.py b/mobilizon_reshare/dataclasses/event_publication_status.py new file mode 100644 index 0000000..276ce54 --- /dev/null +++ b/mobilizon_reshare/dataclasses/event_publication_status.py @@ -0,0 +1,27 @@ +from enum import IntEnum + +from mobilizon_reshare.models.publication import Publication, PublicationStatus + + +class _EventPublicationStatus(IntEnum): + WAITING = 1 + FAILED = 2 + COMPLETED = 3 + PARTIAL = 4 + + +def _compute_event_status(publications: list[Publication],) -> _EventPublicationStatus: + if not publications: + return _EventPublicationStatus.WAITING + + unique_statuses: set[PublicationStatus] = set(pub.status for pub in publications) + + if unique_statuses == { + PublicationStatus.COMPLETED, + PublicationStatus.FAILED, + }: + return _EventPublicationStatus.PARTIAL + elif len(unique_statuses) == 1: + return _EventPublicationStatus[unique_statuses.pop().name] + + raise ValueError(f"Illegal combination of PublicationStatus: {unique_statuses}") diff --git a/mobilizon_reshare/dataclasses/publication.py b/mobilizon_reshare/dataclasses/publication.py new file mode 100644 index 0000000..480a167 --- /dev/null +++ b/mobilizon_reshare/dataclasses/publication.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass +from functools import partial +from typing import List, Iterator +from uuid import UUID + +from tortoise.transactions import atomic + +from mobilizon_reshare.dataclasses.event import _MobilizonEvent +from mobilizon_reshare.models.publication import Publication +from mobilizon_reshare.publishers.abstract import ( + AbstractPlatform, + AbstractEventFormatter, +) +from mobilizon_reshare.storage.query.read import ( + get_event, + prefetch_publication_relations, +) + + +@dataclass +class BasePublication: + publisher: AbstractPlatform + formatter: AbstractEventFormatter + + +@dataclass +class _EventPublication(BasePublication): + event: _MobilizonEvent + id: UUID + + @classmethod + def from_orm(cls, model: Publication, event: _MobilizonEvent): + # imported here to avoid circular dependencies + from mobilizon_reshare.publishers.platforms.platform_mapping import ( + get_publisher_class, + get_formatter_class, + ) + + publisher = get_publisher_class(model.publisher.name)() + formatter = get_formatter_class(model.publisher.name)() + return cls(publisher, formatter, event, model.id,) + + @classmethod + async def retrieve(cls, publication_id): + publication = await prefetch_publication_relations( + Publication.get(id=publication_id) + ) + event = _MobilizonEvent.from_model(publication.event) + return cls.from_orm(publication, event) + + +@dataclass +class RecapPublication(BasePublication): + events: List[_MobilizonEvent] + + +@atomic() +async def build_publications_for_event( + event: _MobilizonEvent, publishers: Iterator[str] +) -> list[_EventPublication]: + publication_models = await event.to_model().build_publications(publishers) + return [_EventPublication.from_orm(m, event) for m in publication_models] + + +async def get_failed_publications_for_event( + event: _MobilizonEvent, +) -> List[_EventPublication]: + event_model = await get_event(event.mobilizon_id) + failed_publications = await event_model.get_failed_publications() + return list( + map(partial(_EventPublication.from_orm, event=event), failed_publications) + ) diff --git a/mobilizon_reshare/event/event_selection_strategies.py b/mobilizon_reshare/event/event_selection_strategies.py index c50bdaa..15183b9 100644 --- a/mobilizon_reshare/event/event_selection_strategies.py +++ b/mobilizon_reshare/event/event_selection_strategies.py @@ -5,7 +5,7 @@ from typing import List, Optional import arrow from mobilizon_reshare.config.config import get_settings -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent logger = logging.getLogger(__name__) diff --git a/mobilizon_reshare/main/publish.py b/mobilizon_reshare/main/publish.py index c08f721..d6fa7d6 100644 --- a/mobilizon_reshare/main/publish.py +++ b/mobilizon_reshare/main/publish.py @@ -2,10 +2,20 @@ import logging.config from typing import Optional, Iterator from mobilizon_reshare.config.command import CommandConfig -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.event import ( + get_published_events, + get_mobilizon_events_without_publications, +) +from mobilizon_reshare.dataclasses.publication import ( + _EventPublication, + build_publications_for_event, +) from mobilizon_reshare.event.event_selection_strategies import select_event_to_publish from mobilizon_reshare.publishers import get_active_publishers -from mobilizon_reshare.publishers.abstract import EventPublication +from mobilizon_reshare.publishers.coordinators.event_publishing.dry_run import ( + DryRunPublisherCoordinator, +) from mobilizon_reshare.publishers.coordinators.event_publishing.notify import ( PublicationFailureNotifiersCoordinator, ) @@ -13,21 +23,13 @@ from mobilizon_reshare.publishers.coordinators.event_publishing.publish import ( PublisherCoordinatorReport, PublisherCoordinator, ) -from mobilizon_reshare.storage.query.read import ( - get_published_events, - build_publications, - events_without_publications, -) from mobilizon_reshare.storage.query.write import save_publication_report -from mobilizon_reshare.publishers.coordinators.event_publishing.dry_run import ( - DryRunPublisherCoordinator, -) logger = logging.getLogger(__name__) async def publish_publications( - publications: list[EventPublication], + publications: list[_EventPublication], ) -> PublisherCoordinatorReport: report = PublisherCoordinator(publications).run() @@ -39,7 +41,7 @@ async def publish_publications( return report -def perform_dry_run(publications: list[EventPublication]): +def perform_dry_run(publications: list[_EventPublication]): return DryRunPublisherCoordinator(publications).run() @@ -53,7 +55,7 @@ async def publish_event( if not (publishers and all(publishers)): publishers = get_active_publishers() - publications = await build_publications(event, publishers) + publications = await build_publications_for_event(event, publishers) if command_config.dry_run: logger.info("Executing in dry run mode. No event is going to be published.") return perform_dry_run(publications) @@ -70,7 +72,7 @@ async def select_and_publish( :return: """ if unpublished_events is None: - unpublished_events = await events_without_publications() + unpublished_events = await get_mobilizon_events_without_publications() event = select_event_to_publish( list(await get_published_events()), unpublished_events, diff --git a/mobilizon_reshare/main/pull.py b/mobilizon_reshare/main/pull.py index a8445d7..9e33f50 100644 --- a/mobilizon_reshare/main/pull.py +++ b/mobilizon_reshare/main/pull.py @@ -1,6 +1,6 @@ import logging.config -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.mobilizon.events import get_mobilizon_future_events from mobilizon_reshare.storage.query.write import create_unpublished_events diff --git a/mobilizon_reshare/main/recap.py b/mobilizon_reshare/main/recap.py index b07622b..25014e8 100644 --- a/mobilizon_reshare/main/recap.py +++ b/mobilizon_reshare/main/recap.py @@ -4,31 +4,32 @@ from typing import Optional, List from arrow import now from mobilizon_reshare.config.command import CommandConfig -from mobilizon_reshare.event.event import EventPublicationStatus, MobilizonEvent +from mobilizon_reshare.dataclasses import EventPublicationStatus +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.event import get_mobilizon_events_with_status +from mobilizon_reshare.dataclasses.publication import RecapPublication from mobilizon_reshare.publishers import get_active_publishers -from mobilizon_reshare.publishers.abstract import RecapPublication +from mobilizon_reshare.publishers.coordinators import BaseCoordinatorReport from mobilizon_reshare.publishers.coordinators.event_publishing.notify import ( PublicationFailureNotifiersCoordinator, ) +from mobilizon_reshare.publishers.coordinators.recap_publishing.dry_run import ( + DryRunRecapCoordinator, +) from mobilizon_reshare.publishers.coordinators.recap_publishing.recap import ( RecapCoordinator, ) -from mobilizon_reshare.publishers.coordinators import BaseCoordinatorReport from mobilizon_reshare.publishers.platforms.platform_mapping import ( get_publisher_class, get_formatter_class, ) -from mobilizon_reshare.storage.query.read import events_with_status -from mobilizon_reshare.publishers.coordinators.recap_publishing.dry_run import ( - DryRunRecapCoordinator, -) logger = logging.getLogger(__name__) async def select_events_to_recap() -> List[MobilizonEvent]: return list( - await events_with_status( + await get_mobilizon_events_with_status( status=[EventPublicationStatus.COMPLETED], from_date=now() ) ) diff --git a/mobilizon_reshare/main/retry.py b/mobilizon_reshare/main/retry.py index 6c23efe..598aa97 100644 --- a/mobilizon_reshare/main/retry.py +++ b/mobilizon_reshare/main/retry.py @@ -2,22 +2,22 @@ import logging from typing import Optional from uuid import UUID +from tortoise.exceptions import DoesNotExist + +from mobilizon_reshare.dataclasses import MobilizonEvent, EventPublication +from mobilizon_reshare.dataclasses.publication import get_failed_publications_for_event from mobilizon_reshare.main.publish import publish_publications from mobilizon_reshare.publishers.coordinators.event_publishing.publish import ( PublisherCoordinatorReport, ) from mobilizon_reshare.storage.query.exceptions import EventNotFound -from mobilizon_reshare.storage.query.read import ( - get_failed_publications_for_event, - get_publication, -) logger = logging.getLogger(__name__) async def retry_event_publications(event_id) -> Optional[PublisherCoordinatorReport]: - - failed_publications = await get_failed_publications_for_event(event_id) + event = await MobilizonEvent.retrieve(event_id) + failed_publications = await get_failed_publications_for_event(event) if not failed_publications: logger.info("No failed publications found.") return @@ -27,8 +27,9 @@ async def retry_event_publications(event_id) -> Optional[PublisherCoordinatorRep async def retry_publication(publication_id) -> Optional[PublisherCoordinatorReport]: - publication = await get_publication(publication_id) - if not publication: + try: + publication = await EventPublication.retrieve(publication_id) + except DoesNotExist: logger.info(f"Publication {publication_id} not found.") return diff --git a/mobilizon_reshare/mobilizon/events.py b/mobilizon_reshare/mobilizon/events.py index 1a41b85..3943e1c 100644 --- a/mobilizon_reshare/mobilizon/events.py +++ b/mobilizon_reshare/mobilizon/events.py @@ -8,7 +8,7 @@ import arrow import requests from mobilizon_reshare.config.config import get_settings -from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus +from mobilizon_reshare.dataclasses import MobilizonEvent, _EventPublicationStatus logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ def parse_event(data): thumbnail_link=parse_picture(data), location=parse_location(data), publication_time=None, - status=EventPublicationStatus.WAITING, + status=_EventPublicationStatus.WAITING, last_update_time=arrow.get(data["updatedAt"]) if "updatedAt" in data else None, ) diff --git a/mobilizon_reshare/models/event.py b/mobilizon_reshare/models/event.py index 65f7207..272d078 100644 --- a/mobilizon_reshare/models/event.py +++ b/mobilizon_reshare/models/event.py @@ -1,5 +1,8 @@ +from typing import Iterator + from tortoise import fields from tortoise.models import Model +from tortoise.transactions import atomic from mobilizon_reshare.models import WithPydantic from mobilizon_reshare.models.publication import PublicationStatus, Publication @@ -42,3 +45,17 @@ class Event(Model, WithPydantic): publisher_id=publisher.id, publisher=publisher, ) + + async def build_publications(self, publishers: Iterator[str]): + return [ + await self.build_publication_by_publisher_name(name) for name in publishers + ] + + @atomic() + async def get_failed_publications(self,) -> list[Publication]: + return list( + filter( + lambda publications: publications.status == PublicationStatus.FAILED, + self.publications, + ) + ) diff --git a/mobilizon_reshare/publishers/abstract.py b/mobilizon_reshare/publishers/abstract.py index cc65cbb..fa14dd7 100644 --- a/mobilizon_reshare/publishers/abstract.py +++ b/mobilizon_reshare/publishers/abstract.py @@ -1,17 +1,14 @@ import inspect import logging from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import List, Optional -from uuid import UUID +from typing import Optional from dynaconf.utils.boxing import DynaBox from jinja2 import Environment, FileSystemLoader, Template from mobilizon_reshare.config.config import get_settings -from mobilizon_reshare.event.event import MobilizonEvent from .exceptions import InvalidAttribute -from ..models.publication import Publication +from ..dataclasses import _MobilizonEvent JINJA_ENV = Environment(loader=FileSystemLoader("/")) @@ -84,10 +81,10 @@ class AbstractPlatform(ABC, LoggerMixin, ConfLoaderMixin): pass @abstractmethod - def _send(self, message: str, event: Optional[MobilizonEvent] = None): + def _send(self, message: str, event: Optional[_MobilizonEvent] = None): raise NotImplementedError # pragma: no cover - def send(self, message: str, event: Optional[MobilizonEvent] = None): + def send(self, message: str, event: Optional[_MobilizonEvent] = None): """ Sends a message to the target channel """ @@ -110,7 +107,7 @@ class AbstractPlatform(ABC, LoggerMixin, ConfLoaderMixin): class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin): @abstractmethod - def _validate_event(self, event: MobilizonEvent) -> None: + def _validate_event(self, event: _MobilizonEvent) -> None: """ Validates publisher's event. Should raise ``PublisherError`` (or one of its subclasses) if event @@ -127,7 +124,7 @@ class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin): """ raise NotImplementedError # pragma: no cover - def validate_event(self, event: MobilizonEvent) -> None: + def validate_event(self, event: _MobilizonEvent) -> None: self._validate_event(event) self._validate_message(self.get_message_from_event(event)) @@ -138,7 +135,7 @@ class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin): """ return event - def get_message_from_event(self, event: MobilizonEvent) -> str: + def get_message_from_event(self, event: _MobilizonEvent) -> str: """ Retrieves a message from the event itself. """ @@ -167,7 +164,7 @@ class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin): ) return JINJA_ENV.get_template(template_path) - def get_recap_fragment(self, event: MobilizonEvent) -> str: + def get_recap_fragment(self, event: _MobilizonEvent) -> str: """ Retrieves the fragment that describes a single event inside the event recap. """ @@ -176,32 +173,3 @@ class AbstractEventFormatter(LoggerMixin, ConfLoaderMixin): def _preprocess_message(self, message: str): return message - - -@dataclass -class BasePublication: - publisher: AbstractPlatform - formatter: AbstractEventFormatter - - -@dataclass -class EventPublication(BasePublication): - event: MobilizonEvent - id: UUID - - @classmethod - def from_orm(cls, model: Publication, event: MobilizonEvent): - # imported here to avoid circular dependencies - from mobilizon_reshare.publishers.platforms.platform_mapping import ( - get_publisher_class, - get_formatter_class, - ) - - publisher = get_publisher_class(model.publisher.name)() - formatter = get_formatter_class(model.publisher.name)() - return cls(publisher, formatter, event, model.id,) - - -@dataclass -class RecapPublication(BasePublication): - events: List[MobilizonEvent] diff --git a/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py b/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py index c64bb8a..1aae03a 100644 --- a/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py +++ b/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py @@ -3,17 +3,16 @@ import logging from dataclasses import dataclass from typing import List, Optional +from mobilizon_reshare.dataclasses.publication import _EventPublication from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.publishers.abstract import EventPublication from mobilizon_reshare.publishers.coordinators import BasePublicationReport - logger = logging.getLogger(__name__) @dataclass class EventPublicationReport(BasePublicationReport): - publication: EventPublication + publication: _EventPublication published_content: Optional[str] = dataclasses.field(default=None) def get_failure_message(self): @@ -29,7 +28,7 @@ class EventPublicationReport(BasePublicationReport): class BaseEventPublishingCoordinator: - def __init__(self, publications: List[EventPublication]): + def __init__(self, publications: List[_EventPublication]): self.publications = publications def _safe_run(self, reasons, f, *args, **kwargs): diff --git a/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py b/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py index bd5dfe4..58514b2 100644 --- a/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py +++ b/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py @@ -1,9 +1,10 @@ import dataclasses +import logging from dataclasses import dataclass from typing import Sequence -import logging + +from mobilizon_reshare.dataclasses.publication import _EventPublication from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.publishers.abstract import EventPublication from mobilizon_reshare.publishers.coordinators import BaseCoordinatorReport from mobilizon_reshare.publishers.coordinators.event_publishing import ( BaseEventPublishingCoordinator, @@ -17,7 +18,7 @@ logger = logging.getLogger(__name__) @dataclass class PublisherCoordinatorReport(BaseCoordinatorReport): reports: Sequence[EventPublicationReport] - publications: Sequence[EventPublication] = dataclasses.field(default_factory=list) + publications: Sequence[_EventPublication] = dataclasses.field(default_factory=list) def __str__(self): platform_messages = [] diff --git a/mobilizon_reshare/publishers/coordinators/recap_publishing/recap.py b/mobilizon_reshare/publishers/coordinators/recap_publishing/recap.py index 957b3c6..5c812b7 100644 --- a/mobilizon_reshare/publishers/coordinators/recap_publishing/recap.py +++ b/mobilizon_reshare/publishers/coordinators/recap_publishing/recap.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Optional, Sequence, List from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.publishers.abstract import RecapPublication +from mobilizon_reshare.dataclasses.publication import RecapPublication from mobilizon_reshare.publishers.coordinators import ( BasePublicationReport, BaseCoordinatorReport, diff --git a/mobilizon_reshare/publishers/platforms/facebook.py b/mobilizon_reshare/publishers/platforms/facebook.py index 92532de..9b6a862 100644 --- a/mobilizon_reshare/publishers/platforms/facebook.py +++ b/mobilizon_reshare/publishers/platforms/facebook.py @@ -4,7 +4,7 @@ import facebook import pkg_resources from facebook import GraphAPIError -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.formatting.description import html_to_plaintext from mobilizon_reshare.publishers.abstract import ( AbstractPlatform, diff --git a/mobilizon_reshare/publishers/platforms/mastodon.py b/mobilizon_reshare/publishers/platforms/mastodon.py index d7d920a..09df153 100644 --- a/mobilizon_reshare/publishers/platforms/mastodon.py +++ b/mobilizon_reshare/publishers/platforms/mastodon.py @@ -5,7 +5,7 @@ import pkg_resources import requests from requests import Response -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.publishers.abstract import ( AbstractPlatform, AbstractEventFormatter, diff --git a/mobilizon_reshare/publishers/platforms/telegram.py b/mobilizon_reshare/publishers/platforms/telegram.py index 9db5486..3ae5b5f 100644 --- a/mobilizon_reshare/publishers/platforms/telegram.py +++ b/mobilizon_reshare/publishers/platforms/telegram.py @@ -6,7 +6,7 @@ import requests from bs4 import BeautifulSoup from requests import Response -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.publishers.abstract import ( AbstractEventFormatter, AbstractPlatform, diff --git a/mobilizon_reshare/publishers/platforms/twitter.py b/mobilizon_reshare/publishers/platforms/twitter.py index 0a1275c..90a8bb3 100644 --- a/mobilizon_reshare/publishers/platforms/twitter.py +++ b/mobilizon_reshare/publishers/platforms/twitter.py @@ -4,7 +4,7 @@ import pkg_resources from tweepy import OAuthHandler, API, TweepyException from tweepy.models import Status -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.publishers.abstract import ( AbstractPlatform, AbstractEventFormatter, diff --git a/mobilizon_reshare/publishers/platforms/zulip.py b/mobilizon_reshare/publishers/platforms/zulip.py index 13238cc..3b4195e 100644 --- a/mobilizon_reshare/publishers/platforms/zulip.py +++ b/mobilizon_reshare/publishers/platforms/zulip.py @@ -6,7 +6,7 @@ import requests from requests import Response from requests.auth import HTTPBasicAuth -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.formatting.description import html_to_markdown from mobilizon_reshare.publishers.abstract import ( AbstractPlatform, diff --git a/mobilizon_reshare/storage/query/read.py b/mobilizon_reshare/storage/query/read.py index 2edc47b..120bd6d 100644 --- a/mobilizon_reshare/storage/query/read.py +++ b/mobilizon_reshare/storage/query/read.py @@ -1,63 +1,16 @@ -from functools import partial -from typing import Iterable, Optional, Iterator +from typing import Iterable, Optional from uuid import UUID from arrow import Arrow -from tortoise.exceptions import DoesNotExist from tortoise.queryset import QuerySet from tortoise.transactions import atomic -from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import Publication, PublicationStatus from mobilizon_reshare.models.publisher import Publisher -from mobilizon_reshare.publishers.abstract import EventPublication - from mobilizon_reshare.storage.query.exceptions import EventNotFound -async def get_published_events( - from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None -) -> Iterable[MobilizonEvent]: - """ - Retrieves events that are not waiting. Function could be renamed to something more fitting - :return: - """ - return await events_with_status( - [ - EventPublicationStatus.COMPLETED, - EventPublicationStatus.PARTIAL, - EventPublicationStatus.FAILED, - ], - from_date=from_date, - to_date=to_date, - ) - - -async def events_with_status( - status: list[EventPublicationStatus], - from_date: Optional[Arrow] = None, - to_date: Optional[Arrow] = None, -) -> Iterable[MobilizonEvent]: - def _filter_event_with_status(event: Event) -> bool: - # This computes the status client-side instead of running in the DB. It shouldn't pose a performance problem - # in the short term, but should be moved to the query if possible. - event_status = MobilizonEvent._compute_event_status(list(event.publications)) - return event_status in status - - query = Event.all() - - return map( - MobilizonEvent.from_model, - filter( - _filter_event_with_status, - await prefetch_event_relations( - _add_date_window(query, "begin_datetime", from_date, to_date) - ), - ), - ) - - async def get_all_publications( from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, ) -> Iterable[Publication]: @@ -66,12 +19,6 @@ async def get_all_publications( ) -async def get_all_mobilizon_events( - from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, -) -> list[MobilizonEvent]: - return [MobilizonEvent.from_model(event) for event in await get_all_events()] - - async def get_all_events( from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None ): @@ -96,7 +43,12 @@ async def prefetch_publication_relations( queryset: QuerySet[Publication], ) -> list[Publication]: publication = ( - await queryset.prefetch_related("publisher", "event") + await queryset.prefetch_related( + "publisher", + "event", + "event__publications", + "event__publications__publisher", + ) .order_by("timestamp") .distinct() ) @@ -129,16 +81,6 @@ async def publications_with_status( ) -async def events_without_publications( - from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, -) -> list[MobilizonEvent]: - query = Event.filter(publications__id=None) - events = await prefetch_event_relations( - _add_date_window(query, "begin_datetime", from_date, to_date) - ) - return [MobilizonEvent.from_model(event) for event in events] - - async def get_event(event_mobilizon_id: UUID) -> Event: events = await prefetch_event_relations( Event.filter(mobilizon_id=event_mobilizon_id) @@ -149,73 +91,10 @@ async def get_event(event_mobilizon_id: UUID) -> Event: return events[0] -async def get_event_publications( - mobilizon_event: MobilizonEvent, -) -> list[EventPublication]: - event = await get_event(mobilizon_event.mobilizon_id) - return [EventPublication.from_orm(p, mobilizon_event) for p in event.publications] - - -async def get_mobilizon_event(event_mobilizon_id: UUID) -> MobilizonEvent: - return MobilizonEvent.from_model(await get_event(event_mobilizon_id)) - - -async def get_publisher_by_name(name) -> Publisher: - return await Publisher.filter(name=name).first() - - -async def is_known(event: MobilizonEvent) -> bool: - try: - await get_event(event.mobilizon_id) - return True - except EventNotFound: - return False - - -@atomic() -async def build_publications( - event: MobilizonEvent, publishers: Iterator[str] -) -> list[EventPublication]: - event_model = await get_event(event.mobilizon_id) - models = [ - await event_model.build_publication_by_publisher_name(name) - for name in publishers - ] - return [EventPublication.from_orm(m, event) for m in models] - - -@atomic() -async def get_failed_publications_for_event( - event_mobilizon_id: UUID, -) -> list[EventPublication]: - event = await get_event(event_mobilizon_id) - failed_publications = list( - filter( - lambda publications: publications.status == PublicationStatus.FAILED, - event.publications, - ) +async def get_events_without_publications( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, +) -> list[Event]: + query = Event.filter(publications__id=None) + return await prefetch_event_relations( + _add_date_window(query, "begin_datetime", from_date, to_date) ) - for p in failed_publications: - await p.fetch_related("publisher") - mobilizon_event = MobilizonEvent.from_model(event) - return list( - map( - partial(EventPublication.from_orm, event=mobilizon_event), - failed_publications, - ) - ) - - -@atomic() -async def get_publication(publication_id: UUID): - try: - publication = await prefetch_publication_relations( - Publication.get(id=publication_id).first() - ) - # TODO: this is redundant but there's some prefetch problem otherwise - publication.event = await get_event(publication.event.mobilizon_id) - return EventPublication.from_orm( - event=MobilizonEvent.from_model(publication.event), model=publication - ) - except DoesNotExist: - return None diff --git a/mobilizon_reshare/storage/query/write.py b/mobilizon_reshare/storage/query/write.py index 8032709..ef662d1 100644 --- a/mobilizon_reshare/storage/query/write.py +++ b/mobilizon_reshare/storage/query/write.py @@ -1,33 +1,32 @@ import logging -from typing import Iterable, Optional +from typing import Iterable import arrow from tortoise.transactions import atomic -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.event import ( + get_mobilizon_events_without_publications, +) from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import Publication from mobilizon_reshare.models.publisher import Publisher +from mobilizon_reshare.publishers.coordinators.event_publishing import ( + EventPublicationReport, +) from mobilizon_reshare.publishers.coordinators.event_publishing.publish import ( PublisherCoordinatorReport, ) -from mobilizon_reshare.storage.query.read import ( - events_without_publications, - is_known, - get_publisher_by_name, - get_event, -) - - -async def create_publisher(name: str, account_ref: Optional[str] = None) -> None: - await Publisher.create(name=name, account_ref=account_ref) +from mobilizon_reshare.storage.query.read import get_event @atomic() -async def upsert_publication(publication_report, event): +async def upsert_publication( + publication_report: EventPublicationReport, event: MobilizonEvent +): - publisher = await get_publisher_by_name( - name=publication_report.publication.publisher.name + publisher_model = await ( + Publisher.get(name=publication_report.publication.publisher.name).first() ) old_publication = await Publication.filter( id=publication_report.publication.id @@ -44,7 +43,7 @@ async def upsert_publication(publication_report, event): await Publication.create( id=publication_report.publication.id, event_id=event.id, - publisher_id=publisher.id, + publisher_id=publisher_model.id, status=publication_report.status, reason=publication_report.reason, timestamp=arrow.now().datetime, @@ -76,7 +75,7 @@ async def create_unpublished_events( """ # There are three cases: for event in events_from_mobilizon: - if not await is_known(event): + if not await Event.exists(mobilizon_id=event.mobilizon_id): # Either an event is unknown await event.to_model().save() else: @@ -86,7 +85,7 @@ async def create_unpublished_events( await event.to_model(db_id=event_model.id).save(force_update=True) # Or it's known and unchanged, in which case we do nothing. - return await events_without_publications() + return await get_mobilizon_events_without_publications() @atomic() @@ -95,4 +94,4 @@ async def update_publishers(names: Iterable[str],) -> None: known_publisher_names = set(p.name for p in await Publisher.all()) for name in names.difference(known_publisher_names): logging.info(f"Creating {name} publisher") - await create_publisher(name) + await Publisher.create(name=name, account_ref=None) diff --git a/poetry.lock b/poetry.lock index 1455eee..7c1d393 100644 --- a/poetry.lock +++ b/poetry.lock @@ -38,7 +38,7 @@ python-versions = "*" [[package]] name = "anyio" -version = "3.6.2" +version = "3.6.1" description = "High level compatibility layer for multiple asynchronous event loop implementations" category = "main" optional = false @@ -51,7 +51,7 @@ sniffio = ">=1.1" [package.extras] doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] -trio = ["trio (>=0.16,<0.22)"] +trio = ["trio (>=0.16)"] [[package]] name = "appdirs" @@ -85,16 +85,16 @@ tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] [[package]] name = "asyncpg" -version = "0.27.0" +version = "0.26.0" description = "An asyncio PostgreSQL driver" category = "main" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.6.0" [package.extras] -dev = ["Cython (>=0.29.24,<0.30.0)", "pytest (>=6.0)", "Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "flake8 (>=5.0.4,<5.1.0)", "uvloop (>=0.15.3)"] +dev = ["Cython (>=0.29.24,<0.30.0)", "pytest (>=6.0)", "Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)"] -test = ["flake8 (>=5.0.4,<5.1.0)", "uvloop (>=0.15.3)"] +test = ["pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"] [[package]] name = "asynctest" @@ -128,7 +128,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "babel" -version = "2.11.0" +version = "2.10.3" description = "Internationalization utilities" category = "dev" optional = false @@ -154,7 +154,7 @@ lxml = ["lxml"] [[package]] name = "certifi" -version = "2022.12.7" +version = "2022.9.24" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -162,11 +162,11 @@ python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.1.1" +version = "2.0.12" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.6.0" +python-versions = ">=3.5.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -184,11 +184,11 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.6" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" @@ -265,7 +265,7 @@ requests = "*" [[package]] name = "fastapi" -version = "0.85.2" +version = "0.85.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" category = "main" optional = false @@ -278,8 +278,8 @@ starlette = "0.20.4" [package.extras] all = ["email-validator (>=1.1.1,<2.0.0)", "itsdangerous (>=1.1.0,<3.0.0)", "jinja2 (>=2.11.2,<4.0.0)", "orjson (>=3.2.1,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"] dev = ["autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "pre-commit (>=2.17.0,<3.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer[all] (>=0.6.1,<0.7.0)"] -test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.982)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest-cov (>=2.12.0,<5.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "sqlalchemy (>=1.3.18,<=1.4.41)", "types-orjson (==3.6.2)", "types-ujson (==5.5.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer (>=0.4.1,<0.7.0)"] +test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.971)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "types-orjson (==3.6.2)", "types-ujson (==5.4.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] [[package]] name = "fastapi-pagination" @@ -311,24 +311,24 @@ scylla-driver = ["scylla-driver (>=3.25.6,<4.0.0)"] [[package]] name = "h11" -version = "0.14.0" +version = "0.12.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.6" [[package]] name = "httpcore" -version = "0.16.2" +version = "0.15.0" description = "A minimal low-level HTTP client." category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -anyio = ">=3.0,<5.0" +anyio = ">=3.0.0,<4.0.0" certifi = "*" -h11 = ">=0.13,<0.15" +h11 = ">=0.11,<0.13" sniffio = ">=1.0.0,<2.0.0" [package.extras] @@ -337,7 +337,7 @@ socks = ["socksio (>=1.0.0,<2.0.0)"] [[package]] name = "httpx" -version = "0.23.1" +version = "0.23.0" description = "The next generation HTTP client." category = "dev" optional = false @@ -345,13 +345,13 @@ python-versions = ">=3.7" [package.dependencies] certifi = "*" -httpcore = ">=0.15.0,<0.17.0" +httpcore = ">=0.15.0,<0.16.0" rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} sniffio = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"] +brotli = ["brotlicffi", "brotli"] +cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10,<13)", "pygments (>=2.0.0,<3.0.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (>=1.0.0,<2.0.0)"] @@ -373,7 +373,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "5.1.0" +version = "5.0.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -385,7 +385,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -640,7 +640,7 @@ unidecode = ["Unidecode (>=1.1.1)"] [[package]] name = "pytz" -version = "2022.6" +version = "2022.4" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -930,11 +930,11 @@ python-versions = ">=3.7" [[package]] name = "tomlkit" -version = "0.11.6" +version = "0.11.5" description = "Style preserving TOML library" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.6,<4.0" [[package]] name = "tortoise-orm" @@ -995,11 +995,11 @@ python-versions = ">=3.5" [[package]] name = "urllib3" -version = "1.26.13" +version = "1.26.12" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] @@ -1024,7 +1024,7 @@ standard = ["websockets (>=10.0)", "httptools (>=0.4.0)", "watchgod (>=0.6)", "p [[package]] name = "zipp" -version = "3.11.0" +version = "3.9.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false @@ -1032,7 +1032,7 @@ python-versions = ">=3.7" [package.extras] docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" @@ -1069,7 +1069,10 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"}, ] certifi = [] -charset-normalizer = [] +charset-normalizer = [ + {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, + {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, +] click = [] colorama = [] coverage = [] @@ -1083,7 +1086,10 @@ dynaconf = [] facebook-sdk = [] fastapi = [] fastapi-pagination = [] -h11 = [] +h11 = [ + {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, + {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, +] httpcore = [] httpx = [] idna = [] diff --git a/tests/commands/test_list.py b/tests/commands/test_list.py index b7639fe..3b571c3 100644 --- a/tests/commands/test_list.py +++ b/tests/commands/test_list.py @@ -3,7 +3,7 @@ from arrow import arrow from mobilizon_reshare.cli.commands.list.list_event import list_events from mobilizon_reshare.cli.commands.list.list_publication import list_publications -from mobilizon_reshare.event.event import EventPublicationStatus +from mobilizon_reshare.dataclasses.event import _EventPublicationStatus from mobilizon_reshare.models.publication import PublicationStatus spec = { @@ -40,7 +40,7 @@ async def test_list_events(capsys, generate_models): @pytest.mark.asyncio async def test_list_events_with_status(capsys, generate_models): await generate_models(spec) - await list_events(status=EventPublicationStatus.WAITING) + await list_events(status=_EventPublicationStatus.WAITING) output = capsys.readouterr() assert clean_output(output) == [ "event_0 WAITING 00000000-0000-0000-0000-000000000000" @@ -114,7 +114,7 @@ async def test_list_publications_empty(capsys, generate_models): @pytest.mark.asyncio async def test_list_events_empty_with_status(capsys, generate_models): - await list_events(status=EventPublicationStatus.FAILED) + await list_events(status=_EventPublicationStatus.FAILED) output = capsys.readouterr() assert clean_output(output) == ["No event found with status: FAILED"] diff --git a/tests/commands/test_publish.py b/tests/commands/test_publish.py index 23a55eb..bf7d4d2 100644 --- a/tests/commands/test_publish.py +++ b/tests/commands/test_publish.py @@ -2,8 +2,9 @@ from logging import DEBUG import pytest +from mobilizon_reshare.dataclasses import EventPublicationStatus +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.main.publish import select_and_publish, publish_event -from mobilizon_reshare.event.event import EventPublicationStatus, MobilizonEvent from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.storage.query.read import get_all_publications diff --git a/tests/commands/test_pull.py b/tests/commands/test_pull.py index 616c870..91c4e84 100644 --- a/tests/commands/test_pull.py +++ b/tests/commands/test_pull.py @@ -2,16 +2,16 @@ from logging import DEBUG, INFO import pytest -from mobilizon_reshare.storage.query.read import ( +from mobilizon_reshare.dataclasses.event import ( get_all_mobilizon_events, - events_without_publications, + get_mobilizon_events_without_publications, ) +from mobilizon_reshare.main.pull import pull +from mobilizon_reshare.main.start import start from tests.commands.conftest import ( second_event_element, first_event_element, ) -from mobilizon_reshare.main.pull import pull -from mobilizon_reshare.main.start import start from tests.conftest import event_0, event_1 empty_specification = {"event": 0, "publications": [], "publisher": []} @@ -74,7 +74,7 @@ async def test_pull( assert ( f"There are now {len(expected_result)} unpublished events." in caplog.text ) - assert expected_result == await events_without_publications() + assert expected_result == await get_mobilizon_events_without_publications() @pytest.mark.asyncio @@ -113,7 +113,7 @@ async def test_pull_start( with caplog.at_level(INFO): assert await pull() == expected_pull assert expected_pull == await get_all_mobilizon_events() - assert expected_pull == await events_without_publications() + assert expected_pull == await get_mobilizon_events_without_publications() report = await start(command_config) assert report.successful @@ -127,7 +127,8 @@ async def test_pull_start( event.mobilizon_id for event in await get_all_mobilizon_events() ) assert (pull_ids - publish_ids) == set( - event.mobilizon_id for event in await events_without_publications() + event.mobilizon_id + for event in await get_mobilizon_events_without_publications() ) @@ -191,7 +192,10 @@ async def test_multiple_pull( assert await pull() assert f"There are now {len(expected_first)} unpublished events." in caplog.text assert expected_first == await get_all_mobilizon_events() - assert await events_without_publications() == await get_all_mobilizon_events() + assert ( + await get_mobilizon_events_without_publications() + == await get_all_mobilizon_events() + ) # I clean the message collector message_collector.data = [] @@ -204,4 +208,7 @@ async def test_multiple_pull( assert set(event.mobilizon_id for event in expected_last) == set( event.mobilizon_id for event in await get_all_mobilizon_events() ) - assert await events_without_publications() == await get_all_mobilizon_events() + assert ( + await get_mobilizon_events_without_publications() + == await get_all_mobilizon_events() + ) diff --git a/tests/commands/test_start.py b/tests/commands/test_start.py index 7db3757..01d307a 100644 --- a/tests/commands/test_start.py +++ b/tests/commands/test_start.py @@ -3,12 +3,13 @@ from logging import DEBUG, INFO import pytest from mobilizon_reshare.config.command import CommandConfig -from mobilizon_reshare.storage.query.read import get_all_mobilizon_events -from tests.commands.conftest import simple_event_element, second_event_element -from mobilizon_reshare.event.event import EventPublicationStatus, MobilizonEvent +from mobilizon_reshare.dataclasses import EventPublicationStatus +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.event import get_all_mobilizon_events from mobilizon_reshare.main.start import start from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import PublicationStatus +from tests.commands.conftest import simple_event_element, second_event_element one_published_event_specification = { "event": 1, diff --git a/tests/conftest.py b/tests/conftest.py index 5a2af02..34bf074 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ from tortoise import Tortoise import mobilizon_reshare from mobilizon_reshare.config.command import CommandConfig from mobilizon_reshare.config.config import get_settings -from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus +from mobilizon_reshare.dataclasses import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.notification import Notification, NotificationStatus from mobilizon_reshare.models.publication import Publication, PublicationStatus diff --git a/tests/formatting/test_output_format.py b/tests/formatting/test_output_format.py index c0420d7..5410837 100644 --- a/tests/formatting/test_output_format.py +++ b/tests/formatting/test_output_format.py @@ -4,18 +4,10 @@ from uuid import UUID import arrow import pytest -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.publishers.platforms.platform_mapping import get_formatter_class -begin_date = arrow.get( - datetime( - year=2021, - month=1, - day=1, - hour=11, - minute=30, - ) -) +begin_date = arrow.get(datetime(year=2021, month=1, day=1, hour=11, minute=30,)) end_date = begin_date.shift(hours=1) diff --git a/tests/mobilizon/test_events.py b/tests/mobilizon/test_events.py index 4dc6be4..494415e 100644 --- a/tests/mobilizon/test_events.py +++ b/tests/mobilizon/test_events.py @@ -3,7 +3,7 @@ from uuid import UUID import arrow import pytest -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.mobilizon.events import ( get_mobilizon_future_events, MobilizonRequestFailed, diff --git a/tests/models/test_event.py b/tests/models/test_event.py index 42329e1..1ce6874 100644 --- a/tests/models/test_event.py +++ b/tests/models/test_event.py @@ -5,7 +5,9 @@ import arrow import pytest import tortoise.timezone -from mobilizon_reshare.event.event import EventPublicationStatus, MobilizonEvent +from mobilizon_reshare.dataclasses import EventPublicationStatus +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.event_publication_status import _compute_event_status from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.publication import PublicationStatus @@ -191,4 +193,4 @@ async def test_mobilizon_event_compute_status_partial( ) await publication.save() publications.append(publication) - assert MobilizonEvent._compute_event_status(publications) == expected_result + assert _compute_event_status(publications) == expected_result diff --git a/tests/publishers/conftest.py b/tests/publishers/conftest.py index eb1abde..da94e2e 100644 --- a/tests/publishers/conftest.py +++ b/tests/publishers/conftest.py @@ -5,7 +5,7 @@ from uuid import UUID import arrow import pytest -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent from mobilizon_reshare.publishers.abstract import ( AbstractPlatform, AbstractEventFormatter, diff --git a/tests/publishers/test_coordinator.py b/tests/publishers/test_coordinator.py index 195d46d..5a1008e 100644 --- a/tests/publishers/test_coordinator.py +++ b/tests/publishers/test_coordinator.py @@ -1,17 +1,20 @@ import logging from datetime import timedelta -from uuid import UUID from unittest.mock import MagicMock +from uuid import UUID import pytest -from mobilizon_reshare.event.event import MobilizonEvent +from mobilizon_reshare.dataclasses import MobilizonEvent +from mobilizon_reshare.dataclasses.publication import ( + _EventPublication, + RecapPublication, +) from mobilizon_reshare.models.publication import ( PublicationStatus, Publication as PublicationModel, ) from mobilizon_reshare.models.publisher import Publisher -from mobilizon_reshare.publishers.abstract import EventPublication, RecapPublication from mobilizon_reshare.publishers.coordinators.event_publishing.notify import ( PublicationFailureNotifiersCoordinator, ) @@ -31,7 +34,7 @@ def failure_report(mock_publisher_invalid, event): return EventPublicationReport( status=PublicationStatus.FAILED, reason="some failure", - publication=EventPublication( + publication=_EventPublication( publisher=mock_publisher_invalid, formatter=None, event=event, @@ -103,7 +106,7 @@ async def mock_publications( timestamp=today + timedelta(hours=i), reason=None, ) - publication = EventPublication.from_orm(publication, test_event) + publication = _EventPublication.from_orm(publication, test_event) publication.publisher = mock_publisher_valid publication.formatter = mock_formatter_valid result.append(publication) diff --git a/tests/publishers/test_zulip.py b/tests/publishers/test_zulip.py index 89cc401..6339673 100644 --- a/tests/publishers/test_zulip.py +++ b/tests/publishers/test_zulip.py @@ -3,6 +3,7 @@ import requests import responses from mobilizon_reshare.config.config import get_settings +from mobilizon_reshare.dataclasses.publication import build_publications_for_event from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.publishers.coordinators.event_publishing.publish import ( PublisherCoordinator, @@ -14,7 +15,7 @@ from mobilizon_reshare.publishers.exceptions import ( HTTPResponseError, ) from mobilizon_reshare.publishers.platforms.zulip import ZulipFormatter, ZulipPublisher -from mobilizon_reshare.storage.query.read import build_publications, get_all_publishers +from mobilizon_reshare.storage.query.read import get_all_publishers one_publication_specification = { "event": 1, @@ -104,7 +105,7 @@ async def setup_db(generate_models): async def unsaved_publications(setup_db, event): await event.to_model().save() publishers = [p.name for p in await get_all_publishers()] - return await build_publications(event, publishers) + return await build_publications_for_event(event, publishers) @pytest.mark.asyncio diff --git a/tests/storage/test_query.py b/tests/storage/test_query.py index 3ce56ba..617d681 100644 --- a/tests/storage/test_query.py +++ b/tests/storage/test_query.py @@ -1,23 +1,20 @@ from datetime import timedelta -from uuid import UUID import arrow import pytest -from mobilizon_reshare.event.event import EventPublicationStatus -from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.publishers.abstract import EventPublication -from mobilizon_reshare.storage.query.read import ( +from mobilizon_reshare.dataclasses.event import ( + _EventPublicationStatus, get_published_events, - events_with_status, - publications_with_status, - events_without_publications, - build_publications, - get_event_publications, + get_mobilizon_events_with_status, + get_mobilizon_events_without_publications, ) +from mobilizon_reshare.dataclasses.publication import build_publications_for_event +from mobilizon_reshare.models.publication import PublicationStatus +from mobilizon_reshare.storage.query.read import publications_with_status from tests import today -from tests.storage import complete_specification from tests.conftest import event_0, event_1, event_3 +from tests.storage import complete_specification from tests.storage import result_publication @@ -57,7 +54,7 @@ async def test_get_published_events(generate_models): ], ) async def test_publications_with_status( - status, mobilizon_id, from_date, to_date, expected_result, generate_models, + status, mobilizon_id, from_date, to_date, expected_result, generate_models, ): await generate_models(complete_specification) publications = await publications_with_status( @@ -70,11 +67,11 @@ async def test_publications_with_status( @pytest.mark.asyncio @pytest.mark.parametrize( "status, expected_events_count", - [(EventPublicationStatus.COMPLETED, 2), (EventPublicationStatus.PARTIAL, 1)], + [(_EventPublicationStatus.COMPLETED, 2), (_EventPublicationStatus.PARTIAL, 1)], ) async def test_event_with_status(generate_models, status, expected_events_count): await generate_models(complete_specification) - result = list(await events_with_status([status])) + result = list(await get_mobilizon_events_with_status([status])) assert len(result) == expected_events_count @@ -84,37 +81,39 @@ async def test_event_with_status(generate_models, status, expected_events_count) "status, expected_events_count, begin_window, end_window", [ ( - EventPublicationStatus.COMPLETED, - 2, - arrow.get(today + timedelta(hours=-1)), - None, + _EventPublicationStatus.COMPLETED, + 2, + arrow.get(today + timedelta(hours=-1)), + None, ), ( - EventPublicationStatus.COMPLETED, - 1, - arrow.get(today + timedelta(hours=1)), - None, + _EventPublicationStatus.COMPLETED, + 1, + arrow.get(today + timedelta(hours=1)), + None, ), ( - EventPublicationStatus.COMPLETED, - 1, - arrow.get(today + timedelta(hours=-2)), - arrow.get(today + timedelta(hours=1)), + _EventPublicationStatus.COMPLETED, + 1, + arrow.get(today + timedelta(hours=-2)), + arrow.get(today + timedelta(hours=1)), ), ( - EventPublicationStatus.COMPLETED, - 0, - arrow.get(today + timedelta(hours=-2)), - arrow.get(today + timedelta(hours=0)), + _EventPublicationStatus.COMPLETED, + 0, + arrow.get(today + timedelta(hours=-2)), + arrow.get(today + timedelta(hours=0)), ), ], ) async def test_event_with_status_window( - generate_models, status, expected_events_count, begin_window, end_window + generate_models, status, expected_events_count, begin_window, end_window ): await generate_models(complete_specification) result = list( - await events_with_status([status], from_date=begin_window, to_date=end_window) + await get_mobilizon_events_with_status( + [status], from_date=begin_window, to_date=end_window + ) ) assert len(result) == expected_events_count @@ -126,30 +125,30 @@ async def test_event_with_status_window( [ ({"event": 2, "publications": [], "publisher": ["zulip"]}, [event_0, event_1],), ( - { - "event": 3, - "publications": [ - { - "event_idx": 1, - "publisher_idx": 0, - "status": PublicationStatus.FAILED, - }, - { - "event_idx": 2, - "publisher_idx": 0, - "status": PublicationStatus.COMPLETED, - }, - ], - "publisher": ["zulip"], - }, - [event_0], + { + "event": 3, + "publications": [ + { + "event_idx": 1, + "publisher_idx": 0, + "status": PublicationStatus.FAILED, + }, + { + "event_idx": 2, + "publisher_idx": 0, + "status": PublicationStatus.COMPLETED, + }, + ], + "publisher": ["zulip"], + }, + [event_0], ), (complete_specification, [event_3],), ], ) async def test_events_without_publications(spec, expected_events, generate_models): await generate_models(spec) - unpublished_events = list(await events_without_publications()) + unpublished_events = list(await get_mobilizon_events_without_publications()) assert len(unpublished_events) == len(expected_events) assert unpublished_events == expected_events @@ -160,82 +159,35 @@ async def test_events_without_publications(spec, expected_events, generate_model [ ([], {"event": 2, "publications": [], "publisher": ["zulip"]}, event_0, 0,), ( - ["zulip"], - {"event": 2, "publications": [], "publisher": ["zulip"]}, - event_0, - 1, + ["zulip"], + {"event": 2, "publications": [], "publisher": ["zulip"]}, + event_0, + 1, ), ( - ["telegram", "zulip", "mastodon", "facebook"], - { - "event": 2, - "publications": [], - "publisher": ["telegram", "zulip", "mastodon", "facebook"], - }, - event_0, - 4, + ["telegram", "zulip", "mastodon", "facebook"], + { + "event": 2, + "publications": [], + "publisher": ["telegram", "zulip", "mastodon", "facebook"], + }, + event_0, + 4, ), ], indirect=["mock_active_publishers"], ) async def test_build_publications( - mock_active_publishers, spec, event, n_publications, generate_models + mock_active_publishers, spec, event, n_publications, generate_models ): await generate_models(spec) - publications = list(await build_publications(event, mock_active_publishers)) + publications = list( + await build_publications_for_event(event, mock_active_publishers) + ) assert len(publications) == n_publications for p in publications: assert p.event == event assert p.publisher.name in mock_active_publishers - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "mock_active_publishers, spec, event, publications_ids", - [ - ( - ["telegram", "zulip", "mastodon", "facebook"], - {"event": 2, "publications": [], "publisher": ["zulip"]}, - event_0, - [], - ), - ( - ["telegram", "zulip", "mastodon", "facebook"], - { - "event": 2, - "publications": [ - { - "event_idx": 1, - "publisher_idx": 0, - "status": PublicationStatus.COMPLETED, - }, - { - "event_idx": 0, - "publisher_idx": 0, - "status": PublicationStatus.FAILED, - }, - ], - "publisher": ["zulip"], - }, - event_1, - # This tuples are made like so: (event_mobilizon_id, publication_id) - [(UUID(int=1), UUID(int=0))], - ), - ], - indirect=["mock_active_publishers"], -) -async def test_get_event_publications( - mock_active_publishers, spec, event, publications_ids, generate_models -): - await generate_models(spec) - - publications = list(await get_event_publications(event)) - - assert len(publications) == len(publications_ids) - - for i, p in enumerate(publications): - assert p.event.mobilizon_id == publications_ids[i][0] - assert p.id == publications_ids[i][1] diff --git a/tests/storage/test_read_query.py b/tests/storage/test_read_query.py index 3464952..6d13b74 100644 --- a/tests/storage/test_read_query.py +++ b/tests/storage/test_read_query.py @@ -2,7 +2,7 @@ from uuid import UUID import pytest -from mobilizon_reshare.storage.query.read import get_all_mobilizon_events +from mobilizon_reshare.dataclasses.event import get_all_mobilizon_events @pytest.mark.asyncio diff --git a/tests/storage/test_update.py b/tests/storage/test_update.py index 52c1b3b..f87f624 100644 --- a/tests/storage/test_update.py +++ b/tests/storage/test_update.py @@ -2,9 +2,9 @@ from uuid import UUID import pytest +from mobilizon_reshare.dataclasses.publication import _EventPublication from mobilizon_reshare.models.publication import PublicationStatus, Publication from mobilizon_reshare.models.publisher import Publisher -from mobilizon_reshare.publishers.abstract import EventPublication from mobilizon_reshare.publishers.coordinators.event_publishing.publish import ( EventPublicationReport, PublisherCoordinatorReport, @@ -18,8 +18,8 @@ from mobilizon_reshare.storage.query.write import ( update_publishers, create_unpublished_events, ) -from tests.storage import complete_specification from tests.conftest import event_6, event_0, event_1, event_2, event_3, event_3_updated +from tests.storage import complete_specification two_publishers_specification = {"publisher": ["telegram", "twitter"]} @@ -144,7 +144,7 @@ async def test_create_unpublished_events( EventPublicationReport( status=PublicationStatus.COMPLETED, reason="", - publication=EventPublication( + publication=_EventPublication( id=UUID(int=6), formatter=TelegramFormatter(), event=event_1,