diff --git a/mobilizon_bots/cli.py b/mobilizon_bots/cli.py deleted file mode 100644 index dd04fe7..0000000 --- a/mobilizon_bots/cli.py +++ /dev/null @@ -1,21 +0,0 @@ -import asyncio - -import click - -from mobilizon_bots.main import main - - -@click.group() -def mobilizon_bots(): - pass - - -@mobilizon_bots.command() -@click.option("--settings-file", type=click.Path(exists=True)) -def start(settings_file): - - asyncio.run(main([settings_file] if settings_file else None)) - - -if __name__ == "__main__": - mobilizon_bots() diff --git a/mobilizon_bots/cli/__init__.py b/mobilizon_bots/cli/__init__.py new file mode 100644 index 0000000..2587bbb --- /dev/null +++ b/mobilizon_bots/cli/__init__.py @@ -0,0 +1,39 @@ +import asyncio +import logging +import traceback +from logging.config import dictConfig +from pathlib import Path + +from mobilizon_bots.config.config import update_settings_files +from mobilizon_bots.storage.db import tear_down, MobilizonBotsDB + +logger = logging.getLogger(__name__) + + +async def graceful_exit(code): + await tear_down() + exit(code) + + +async def init(settings_file): + settings = update_settings_files(settings_file) + dictConfig(settings["logging"]) + db_path = Path(settings.db_path) + db = MobilizonBotsDB(db_path) + await db.setup() + + +async def _safe_execution(f, settings_file): + await init(settings_file) + return_code = 1 + try: + return_code = await f() + except Exception: + traceback.print_exc() + finally: + logger.debug("Closing") + await graceful_exit(return_code) + + +def safe_execution(f, settings_file): + asyncio.run(_safe_execution(f, settings_file)) diff --git a/mobilizon_bots/cli/cli.py b/mobilizon_bots/cli/cli.py new file mode 100644 index 0000000..2af14dc --- /dev/null +++ b/mobilizon_bots/cli/cli.py @@ -0,0 +1,106 @@ +import functools + +import click +from arrow import Arrow +from click import pass_context, pass_obj + +from mobilizon_bots.cli import safe_execution +from mobilizon_bots.cli.inspect import inspect_events +from mobilizon_bots.cli.main import main +from mobilizon_bots.event.event import EventPublicationStatus + +settings_file_option = click.option("--settings-file", type=click.Path(exists=True)) +from_date_option = click.option("--begin", type=click.DateTime(), expose_value=True) +to_date_option = click.option("--end", type=click.DateTime(), expose_value=True) + + +@click.group() +def mobilizon_bots(): + pass + + +@mobilizon_bots.command() +@settings_file_option +def start(settings_file): + safe_execution(main, settings_file=settings_file) + + +@mobilizon_bots.group() +@from_date_option +@to_date_option +@pass_context +def inspect(ctx, begin, end): + ctx.ensure_object(dict) + ctx.obj["begin"] = Arrow.fromdatetime(begin) if begin else None + ctx.obj["end"] = Arrow.fromdatetime(end) if end else None + pass + + +@inspect.command() +@settings_file_option +def all(settings_file): + safe_execution(inspect_events, settings_file) + + +@inspect.command() +@settings_file_option +@pass_obj +def waiting(obj, settings_file): + safe_execution( + functools.partial( + inspect_events, + EventPublicationStatus.WAITING, + frm=obj["begin"], + to=obj["end"], + ), + settings_file, + ) + + +@inspect.command() +@settings_file_option +@pass_obj +def failed(obj, settings_file): + safe_execution( + functools.partial( + inspect_events, + EventPublicationStatus.FAILED, + frm=obj["begin"], + to=obj["end"], + ), + settings_file, + ) + + +@inspect.command() +@settings_file_option +@pass_obj +def partial(obj, settings_file): + safe_execution( + functools.partial( + inspect_events, + EventPublicationStatus.PARTIAL, + frm=obj["begin"], + to=obj["end"], + ), + settings_file, + ) + + +@inspect.command() +@settings_file_option +@pass_obj +def completed(obj, settings_file): + safe_execution( + functools.partial( + inspect_events, + EventPublicationStatus.COMPLETED, + frm=obj["begin"], + to=obj["end"], + ), + settings_file, + ) + + +if __name__ == "__main__": + mobilizon_bots() diff --git a/mobilizon_bots/cli/inspect.py b/mobilizon_bots/cli/inspect.py new file mode 100644 index 0000000..27998ba --- /dev/null +++ b/mobilizon_bots/cli/inspect.py @@ -0,0 +1,44 @@ +from typing import Iterable + +import click +from arrow import Arrow + +from mobilizon_bots.event.event import EventPublicationStatus +from mobilizon_bots.event.event import MobilizonEvent +from mobilizon_bots.storage.query import get_all_events +from mobilizon_bots.storage.query import events_with_status + + +status_to_color = { + EventPublicationStatus.COMPLETED: "green", + EventPublicationStatus.FAILED: "red", + EventPublicationStatus.PARTIAL: "yellow", + EventPublicationStatus.WAITING: "white", +} + + +def show_events(events: Iterable[MobilizonEvent]): + click.echo_via_pager("\n".join(map(pretty, events))) + + +def pretty(event: MobilizonEvent): + return ( + f"{event.name}|{click.style(event.status.name, fg=status_to_color[event.status])}" + f"|{event.mobilizon_id}|{event.publication_time['telegram'].isoformat()}" + ) + + +async def inspect_events( + status: EventPublicationStatus = None, frm: Arrow = None, to: Arrow = None +): + + events = ( + await events_with_status([status], from_date=frm, to_date=to) + if status + else await get_all_events(from_date=frm, to_date=to) + ) + + if events: + show_events(events) + else: + click.echo(f"No event found with status: {status}") diff --git a/mobilizon_bots/main.py b/mobilizon_bots/cli/main.py similarity index 78% rename from mobilizon_bots/main.py rename to mobilizon_bots/cli/main.py index 1d7c9d9..e1d52e6 100644 --- a/mobilizon_bots/main.py +++ b/mobilizon_bots/cli/main.py @@ -1,12 +1,10 @@ import logging.config -from pathlib import Path -from mobilizon_bots.config.config import update_settings_files from mobilizon_bots.event.event_selection_strategies import select_event_to_publish from mobilizon_bots.mobilizon.events import get_unpublished_events from mobilizon_bots.publishers import get_active_publishers from mobilizon_bots.publishers.coordinator import PublisherCoordinator -from mobilizon_bots.storage.db import MobilizonBotsDB, tear_down +from mobilizon_bots.storage.db import tear_down from mobilizon_bots.storage.query import get_published_events, create_unpublished_events logger = logging.getLogger(__name__) @@ -17,19 +15,14 @@ async def graceful_exit(code): exit(code) -async def main(settings_file): +async def main(): """ STUB :return: """ - settings = update_settings_files(settings_file) - logging.config.dictConfig(settings["logging"]) active_publishers = get_active_publishers() - db = MobilizonBotsDB(Path(settings.db_path)) - await db.setup() - # Load past events published_events = list(await get_published_events()) diff --git a/mobilizon_bots/config/config.py b/mobilizon_bots/config/config.py index e3048bd..7cca645 100644 --- a/mobilizon_bots/config/config.py +++ b/mobilizon_bots/config/config.py @@ -103,7 +103,6 @@ class CustomConfig: def __new__(cls, settings_files: List[str] = None): if cls._instance is None: - print("Creating the object") cls._instance = super(CustomConfig, cls).__new__(cls) cls.settings = build_and_validate_settings(settings_files) return cls._instance diff --git a/mobilizon_bots/storage/query.py b/mobilizon_bots/storage/query.py index e13753d..fef5c35 100644 --- a/mobilizon_bots/storage/query.py +++ b/mobilizon_bots/storage/query.py @@ -1,11 +1,11 @@ +from typing import Iterable, Optional, List + import sys - -from typing import Iterable, Optional - +from arrow import Arrow from tortoise.queryset import QuerySet from tortoise.transactions import atomic -from mobilizon_bots.event.event import MobilizonEvent +from mobilizon_bots.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_bots.models.event import Event from mobilizon_bots.models.publication import Publication, PublicationStatus from mobilizon_bots.models.publisher import Publisher @@ -18,7 +18,7 @@ from mobilizon_bots.publishers.coordinator import PublisherCoordinatorReport CONNECTION_NAME = "models" if "pytest" in sys.modules else None -async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]: +async def prefetch_event_relations(queryset: QuerySet[Event]) -> List[Event]: return ( await queryset.prefetch_related("publications__publisher") .order_by("begin_datetime") @@ -26,26 +26,54 @@ async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]: ) +def _add_date_window( + query, from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, +): + + if from_date: + query = query.filter(end_datetime__gt=from_date.datetime) + if to_date: + query = query.filter(end_datetime__lt=to_date.datetime) + return query + + async def events_with_status( - statuses: list[PublicationStatus], + status: List[EventPublicationStatus], + from_date: Optional[Arrow] = None, + to_date: Optional[Arrow] = None, ) -> Iterable[MobilizonEvent]: + def _filter_event_with_status(event: Event) -> bool: + # This computes the status client-side instead of running in the DB. It shouldn't pose a performance problem + # in the short term, but should be moved to the query if possible. + event_status = MobilizonEvent.compute_status(list(event.publications)) + return event_status in status + + query = Event.all() + _add_date_window(query, from_date, to_date) return map( MobilizonEvent.from_model, - await prefetch_event_relations(Event.filter(publications__status__in=statuses)), + filter(_filter_event_with_status, await prefetch_event_relations(query),), ) -async def get_published_events() -> Iterable[MobilizonEvent]: +async def get_all_events( + from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, +) -> Iterable[MobilizonEvent]: + return map( MobilizonEvent.from_model, await prefetch_event_relations( - Event.filter(publications__status=PublicationStatus.COMPLETED) + _add_date_window(Event.all(), from_date, to_date) ), ) +async def get_published_events() -> Iterable[MobilizonEvent]: + return await events_with_status([EventPublicationStatus.COMPLETED]) + + async def get_unpublished_events() -> Iterable[MobilizonEvent]: - return await events_with_status([PublicationStatus.WAITING]) + return await events_with_status([EventPublicationStatus.WAITING]) async def save_event(event): @@ -59,9 +87,7 @@ async def save_publication(publisher_name, event_model, status: PublicationStatu publisher = await Publisher.filter(name=publisher_name).first() await Publication.create( - status=status, - event_id=event_model.id, - publisher_id=publisher.id, + status=status, event_id=event_model.id, publisher_id=publisher.id, ) diff --git a/pyproject.toml b/pyproject.toml index b068b81..7b14382 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,4 +27,4 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -mobilizon-bots="mobilizon_bots.cli:mobilizon_bots" +mobilizon-bots="mobilizon_bots.cli.cli:mobilizon_bots" diff --git a/tests/storage/test_query.py b/tests/storage/test_query.py index 2be6c5b..a322838 100644 --- a/tests/storage/test_query.py +++ b/tests/storage/test_query.py @@ -34,9 +34,11 @@ def setup(): event_1 = event_model_generator(begin_date=today) event_2 = event_model_generator(idx=2, begin_date=today + timedelta(days=2)) event_3 = event_model_generator(idx=3, begin_date=today + timedelta(days=-2)) + event_4 = event_model_generator(idx=4, begin_date=today + timedelta(days=-4)) await event_1.save() await event_2.save() await event_3.save() + await event_4.save() publication_1 = publication_model_generator( event_id=event_1.id, publisher_id=publisher_1.id @@ -56,13 +58,19 @@ def setup(): publisher_id=publisher_2.id, status=PublicationStatus.WAITING, ) + publication_5 = publication_model_generator( + event_id=event_4.id, + publisher_id=publisher_2.id, + status=PublicationStatus.COMPLETED, + ) await publication_1.save() await publication_2.save() await publication_3.save() await publication_4.save() + await publication_5.save() return ( - [event_1, event_2, event_3], - [publication_1, publication_2, publication_3, publication_4], + [event_1, event_2, event_3, event_4], + [publication_1, publication_2, publication_3, publication_4, publication_5], [publisher_1, publisher_2], today, ) @@ -81,9 +89,9 @@ async def test_get_published_events( published_events = list(await get_published_events()) assert len(published_events) == 1 - assert published_events[0].mobilizon_id == events[0].mobilizon_id + assert published_events[0].mobilizon_id == events[3].mobilizon_id - assert published_events[0].begin_datetime == arrow.get(today) + assert published_events[0].begin_datetime == arrow.get(today + timedelta(days=-4)) @pytest.mark.asyncio @@ -95,12 +103,10 @@ async def test_get_unpublished_events( ) unpublished_events = list(await get_unpublished_events()) - assert len(unpublished_events) == 2 + assert len(unpublished_events) == 1 assert unpublished_events[0].mobilizon_id == events[2].mobilizon_id - assert unpublished_events[1].mobilizon_id == events[0].mobilizon_id assert unpublished_events[0].begin_datetime == events[2].begin_datetime - assert unpublished_events[1].begin_datetime == events[0].begin_datetime @pytest.mark.asyncio