extend cli (#48)

* fixed teardown

* Updated statuses management, tests (#41)

* Updated statuses management, tests

* storage: query: Generalize event loading logic.

* reformat

* storage: query: Rename load_events to prefetch_event_relations.

Co-authored-by: Giacomo Leidi <goodoldpaul@autistici.org>

* fixed test_window_no_event

* added strategy tests

* removed unused code

* added config tests

* added more config tests

* refactored tests

* updated pytest

* added inspect_all

* temp

* added colors

* storage: events_with_status: Use `EventPublicationStatus`.

* storage: events_with_status: Enable closed ranges.

* added time window to CLI

Co-authored-by: SlyK182 <60148777+SlyK182@users.noreply.github.com>
Co-authored-by: Giacomo Leidi <goodoldpaul@autistici.org>

mobilizon_bots/cli.py

@@ -1,21 +0,0 @@
-import asyncio
-
-import click
-
-from mobilizon_bots.main import main
-
-
-@click.group()
-def mobilizon_bots():
-    pass
-
-
-@mobilizon_bots.command()
-@click.option("--settings-file", type=click.Path(exists=True))
-def start(settings_file):
-
-    asyncio.run(main([settings_file] if settings_file else None))
-
-
-if __name__ == "__main__":
-    mobilizon_bots()

mobilizon_bots/cli/__init__.py

@@ -0,0 +1,39 @@
+import asyncio
+import logging
+import traceback
+from logging.config import dictConfig
+from pathlib import Path
+
+from mobilizon_bots.config.config import update_settings_files
+from mobilizon_bots.storage.db import tear_down, MobilizonBotsDB
+
+logger = logging.getLogger(__name__)
+
+
+async def graceful_exit(code):
+    await tear_down()
+    exit(code)
+
+
+async def init(settings_file):
+    settings = update_settings_files(settings_file)
+    dictConfig(settings["logging"])
+    db_path = Path(settings.db_path)
+    db = MobilizonBotsDB(db_path)
+    await db.setup()
+
+
+async def _safe_execution(f, settings_file):
+    await init(settings_file)
+    return_code = 1
+    try:
+        return_code = await f()
+    except Exception:
+        traceback.print_exc()
+    finally:
+        logger.debug("Closing")
+        await graceful_exit(return_code)
+
+
+def safe_execution(f, settings_file):
+    asyncio.run(_safe_execution(f, settings_file))

mobilizon_bots/cli/cli.py

@@ -0,0 +1,106 @@
+import functools
+
+import click
+from arrow import Arrow
+from click import pass_context, pass_obj
+
+from mobilizon_bots.cli import safe_execution
+from mobilizon_bots.cli.inspect import inspect_events
+from mobilizon_bots.cli.main import main
+from mobilizon_bots.event.event import EventPublicationStatus
+
+settings_file_option = click.option("--settings-file", type=click.Path(exists=True))
+from_date_option = click.option("--begin", type=click.DateTime(), expose_value=True)
+to_date_option = click.option("--end", type=click.DateTime(), expose_value=True)
+
+
+@click.group()
+def mobilizon_bots():
+    pass
+
+
+@mobilizon_bots.command()
+@settings_file_option
+def start(settings_file):
+    safe_execution(main, settings_file=settings_file)
+
+
+@mobilizon_bots.group()
+@from_date_option
+@to_date_option
+@pass_context
+def inspect(ctx, begin, end):
+    ctx.ensure_object(dict)
+    ctx.obj["begin"] = Arrow.fromdatetime(begin) if begin else None
+    ctx.obj["end"] = Arrow.fromdatetime(end) if end else None
+    pass
+
+
+@inspect.command()
+@settings_file_option
+def all(settings_file):
+    safe_execution(inspect_events, settings_file)
+
+
+@inspect.command()
+@settings_file_option
+@pass_obj
+def waiting(obj, settings_file):
+    safe_execution(
+        functools.partial(
+            inspect_events,
+            EventPublicationStatus.WAITING,
+            frm=obj["begin"],
+            to=obj["end"],
+        ),
+        settings_file,
+    )
+
+
+@inspect.command()
+@settings_file_option
+@pass_obj
+def failed(obj, settings_file):
+    safe_execution(
+        functools.partial(
+            inspect_events,
+            EventPublicationStatus.FAILED,
+            frm=obj["begin"],
+            to=obj["end"],
+        ),
+        settings_file,
+    )
+
+
+@inspect.command()
+@settings_file_option
+@pass_obj
+def partial(obj, settings_file):
+    safe_execution(
+        functools.partial(
+            inspect_events,
+            EventPublicationStatus.PARTIAL,
+            frm=obj["begin"],
+            to=obj["end"],
+        ),
+        settings_file,
+    )
+
+
+@inspect.command()
+@settings_file_option
+@pass_obj
+def completed(obj, settings_file):
+    safe_execution(
+        functools.partial(
+            inspect_events,
+            EventPublicationStatus.COMPLETED,
+            frm=obj["begin"],
+            to=obj["end"],
+        ),
+        settings_file,
+    )
+
+
+if __name__ == "__main__":
+    mobilizon_bots()

mobilizon_bots/cli/inspect.py

@@ -0,0 +1,44 @@
+from typing import Iterable
+
+import click
+from arrow import Arrow
+
+from mobilizon_bots.event.event import EventPublicationStatus
+from mobilizon_bots.event.event import MobilizonEvent
+from mobilizon_bots.storage.query import get_all_events
+from mobilizon_bots.storage.query import events_with_status
+
+
+status_to_color = {
+    EventPublicationStatus.COMPLETED: "green",
+    EventPublicationStatus.FAILED: "red",
+    EventPublicationStatus.PARTIAL: "yellow",
+    EventPublicationStatus.WAITING: "white",
+}
+
+
+def show_events(events: Iterable[MobilizonEvent]):
+    click.echo_via_pager("\n".join(map(pretty, events)))
+
+
+def pretty(event: MobilizonEvent):
+    return (
+        f"{event.name}|{click.style(event.status.name, fg=status_to_color[event.status])}"
+        f"|{event.mobilizon_id}|{event.publication_time['telegram'].isoformat()}"
+    )
+
+
+async def inspect_events(
+    status: EventPublicationStatus = None, frm: Arrow = None, to: Arrow = None
+):
+
+    events = (
+        await events_with_status([status], from_date=frm, to_date=to)
+        if status
+        else await get_all_events(from_date=frm, to_date=to)
+    )
+
+    if events:
+        show_events(events)
+    else:
+        click.echo(f"No event found with status: {status}")

mobilizon_bots/cli/main.py

@@ -1,12 +1,10 @@
 import logging.config
-from pathlib import Path
 
-from mobilizon_bots.config.config import update_settings_files
 from mobilizon_bots.event.event_selection_strategies import select_event_to_publish
 from mobilizon_bots.mobilizon.events import get_unpublished_events
 from mobilizon_bots.publishers import get_active_publishers
 from mobilizon_bots.publishers.coordinator import PublisherCoordinator
-from mobilizon_bots.storage.db import MobilizonBotsDB, tear_down
+from mobilizon_bots.storage.db import tear_down
 from mobilizon_bots.storage.query import get_published_events, create_unpublished_events
 
 logger = logging.getLogger(__name__)
@@ -17,19 +15,14 @@ async def graceful_exit(code):
     exit(code)
 
 
-async def main(settings_file):
+async def main():
     """
     STUB
     :return:
     """
-    settings = update_settings_files(settings_file)
 
-    logging.config.dictConfig(settings["logging"])
     active_publishers = get_active_publishers()
 
-    db = MobilizonBotsDB(Path(settings.db_path))
-    await db.setup()
-
     # Load past events
     published_events = list(await get_published_events())
 

mobilizon_bots/config/config.py

@@ -103,7 +103,6 @@ class CustomConfig:
 
     def __new__(cls, settings_files: List[str] = None):
         if cls._instance is None:
-            print("Creating the object")
             cls._instance = super(CustomConfig, cls).__new__(cls)
             cls.settings = build_and_validate_settings(settings_files)
         return cls._instance

mobilizon_bots/storage/query.py

@@ -1,11 +1,11 @@
+from typing import Iterable, Optional, List
+
 import sys
-
+from arrow import Arrow
-from typing import Iterable, Optional
-
 from tortoise.queryset import QuerySet
 from tortoise.transactions import atomic
 
-from mobilizon_bots.event.event import MobilizonEvent
+from mobilizon_bots.event.event import MobilizonEvent, EventPublicationStatus
 from mobilizon_bots.models.event import Event
 from mobilizon_bots.models.publication import Publication, PublicationStatus
 from mobilizon_bots.models.publisher import Publisher
@@ -18,7 +18,7 @@ from mobilizon_bots.publishers.coordinator import PublisherCoordinatorReport
 CONNECTION_NAME = "models" if "pytest" in sys.modules else None
 
 
-async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]:
+async def prefetch_event_relations(queryset: QuerySet[Event]) -> List[Event]:
     return (
         await queryset.prefetch_related("publications__publisher")
         .order_by("begin_datetime")
@@ -26,26 +26,54 @@ async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]:
     )
 
 
+def _add_date_window(
+    query, from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
+):
+
+    if from_date:
+        query = query.filter(end_datetime__gt=from_date.datetime)
+    if to_date:
+        query = query.filter(end_datetime__lt=to_date.datetime)
+    return query
+
+
 async def events_with_status(
-    statuses: list[PublicationStatus],
+    status: List[EventPublicationStatus],
+    from_date: Optional[Arrow] = None,
+    to_date: Optional[Arrow] = None,
 ) -> Iterable[MobilizonEvent]:
+    def _filter_event_with_status(event: Event) -> bool:
+        # This computes the status client-side instead of running in the DB. It shouldn't pose a performance problem
+        # in the short term, but should be moved to the query if possible.
+        event_status = MobilizonEvent.compute_status(list(event.publications))
+        return event_status in status
+
+    query = Event.all()
+    _add_date_window(query, from_date, to_date)
     return map(
         MobilizonEvent.from_model,
-        await prefetch_event_relations(Event.filter(publications__status__in=statuses)),
+        filter(_filter_event_with_status, await prefetch_event_relations(query),),
     )
 
 
-async def get_published_events() -> Iterable[MobilizonEvent]:
+async def get_all_events(
+    from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
+) -> Iterable[MobilizonEvent]:
+
     return map(
         MobilizonEvent.from_model,
         await prefetch_event_relations(
-            Event.filter(publications__status=PublicationStatus.COMPLETED)
+            _add_date_window(Event.all(), from_date, to_date)
         ),
     )
 
 
+async def get_published_events() -> Iterable[MobilizonEvent]:
+    return await events_with_status([EventPublicationStatus.COMPLETED])
+
+
 async def get_unpublished_events() -> Iterable[MobilizonEvent]:
-    return await events_with_status([PublicationStatus.WAITING])
+    return await events_with_status([EventPublicationStatus.WAITING])
 
 
 async def save_event(event):
@@ -59,9 +87,7 @@ async def save_publication(publisher_name, event_model, status: PublicationStatu
 
     publisher = await Publisher.filter(name=publisher_name).first()
     await Publication.create(
-        status=status,
+        status=status, event_id=event_model.id, publisher_id=publisher.id,
-        event_id=event_model.id,
-        publisher_id=publisher.id,
     )
 
 

pyproject.toml

@@ -27,4 +27,4 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
-mobilizon-bots="mobilizon_bots.cli:mobilizon_bots"
+mobilizon-bots="mobilizon_bots.cli.cli:mobilizon_bots"

tests/storage/test_query.py

@@ -34,9 +34,11 @@ def setup():
         event_1 = event_model_generator(begin_date=today)
         event_2 = event_model_generator(idx=2, begin_date=today + timedelta(days=2))
         event_3 = event_model_generator(idx=3, begin_date=today + timedelta(days=-2))
+        event_4 = event_model_generator(idx=4, begin_date=today + timedelta(days=-4))
         await event_1.save()
         await event_2.save()
         await event_3.save()
+        await event_4.save()
 
         publication_1 = publication_model_generator(
             event_id=event_1.id, publisher_id=publisher_1.id
@@ -56,13 +58,19 @@ def setup():
             publisher_id=publisher_2.id,
             status=PublicationStatus.WAITING,
         )
+        publication_5 = publication_model_generator(
+            event_id=event_4.id,
+            publisher_id=publisher_2.id,
+            status=PublicationStatus.COMPLETED,
+        )
         await publication_1.save()
         await publication_2.save()
         await publication_3.save()
         await publication_4.save()
+        await publication_5.save()
         return (
-            [event_1, event_2, event_3],
+            [event_1, event_2, event_3, event_4],
-            [publication_1, publication_2, publication_3, publication_4],
+            [publication_1, publication_2, publication_3, publication_4, publication_5],
             [publisher_1, publisher_2],
             today,
         )
@@ -81,9 +89,9 @@ async def test_get_published_events(
     published_events = list(await get_published_events())
     assert len(published_events) == 1
 
-    assert published_events[0].mobilizon_id == events[0].mobilizon_id
+    assert published_events[0].mobilizon_id == events[3].mobilizon_id
 
-    assert published_events[0].begin_datetime == arrow.get(today)
+    assert published_events[0].begin_datetime == arrow.get(today + timedelta(days=-4))
 
 
 @pytest.mark.asyncio
@@ -95,12 +103,10 @@ async def test_get_unpublished_events(
     )
 
     unpublished_events = list(await get_unpublished_events())
-    assert len(unpublished_events) == 2
+    assert len(unpublished_events) == 1
 
     assert unpublished_events[0].mobilizon_id == events[2].mobilizon_id
-    assert unpublished_events[1].mobilizon_id == events[0].mobilizon_id
     assert unpublished_events[0].begin_datetime == events[2].begin_datetime
-    assert unpublished_events[1].begin_datetime == events[0].begin_datetime
 
 
 @pytest.mark.asyncio