rework maintenance cli (#140)

* added default

* moved object before verb

* added event retry

* added publication_retry

* renamed inspect to list

* fixed retry publication

* fixed retry event and structure

mobilizon_reshare/cli/cli.py

@@ -1,20 +1,18 @@
 import functools
 
 import click
-from arrow import Arrow
 from click import pass_context
 
 from mobilizon_reshare.cli import safe_execution
 from mobilizon_reshare.cli.commands.format.format import format_event
-from mobilizon_reshare.cli.commands.inspect.inspect_event import inspect_events
+from mobilizon_reshare.cli.commands.list.list_event import list_events
-from mobilizon_reshare.cli.commands.inspect.inspect_publication import (
+from mobilizon_reshare.cli.commands.list.list_publication import list_publications
-    inspect_publications,
-)
 from mobilizon_reshare.cli.commands.recap.main import main as recap_main
 from mobilizon_reshare.cli.commands.start.main import main as start_main
 from mobilizon_reshare.config.config import current_version
 from mobilizon_reshare.config.publishers import publisher_names
 from mobilizon_reshare.event.event import EventPublicationStatus
+from mobilizon_reshare.main.retry import retry, retry_publication
 from mobilizon_reshare.models.publication import PublicationStatus
 
 status_name_to_enum = {
@@ -45,21 +43,17 @@ to_date_option = click.option(
     expose_value=True,
     help="Include only events that begin before this datetime.",
 )
-event_status_option = click.option(
+event_status_option = click.argument(
-    "-s",
+    "status",
-    "--status",
     type=click.Choice(list(status_name_to_enum["event"].keys())),
     default="all",
     expose_value=True,
-    help="Include only events with the given STATUS.",
 )
-publication_status_option = click.option(
+publication_status_option = click.argument(
-    "-s",
+    "status",
-    "--status",
     type=click.Choice(list(status_name_to_enum["publication"].keys())),
     default="all",
     expose_value=True,
-    help="Include only publications with the given STATUS.",
 )
 
 
@@ -91,51 +85,45 @@ def recap():
     safe_execution(recap_main,)
 
 
-@mobilizon_reshare.group(help="List objects in the database with different criteria.")
+@mobilizon_reshare.group(help="Operations that pertain to events")
+def event():
+    pass
+
+
+@mobilizon_reshare.group(help="Operations that pertain to publications")
+def publication():
+    pass
+
+
+@event.command(help="Query for events in the database.", name="list")
+@event_status_option
 @from_date_option
 @to_date_option
-@pass_context
+def event_list(status, begin, end):
-def inspect(ctx, begin, end):
-    ctx.ensure_object(dict)
-    ctx.obj["begin"] = Arrow.fromdatetime(begin) if begin else None
-    ctx.obj["end"] = Arrow.fromdatetime(end) if end else None
 
-
-@inspect.command(help="Query for events in the database.")
-@event_status_option
-@pass_context
-def event(
-    ctx, status,
-):
-    ctx.ensure_object(dict)
     safe_execution(
         functools.partial(
-            inspect_events,
+            list_events, status_name_to_enum["event"][status], frm=begin, to=end,
-            status_name_to_enum["event"][status],
-            frm=ctx.obj["begin"],
-            to=ctx.obj["end"],
         ),
     )
 
 
-@inspect.command(help="Query for publications in the database.")
+@publication.command(help="Query for publications in the database.", name="list")
 @publication_status_option
-@pass_context
+@from_date_option
-def publication(
+@to_date_option
-    ctx, status,
+def publication_list(status, begin, end):
-):
-    ctx.ensure_object(dict)
     safe_execution(
         functools.partial(
-            inspect_publications,
+            list_publications,
             status_name_to_enum["publication"][status],
-            frm=ctx.obj["begin"],
+            frm=begin,
-            to=ctx.obj["end"],
+            to=end,
         ),
     )
 
 
-@mobilizon_reshare.command(
+@event.command(
     help="Format and print event with EVENT-ID using the publisher's format named "
     "PUBLISHER."
 )
@@ -147,5 +135,17 @@ def format(
     safe_execution(functools.partial(format_event, event_id, publisher),)
 
 
+@event.command(name="retry", help="Retries all the failed publications")
+@click.argument("event-id", type=click.UUID)
+def event_retry(event_id):
+    safe_execution(functools.partial(retry, event_id),)
+
+
+@publication.command(name="retry", help="Retries a specific publication")
+@click.argument("publication-id", type=click.UUID)
+def publication_retry(publication_id):
+    safe_execution(functools.partial(retry_publication, publication_id),)
+
+
 if __name__ == "__main__":
     mobilizon_reshare(obj={})

mobilizon_reshare/cli/commands/format/format.py

@@ -1,9 +1,8 @@
 import click
 
-from mobilizon_reshare.event.event import MobilizonEvent
 from mobilizon_reshare.models.event import Event
 from mobilizon_reshare.publishers.platforms.platform_mapping import get_formatter_class
-from mobilizon_reshare.storage.query import from_model
+from mobilizon_reshare.storage.query.event_converter import from_model
 
 
 async def format_event(event_id, publisher_name: str):

mobilizon_reshare/cli/commands/list/list_event.py

@@ -32,24 +32,24 @@ def pretty(event: MobilizonEvent):
     )
 
 
-async def inspect_unpublished_events(frm: Arrow = None, to: Arrow = None):
+async def list_unpublished_events(frm: Arrow = None, to: Arrow = None):
     return select_unpublished_events(
         list(await get_published_events(from_date=frm, to_date=to)),
         list(await events_without_publications(from_date=frm, to_date=to)),
     )
 
 
-async def inspect_events(
+async def list_events(
     status: EventPublicationStatus = None, frm: Arrow = None, to: Arrow = None
 ):
     if status is None:
         events = await get_all_events(from_date=frm, to_date=to)
     elif status == EventPublicationStatus.WAITING:
-        events = await inspect_unpublished_events(frm=frm, to=to)
+        events = await list_unpublished_events(frm=frm, to=to)
     else:
         events = await events_with_status([status], from_date=frm, to_date=to)
 
     if events:
         show_events(events)
     else:
-        click.echo(f"No event found with status: {status}")
+        click.echo(f"No event found with status: {status.name}")

mobilizon_reshare/cli/commands/list/list_publication.py

@@ -27,7 +27,7 @@ def pretty(publication: Publication):
     )
 
 
-async def inspect_publications(
+async def list_publications(
     status: PublicationStatus = None, frm: Arrow = None, to: Arrow = None
 ):
     if status is None:
@@ -38,4 +38,4 @@ async def inspect_publications(
     if publications:
         show_publications(list(publications))
     else:
-        click.echo(f"No publication found with status: {status}")
+        click.echo(f"No publication found with status: {status.name}")

mobilizon_reshare/main/retry.py

@@ -6,7 +6,10 @@ from mobilizon_reshare.publishers.coordinator import (
     PublicationFailureNotifiersCoordinator,
 )
 from mobilizon_reshare.storage.query.exceptions import EventNotFound
-from mobilizon_reshare.storage.query.read import get_failed_publications_for_event
+from mobilizon_reshare.storage.query.read import (
+    get_failed_publications_for_event,
+    get_publication,
+)
 from mobilizon_reshare.storage.query.write import save_publication_report
 
 logger = logging.getLogger(__name__)
@@ -23,6 +26,17 @@ async def retry_event_publications(event_id):
     return PublisherCoordinator(failed_publications).run()
 
 
+async def retry_publication(publication_id):
+    # TODO test this function
+    publication = await get_publication(publication_id)
+    if not publication:
+        logger.info(f"Publication {publication_id} not found.")
+        return
+
+    logger.info(f"Publication {publication_id} found.")
+    return PublisherCoordinator([publication]).run()
+
+
 async def retry(mobilizon_event_id: UUID = None):
     if mobilizon_event_id is None:
         raise NotImplementedError(

mobilizon_reshare/publishers/platforms/facebook.py

@@ -2,6 +2,7 @@ from typing import Optional
 
 import facebook
 import pkg_resources
+from facebook import GraphAPIError
 
 from mobilizon_reshare.event.event import MobilizonEvent
 from mobilizon_reshare.publishers.abstract import (
@@ -61,13 +62,16 @@ class FacebookPlatform(AbstractPlatform):
     def validate_credentials(self):
 
         try:
+            self._log_debug("Validating Facebook credentials")
             self._get_api().get_object(id="me", field="name")
-        except Exception:
+        except GraphAPIError:
             self._log_error(
                 "Invalid Facebook credentials. Authentication Failed",
                 raise_error=InvalidCredentials,
             )
 
+            self._log_debug("Facebook credentials are valid")
+
     def _validate_response(self, response):
         pass
 

mobilizon_reshare/storage/query/__init__.py

@@ -1,79 +1,3 @@
 import sys
-from typing import Optional
-from uuid import UUID
-
-import arrow
-import tortoise.timezone
-
-from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus
-from mobilizon_reshare.models.event import Event
-from mobilizon_reshare.models.publication import PublicationStatus, Publication
 
 CONNECTION_NAME = "models" if "pytest" in sys.modules else None
-
-
-def to_model(event: MobilizonEvent, db_id: Optional[UUID] = None) -> Event:
-    kwargs = {
-        "name": event.name,
-        "description": event.description,
-        "mobilizon_id": event.mobilizon_id,
-        "mobilizon_link": event.mobilizon_link,
-        "thumbnail_link": event.thumbnail_link,
-        "location": event.location,
-        "begin_datetime": event.begin_datetime.astimezone(event.begin_datetime.tzinfo),
-        "end_datetime": event.end_datetime.astimezone(event.end_datetime.tzinfo),
-        "last_update_time": event.last_update_time.astimezone(
-            event.last_update_time.tzinfo
-        ),
-    }
-    if db_id is not None:
-        kwargs.update({"id": db_id})
-    return Event(**kwargs)
-
-
-def from_model(event: Event, tz: str = "UTC"):
-    publication_status = compute_status(list(event.publications))
-    publication_time = {}
-
-    for pub in event.publications:
-        if publication_status != EventPublicationStatus.WAITING:
-            assert pub.timestamp is not None
-            publication_time[pub.publisher.name] = arrow.get(
-                tortoise.timezone.localtime(value=pub.timestamp, timezone=tz)
-            ).to("local")
-    return MobilizonEvent(
-        name=event.name,
-        description=event.description,
-        begin_datetime=arrow.get(
-            tortoise.timezone.localtime(value=event.begin_datetime, timezone=tz)
-        ).to("local"),
-        end_datetime=arrow.get(
-            tortoise.timezone.localtime(value=event.end_datetime, timezone=tz)
-        ).to("local"),
-        mobilizon_link=event.mobilizon_link,
-        mobilizon_id=event.mobilizon_id,
-        thumbnail_link=event.thumbnail_link,
-        location=event.location,
-        publication_time=publication_time,
-        status=publication_status,
-        last_update_time=arrow.get(
-            tortoise.timezone.localtime(value=event.last_update_time, timezone=tz)
-        ).to("local"),
-    )
-
-
-def compute_status(publications: list[Publication]) -> EventPublicationStatus:
-    if not publications:
-        return EventPublicationStatus.WAITING
-
-    unique_statuses: set[PublicationStatus] = set(pub.status for pub in publications)
-
-    if unique_statuses == {
-        PublicationStatus.COMPLETED,
-        PublicationStatus.FAILED,
-    }:
-        return EventPublicationStatus.PARTIAL
-    elif len(unique_statuses) == 1:
-        return EventPublicationStatus[unique_statuses.pop().name]
-
-    raise ValueError(f"Illegal combination of PublicationStatus: {unique_statuses}")

mobilizon_reshare/storage/query/event_converter.py

@@ -0,0 +1,77 @@
+from typing import Optional
+from uuid import UUID
+
+import arrow
+import tortoise.timezone
+
+from mobilizon_reshare.event.event import EventPublicationStatus, MobilizonEvent
+from mobilizon_reshare.models.event import Event
+from mobilizon_reshare.models.publication import Publication, PublicationStatus
+
+
+def from_model(event: Event, tz: str = "UTC"):
+
+    publication_status = compute_status(list(event.publications))
+    publication_time = {}
+
+    for pub in event.publications:
+        if publication_status != EventPublicationStatus.WAITING:
+            assert pub.timestamp is not None
+            publication_time[pub.publisher.name] = arrow.get(
+                tortoise.timezone.localtime(value=pub.timestamp, timezone=tz)
+            ).to("local")
+    return MobilizonEvent(
+        name=event.name,
+        description=event.description,
+        begin_datetime=arrow.get(
+            tortoise.timezone.localtime(value=event.begin_datetime, timezone=tz)
+        ).to("local"),
+        end_datetime=arrow.get(
+            tortoise.timezone.localtime(value=event.end_datetime, timezone=tz)
+        ).to("local"),
+        mobilizon_link=event.mobilizon_link,
+        mobilizon_id=event.mobilizon_id,
+        thumbnail_link=event.thumbnail_link,
+        location=event.location,
+        publication_time=publication_time,
+        status=publication_status,
+        last_update_time=arrow.get(
+            tortoise.timezone.localtime(value=event.last_update_time, timezone=tz)
+        ).to("local"),
+    )
+
+
+def to_model(event: MobilizonEvent, db_id: Optional[UUID] = None) -> Event:
+    kwargs = {
+        "name": event.name,
+        "description": event.description,
+        "mobilizon_id": event.mobilizon_id,
+        "mobilizon_link": event.mobilizon_link,
+        "thumbnail_link": event.thumbnail_link,
+        "location": event.location,
+        "begin_datetime": event.begin_datetime.astimezone(event.begin_datetime.tzinfo),
+        "end_datetime": event.end_datetime.astimezone(event.end_datetime.tzinfo),
+        "last_update_time": event.last_update_time.astimezone(
+            event.last_update_time.tzinfo
+        ),
+    }
+    if db_id is not None:
+        kwargs.update({"id": db_id})
+    return Event(**kwargs)
+
+
+def compute_status(publications: list[Publication]) -> EventPublicationStatus:
+    if not publications:
+        return EventPublicationStatus.WAITING
+
+    unique_statuses: set[PublicationStatus] = set(pub.status for pub in publications)
+
+    if unique_statuses == {
+        PublicationStatus.COMPLETED,
+        PublicationStatus.FAILED,
+    }:
+        return EventPublicationStatus.PARTIAL
+    elif len(unique_statuses) == 1:
+        return EventPublicationStatus[unique_statuses.pop().name]
+
+    raise ValueError(f"Illegal combination of PublicationStatus: {unique_statuses}")

mobilizon_reshare/storage/query/read.py

@@ -4,6 +4,7 @@ from typing import Iterable, Optional
 from uuid import UUID
 
 from arrow import Arrow
+from tortoise.exceptions import DoesNotExist
 from tortoise.queryset import QuerySet
 from tortoise.transactions import atomic
 
@@ -13,8 +14,9 @@ from mobilizon_reshare.models.publication import Publication, PublicationStatus
 from mobilizon_reshare.models.publisher import Publisher
 from mobilizon_reshare.publishers import get_active_publishers
 from mobilizon_reshare.publishers.abstract import EventPublication
-from mobilizon_reshare.storage.query.exceptions import EventNotFound, DuplicateEvent
+from mobilizon_reshare.storage.query import CONNECTION_NAME
-from mobilizon_reshare.storage.query import CONNECTION_NAME, from_model, compute_status
+from mobilizon_reshare.storage.query.event_converter import from_model, compute_status
+from mobilizon_reshare.storage.query.exceptions import EventNotFound
 
 
 async def get_published_events(
@@ -89,11 +91,12 @@ async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]:
 async def prefetch_publication_relations(
     queryset: QuerySet[Publication],
 ) -> list[Publication]:
-    return (
+    publication = (
         await queryset.prefetch_related("publisher", "event")
         .order_by("timestamp")
         .distinct()
     )
+    return publication
 
 
 def _add_date_window(
@@ -188,3 +191,18 @@ async def get_failed_publications_for_event(
     return list(
         map(partial(EventPublication.from_orm, event=event), failed_publications)
     )
+
+
+@atomic(CONNECTION_NAME)
+async def get_publication(publication_id):
+    try:
+        publication = await prefetch_publication_relations(
+            Publication.get(id=publication_id).first()
+        )
+        # TODO: this is redundant but there's some prefetch problem otherwise
+        publication.event = await get_event(publication.event.mobilizon_id)
+        return EventPublication.from_orm(
+            publication, event=from_model(publication.event)
+        )
+    except DoesNotExist:
+        return None

mobilizon_reshare/storage/query/write.py

@@ -9,7 +9,8 @@ from mobilizon_reshare.models.event import Event
 from mobilizon_reshare.models.publication import Publication
 from mobilizon_reshare.models.publisher import Publisher
 from mobilizon_reshare.publishers.coordinator import PublisherCoordinatorReport
-from mobilizon_reshare.storage.query import CONNECTION_NAME, to_model
+from mobilizon_reshare.storage.query import CONNECTION_NAME
+from mobilizon_reshare.storage.query.event_converter import to_model
 from mobilizon_reshare.storage.query.read import (
     events_without_publications,
     is_known,
@@ -91,9 +92,7 @@ async def create_unpublished_events(
 
 
 @atomic(CONNECTION_NAME)
-async def update_publishers(
+async def update_publishers(names: Iterable[str],) -> None:
-    names: Iterable[str],
-) -> None:
     names = set(names)
     known_publisher_names = set(p.name for p in await Publisher.all())
     for name in names.difference(known_publisher_names):

tests/commands/test_format.py

@@ -7,7 +7,7 @@ from mobilizon_reshare.publishers.platforms.platform_mapping import (
     get_formatter_class,
     name_to_formatter_class,
 )
-from mobilizon_reshare.storage.query import to_model
+from mobilizon_reshare.storage.query.event_converter import to_model
 
 
 @pytest.mark.timezone_sensitive

tests/commands/test_start.py

@@ -4,7 +4,7 @@ from logging import DEBUG, INFO
 import arrow
 import pytest
 
-from mobilizon_reshare.storage.query import to_model, from_model
+from mobilizon_reshare.storage.query.event_converter import from_model, to_model
 from mobilizon_reshare.storage.query.read import get_all_events
 from tests.commands.conftest import simple_event_element
 from mobilizon_reshare.event.event import EventPublicationStatus
@@ -15,8 +15,7 @@ from mobilizon_reshare.models.publication import PublicationStatus
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "elements",
+    "elements", [[]],
-    [[]],
 )
 async def test_start_no_event(
     mock_mobilizon_success_answer, mobilizon_answer, caplog, elements
@@ -75,10 +74,7 @@ async def test_start_new_event(
             assert p.status == PublicationStatus.COMPLETED
 
         # the derived status for the event should be COMPLETED
-        assert (
+        assert from_model(all_events[0]).status == EventPublicationStatus.COMPLETED
-            from_model(all_events[0]).status
-            == EventPublicationStatus.COMPLETED
-        )
 
 
 @pytest.mark.asyncio
@@ -86,8 +82,7 @@ async def test_start_new_event(
     "publisher_class", [pytest.lazy_fixture("mock_publisher_class")]
 )
 @pytest.mark.parametrize(
-    "elements",
+    "elements", [[]],
-    [[]],
 )
 async def test_start_event_from_db(
     mock_mobilizon_success_answer,
@@ -129,8 +124,7 @@ async def test_start_event_from_db(
     "publisher_class", [pytest.lazy_fixture("mock_publisher_invalid_class")]
 )
 @pytest.mark.parametrize(
-    "elements",
+    "elements", [[]],
-    [[]],
 )
 async def test_start_publisher_failure(
     mock_mobilizon_success_answer,
@@ -208,8 +202,7 @@ def second_event_element():
     "publisher_class", [pytest.lazy_fixture("mock_publisher_class")]
 )
 @pytest.mark.parametrize(
-    "elements",
+    "elements", [[second_event_element()]],
-    [[second_event_element()]],
 )
 async def test_start_second_execution(
     mock_mobilizon_success_answer,

tests/conftest.py

@@ -22,7 +22,7 @@ from mobilizon_reshare.publishers.abstract import (
     AbstractEventFormatter,
 )
 from mobilizon_reshare.publishers.exceptions import PublisherError, InvalidResponse
-from mobilizon_reshare.storage.query import to_model
+from mobilizon_reshare.storage.query.event_converter import to_model
 from mobilizon_reshare.storage.query.write import get_publisher_by_name
 from tests import today
 
@@ -167,9 +167,7 @@ def event_model_generator():
 
 @pytest.fixture()
 def publisher_model_generator():
-    def _publisher_model_generator(
+    def _publisher_model_generator(idx=1,):
-        idx=1,
-    ):
         return Publisher(name=f"publisher_{idx}", account_ref=f"account_ref_{idx}")
 
     return _publisher_model_generator
@@ -310,10 +308,7 @@ def mock_mobilizon_success_answer(mobilizon_answer, mobilizon_url):
     with responses.RequestsMock() as rsps:
 
         rsps.add(
-            responses.POST,
+            responses.POST, mobilizon_url, json=mobilizon_answer, status=200,
-            mobilizon_url,
-            json=mobilizon_answer,
-            status=200,
         )
         yield
 

tests/models/test_event.py

@@ -6,10 +6,13 @@ import pytest
 import tortoise.timezone
 
 from mobilizon_reshare.event.event import EventPublicationStatus
-from mobilizon_reshare.event.event import MobilizonEvent
 from mobilizon_reshare.models.event import Event
 from mobilizon_reshare.models.publication import PublicationStatus
-from mobilizon_reshare.storage.query import to_model, from_model, compute_status
+from mobilizon_reshare.storage.query.event_converter import (
+    from_model,
+    to_model,
+    compute_status,
+)
 
 
 @pytest.mark.asyncio

tests/publishers/test_coordinator.py

@@ -19,7 +19,7 @@ from mobilizon_reshare.publishers.coordinator import (
     PublicationFailureNotifiersCoordinator,
     RecapCoordinator,
 )
-from mobilizon_reshare.storage.query import to_model
+from mobilizon_reshare.storage.query.event_converter import to_model
 from tests import today
 
 
@@ -109,12 +109,8 @@ async def mock_publications(
 
 @pytest.mark.parametrize("num_publications", [2])
 @pytest.mark.asyncio
-async def test_publication_coordinator_run_success(
+async def test_publication_coordinator_run_success(mock_publications,):
-    mock_publications,
+    coordinator = PublisherCoordinator(publications=mock_publications,)
-):
-    coordinator = PublisherCoordinator(
-        publications=mock_publications,
-    )
     report = coordinator.run()
     assert len(report.reports) == 2
     assert report.successful, "\n".join(map(lambda rep: rep.reason, report.reports))

tests/publishers/test_zulip.py

@@ -13,7 +13,7 @@ from mobilizon_reshare.publishers.exceptions import (
     HTTPResponseError,
 )
 from mobilizon_reshare.publishers.platforms.zulip import ZulipFormatter, ZulipPublisher
-from mobilizon_reshare.storage.query import to_model
+from mobilizon_reshare.storage.query.event_converter import to_model
 from mobilizon_reshare.storage.query.read import build_publications
 
 api_uri = "https://zulip.twc-italia.org/api/v1/"
@@ -42,10 +42,7 @@ users_me = {
 def mocked_valid_response():
     with responses.RequestsMock() as rsps:
         rsps.add(
-            responses.GET,
+            responses.GET, api_uri + "users/me", json=users_me, status=200,
-            api_uri + "users/me",
-            json=users_me,
-            status=200,
         )
         rsps.add(
             responses.POST,
@@ -72,10 +69,7 @@ def mocked_credential_error_response():
 def mocked_client_error_response():
     with responses.RequestsMock() as rsps:
         rsps.add(
-            responses.GET,
+            responses.GET, api_uri + "users/me", json=users_me, status=200,
-            api_uri + "users/me",
-            json=users_me,
-            status=200,
         )
         rsps.add(
             responses.POST,

tests/storage/test_read_query.py

@@ -2,10 +2,8 @@ from uuid import UUID
 
 import pytest
 
-from mobilizon_reshare.storage.query import to_model
+from mobilizon_reshare.storage.query.event_converter import to_model
-from mobilizon_reshare.storage.query.read import (
+from mobilizon_reshare.storage.query.read import get_all_events
-    get_all_events,
-)
 
 
 @pytest.mark.asyncio