Mobilizon
/
Mobilizon-Reshare-condividi-sui-social
mirror of https://github.com/Tech-Workers-Coalition-Italia/mobilizon-reshare.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

extend cli (#48) 

* fixed teardown

* Updated statuses management, tests (#41)

* Updated statuses management, tests

* storage: query: Generalize event loading logic.

* reformat

* storage: query: Rename load_events to prefetch_event_relations.

Co-authored-by: Giacomo Leidi <goodoldpaul@autistici.org>

* fixed test_window_no_event

* added strategy tests

* removed unused code

* added config tests

* added more config tests

* refactored tests

* updated pytest

* added inspect_all

* temp

* added colors

* storage: events_with_status: Use `EventPublicationStatus`.

* storage: events_with_status: Enable closed ranges.

* added time window to CLI

Co-authored-by: SlyK182 <60148777+SlyK182@users.noreply.github.com>
Co-authored-by: Giacomo Leidi <goodoldpaul@autistici.org>
This commit is contained in:
Simone Robutti 2021-08-04 18:53:58 +02:00 committed by GitHub
parent 929e3aa78e
commit 880a34115f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 244 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
mobilizon_bots/cli.py
View File

@ -1,21 +0,0 @@
import asyncio
import click
from mobilizon_bots.main import main
@click.group()
def mobilizon_bots():
pass
@mobilizon_bots.command()
@click.option("--settings-file", type=click.Path(exists=True))
def start(settings_file):
asyncio.run(main([settings_file] if settings_file else None))
if __name__ == "__main__":
mobilizon_bots()

39
mobilizon_bots/cli/__init__.py Normal file
View File

@ -0,0 +1,39 @@
import asyncio
import logging
import traceback
from logging.config import dictConfig
from pathlib import Path
from mobilizon_bots.config.config import update_settings_files
from mobilizon_bots.storage.db import tear_down, MobilizonBotsDB
logger = logging.getLogger(__name__)
async def graceful_exit(code):
await tear_down()
exit(code)
async def init(settings_file):
settings = update_settings_files(settings_file)
dictConfig(settings["logging"])
db_path = Path(settings.db_path)
db = MobilizonBotsDB(db_path)
await db.setup()
async def _safe_execution(f, settings_file):
await init(settings_file)
return_code = 1
try:
return_code = await f()
except Exception:
traceback.print_exc()
finally:
logger.debug("Closing")
await graceful_exit(return_code)
def safe_execution(f, settings_file):
asyncio.run(_safe_execution(f, settings_file))

106
mobilizon_bots/cli/cli.py Normal file
View File

@ -0,0 +1,106 @@
import functools
import click
from arrow import Arrow
from click import pass_context, pass_obj
from mobilizon_bots.cli import safe_execution
from mobilizon_bots.cli.inspect import inspect_events
from mobilizon_bots.cli.main import main
from mobilizon_bots.event.event import EventPublicationStatus
settings_file_option = click.option("--settings-file", type=click.Path(exists=True))
from_date_option = click.option("--begin", type=click.DateTime(), expose_value=True)
to_date_option = click.option("--end", type=click.DateTime(), expose_value=True)
@click.group()
def mobilizon_bots():
pass
@mobilizon_bots.command()
@settings_file_option
def start(settings_file):
safe_execution(main, settings_file=settings_file)
@mobilizon_bots.group()
@from_date_option
@to_date_option
@pass_context
def inspect(ctx, begin, end):
ctx.ensure_object(dict)
ctx.obj["begin"] = Arrow.fromdatetime(begin) if begin else None
ctx.obj["end"] = Arrow.fromdatetime(end) if end else None
pass
@inspect.command()
@settings_file_option
def all(settings_file):
safe_execution(inspect_events, settings_file)
@inspect.command()
@settings_file_option
@pass_obj
def waiting(obj, settings_file):
safe_execution(
functools.partial(
inspect_events,
EventPublicationStatus.WAITING,
frm=obj["begin"],
to=obj["end"],
),
settings_file,
)
@inspect.command()
@settings_file_option
@pass_obj
def failed(obj, settings_file):
safe_execution(
functools.partial(
inspect_events,
EventPublicationStatus.FAILED,
frm=obj["begin"],
to=obj["end"],
),
settings_file,
)
@inspect.command()
@settings_file_option
@pass_obj
def partial(obj, settings_file):
safe_execution(
functools.partial(
inspect_events,
EventPublicationStatus.PARTIAL,
frm=obj["begin"],
to=obj["end"],
),
settings_file,
)
@inspect.command()
@settings_file_option
@pass_obj
def completed(obj, settings_file):
safe_execution(
functools.partial(
inspect_events,
EventPublicationStatus.COMPLETED,
frm=obj["begin"],
to=obj["end"],
),
settings_file,
)
if __name__ == "__main__":
mobilizon_bots()

44
mobilizon_bots/cli/inspect.py Normal file
View File

@ -0,0 +1,44 @@
from typing import Iterable
import click
from arrow import Arrow
from mobilizon_bots.event.event import EventPublicationStatus
from mobilizon_bots.event.event import MobilizonEvent
from mobilizon_bots.storage.query import get_all_events
from mobilizon_bots.storage.query import events_with_status
status_to_color = {
EventPublicationStatus.COMPLETED: "green",
EventPublicationStatus.FAILED: "red",
EventPublicationStatus.PARTIAL: "yellow",
EventPublicationStatus.WAITING: "white",
}
def show_events(events: Iterable[MobilizonEvent]):
click.echo_via_pager("\n".join(map(pretty, events)))
def pretty(event: MobilizonEvent):
return (
f"{event.name}|{click.style(event.status.name, fg=status_to_color[event.status])}"
f"|{event.mobilizon_id}|{event.publication_time['telegram'].isoformat()}"
)
async def inspect_events(
status: EventPublicationStatus = None, frm: Arrow = None, to: Arrow = None
):
events = (
await events_with_status([status], from_date=frm, to_date=to)
if status
else await get_all_events(from_date=frm, to_date=to)
)
if events:
show_events(events)
else:
click.echo(f"No event found with status: {status}")

11
mobilizon_bots/main.py → mobilizon_bots/cli/main.py
View File

@ -1,12 +1,10 @@
import logging.config
from pathlib import Path
from mobilizon_bots.config.config import update_settings_files
from mobilizon_bots.event.event_selection_strategies import select_event_to_publish
from mobilizon_bots.mobilizon.events import get_unpublished_events
from mobilizon_bots.publishers import get_active_publishers
from mobilizon_bots.publishers.coordinator import PublisherCoordinator
from mobilizon_bots.storage.db import MobilizonBotsDB, tear_down
from mobilizon_bots.storage.db import tear_down
from mobilizon_bots.storage.query import get_published_events, create_unpublished_events
logger = logging.getLogger(__name__)
@ -17,19 +15,14 @@ async def graceful_exit(code):
exit(code)
async def main(settings_file):
async def main():
"""
STUB
:return:
"""
settings = update_settings_files(settings_file)
logging.config.dictConfig(settings["logging"])
active_publishers = get_active_publishers()
db = MobilizonBotsDB(Path(settings.db_path))
await db.setup()
# Load past events
published_events = list(await get_published_events())

1
mobilizon_bots/config/config.py
View File

@ -103,7 +103,6 @@ class CustomConfig:
def __new__(cls, settings_files: List[str] = None):
if cls._instance is None:
print("Creating the object")
cls._instance = super(CustomConfig, cls).__new__(cls)
cls.settings = build_and_validate_settings(settings_files)
return cls._instance

52
mobilizon_bots/storage/query.py
View File

@ -1,11 +1,11 @@
from typing import Iterable, Optional, List
import sys
from typing import Iterable, Optional
from arrow import Arrow
from tortoise.queryset import QuerySet
from tortoise.transactions import atomic
from mobilizon_bots.event.event import MobilizonEvent
from mobilizon_bots.event.event import MobilizonEvent, EventPublicationStatus
from mobilizon_bots.models.event import Event
from mobilizon_bots.models.publication import Publication, PublicationStatus
from mobilizon_bots.models.publisher import Publisher
@ -18,7 +18,7 @@ from mobilizon_bots.publishers.coordinator import PublisherCoordinatorReport
CONNECTION_NAME = "models" if "pytest" in sys.modules else None
async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]:
async def prefetch_event_relations(queryset: QuerySet[Event]) -> List[Event]:
return (
await queryset.prefetch_related("publications__publisher")
.order_by("begin_datetime")
@ -26,26 +26,54 @@ async def prefetch_event_relations(queryset: QuerySet[Event]) -> list[Event]:
)
def _add_date_window(
query, from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
):
if from_date:
query = query.filter(end_datetime__gt=from_date.datetime)
if to_date:
query = query.filter(end_datetime__lt=to_date.datetime)
return query
async def events_with_status(
statuses: list[PublicationStatus],
status: List[EventPublicationStatus],
from_date: Optional[Arrow] = None,
to_date: Optional[Arrow] = None,
) -> Iterable[MobilizonEvent]:
def _filter_event_with_status(event: Event) -> bool:
# This computes the status client-side instead of running in the DB. It shouldn't pose a performance problem
# in the short term, but should be moved to the query if possible.
event_status = MobilizonEvent.compute_status(list(event.publications))
return event_status in status
query = Event.all()
_add_date_window(query, from_date, to_date)
return map(
MobilizonEvent.from_model,
await prefetch_event_relations(Event.filter(publications__status__in=statuses)),
filter(_filter_event_with_status, await prefetch_event_relations(query),),
)
async def get_published_events() -> Iterable[MobilizonEvent]:
async def get_all_events(
from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
) -> Iterable[MobilizonEvent]:
return map(
MobilizonEvent.from_model,
await prefetch_event_relations(
Event.filter(publications__status=PublicationStatus.COMPLETED)
_add_date_window(Event.all(), from_date, to_date)
),
)
async def get_published_events() -> Iterable[MobilizonEvent]:
return await events_with_status([EventPublicationStatus.COMPLETED])
async def get_unpublished_events() -> Iterable[MobilizonEvent]:
return await events_with_status([PublicationStatus.WAITING])
return await events_with_status([EventPublicationStatus.WAITING])
async def save_event(event):
@ -59,9 +87,7 @@ async def save_publication(publisher_name, event_model, status: PublicationStatu
publisher = await Publisher.filter(name=publisher_name).first()
await Publication.create(
status=status,
event_id=event_model.id,
publisher_id=publisher.id,
status=status, event_id=event_model.id, publisher_id=publisher.id,
)

2
pyproject.toml
View File

@ -27,4 +27,4 @@ requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
mobilizon-bots="mobilizon_bots.cli:mobilizon_bots"
mobilizon-bots="mobilizon_bots.cli.cli:mobilizon_bots"

20
tests/storage/test_query.py
View File

@ -34,9 +34,11 @@ def setup():
event_1 = event_model_generator(begin_date=today)
event_2 = event_model_generator(idx=2, begin_date=today + timedelta(days=2))
event_3 = event_model_generator(idx=3, begin_date=today + timedelta(days=-2))
event_4 = event_model_generator(idx=4, begin_date=today + timedelta(days=-4))
await event_1.save()
await event_2.save()
await event_3.save()
await event_4.save()
publication_1 = publication_model_generator(
event_id=event_1.id, publisher_id=publisher_1.id
@ -56,13 +58,19 @@ def setup():
publisher_id=publisher_2.id,
status=PublicationStatus.WAITING,
)
publication_5 = publication_model_generator(
event_id=event_4.id,
publisher_id=publisher_2.id,
status=PublicationStatus.COMPLETED,
)
await publication_1.save()
await publication_2.save()
await publication_3.save()
await publication_4.save()
await publication_5.save()
return (
[event_1, event_2, event_3],
[publication_1, publication_2, publication_3, publication_4],
[event_1, event_2, event_3, event_4],
[publication_1, publication_2, publication_3, publication_4, publication_5],
[publisher_1, publisher_2],
today,
)
@ -81,9 +89,9 @@ async def test_get_published_events(
published_events = list(await get_published_events())
assert len(published_events) == 1
assert published_events[0].mobilizon_id == events[0].mobilizon_id
assert published_events[0].mobilizon_id == events[3].mobilizon_id
assert published_events[0].begin_datetime == arrow.get(today)
assert published_events[0].begin_datetime == arrow.get(today + timedelta(days=-4))
@pytest.mark.asyncio
@ -95,12 +103,10 @@ async def test_get_unpublished_events(
)
unpublished_events = list(await get_unpublished_events())
assert len(unpublished_events) == 2
assert len(unpublished_events) == 1
assert unpublished_events[0].mobilizon_id == events[2].mobilizon_id
assert unpublished_events[1].mobilizon_id == events[0].mobilizon_id
assert unpublished_events[0].begin_datetime == events[2].begin_datetime
assert unpublished_events[1].begin_datetime == events[0].begin_datetime
@pytest.mark.asyncio