Query refactoring (#102)

* Rename query modules.

* storage: save_publication_report: Create publications.

* Remove placeholder PublicationStatus.UNSAVED

* Minor fixes.
This commit is contained in:
Giacomo Leidi 2021-11-24 23:58:06 +01:00 committed by GitHub
parent ed8f752fe6
commit 1efa191771
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 392 additions and 311 deletions

View File

@ -6,7 +6,7 @@ from arrow import Arrow
from mobilizon_reshare.event.event import EventPublicationStatus from mobilizon_reshare.event.event import EventPublicationStatus
from mobilizon_reshare.event.event import MobilizonEvent from mobilizon_reshare.event.event import MobilizonEvent
from mobilizon_reshare.event.event_selection_strategies import select_unpublished_events from mobilizon_reshare.event.event_selection_strategies import select_unpublished_events
from mobilizon_reshare.storage.query.read_query import ( from mobilizon_reshare.storage.query.read import (
get_published_events, get_published_events,
events_with_status, events_with_status,
get_all_events, get_all_events,

View File

@ -15,7 +15,7 @@ from mobilizon_reshare.publishers.platforms.platform_mapping import (
get_publisher_class, get_publisher_class,
get_formatter_class, get_formatter_class,
) )
from mobilizon_reshare.storage.query.read_query import events_with_status from mobilizon_reshare.storage.query.read import events_with_status
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -33,7 +33,7 @@ async def recap() -> Optional[BaseCoordinatorReport]:
events_to_recap = await select_events_to_recap() events_to_recap = await select_events_to_recap()
if events_to_recap: if events_to_recap:
logger.debug(f"Found {len(events_to_recap)} events to recap.") logger.info(f"Found {len(events_to_recap)} events to recap.")
recap_publications = [ recap_publications = [
RecapPublication( RecapPublication(
get_publisher_class(publisher)(), get_publisher_class(publisher)(),
@ -49,4 +49,4 @@ async def recap() -> Optional[BaseCoordinatorReport]:
PublicationFailureNotifiersCoordinator(report).notify_failure() PublicationFailureNotifiersCoordinator(report).notify_failure()
return reports return reports
else: else:
logger.debug("Found no events") logger.info("Found no events")

View File

@ -2,16 +2,15 @@ import logging.config
from mobilizon_reshare.event.event_selection_strategies import select_event_to_publish from mobilizon_reshare.event.event_selection_strategies import select_event_to_publish
from mobilizon_reshare.mobilizon.events import get_unpublished_events from mobilizon_reshare.mobilizon.events import get_unpublished_events
from mobilizon_reshare.publishers.abstract import EventPublication
from mobilizon_reshare.publishers.coordinator import ( from mobilizon_reshare.publishers.coordinator import (
PublicationFailureNotifiersCoordinator, PublicationFailureNotifiersCoordinator,
) )
from mobilizon_reshare.publishers.coordinator import PublisherCoordinator from mobilizon_reshare.publishers.coordinator import PublisherCoordinator
from mobilizon_reshare.storage.query.model_creation import ( from mobilizon_reshare.storage.query.read import (
create_event_publication_models, get_published_events,
build_publications,
) )
from mobilizon_reshare.storage.query.read_query import get_published_events from mobilizon_reshare.storage.query.write import (
from mobilizon_reshare.storage.query.save_query import (
create_unpublished_events, create_unpublished_events,
save_publication_report, save_publication_report,
) )
@ -46,11 +45,10 @@ async def start():
if event: if event:
logger.info(f"Event to publish found: {event.name}") logger.info(f"Event to publish found: {event.name}")
models = await create_event_publication_models(event) publications = await build_publications(event)
publications = list(EventPublication.from_orm(m, event) for m in models)
reports = PublisherCoordinator(publications).run() reports = PublisherCoordinator(publications).run()
await save_publication_report(reports, models) await save_publication_report(reports)
for report in reports.reports: for report in reports.reports:
if not report.succesful: if not report.succesful:
PublicationFailureNotifiersCoordinator(report,).notify_failure() PublicationFailureNotifiersCoordinator(report,).notify_failure()

View File

@ -3,7 +3,6 @@ from tortoise.models import Model
from mobilizon_reshare.models.publication import PublicationStatus, Publication from mobilizon_reshare.models.publication import PublicationStatus, Publication
from mobilizon_reshare.models.publisher import Publisher from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers import get_active_publishers
class Event(Model): class Event(Model):
@ -31,19 +30,8 @@ class Event(Model):
class Meta: class Meta:
table = "event" table = "event"
async def build_unsaved_publication_models(self):
result = []
publishers = get_active_publishers()
for publisher in publishers:
result.append(
await self.build_publication_by_publisher_name(
publisher, PublicationStatus.UNSAVED
)
)
return result
async def build_publication_by_publisher_name( async def build_publication_by_publisher_name(
self, publisher_name: str, status: PublicationStatus self, publisher_name: str, status: PublicationStatus = PublicationStatus.FAILED
) -> Publication: ) -> Publication:
publisher = await Publisher.filter(name=publisher_name).first() publisher = await Publisher.filter(name=publisher_name).first()
return Publication( return Publication(

View File

@ -5,9 +5,8 @@ from tortoise.models import Model
class PublicationStatus(IntEnum): class PublicationStatus(IntEnum):
UNSAVED = 0 FAILED = 0
FAILED = 1 COMPLETED = 1
COMPLETED = 2
class Publication(Model): class Publication(Model):

View File

@ -1,12 +1,10 @@
import asyncio
import atexit
import logging import logging
from pathlib import Path from pathlib import Path
from tortoise import Tortoise from tortoise import Tortoise
from mobilizon_reshare.config.publishers import publisher_names from mobilizon_reshare.config.publishers import publisher_names
from mobilizon_reshare.storage.query.save_query import update_publishers from mobilizon_reshare.storage.query.write import update_publishers
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -1,14 +0,0 @@
from tortoise.transactions import atomic
from mobilizon_reshare.event.event import MobilizonEvent
from mobilizon_reshare.models.event import Event
from mobilizon_reshare.models.publication import Publication
from mobilizon_reshare.storage.query import CONNECTION_NAME
from mobilizon_reshare.storage.query.read_query import prefetch_event_relations
@atomic(CONNECTION_NAME)
async def create_event_publication_models(event: MobilizonEvent) -> list[Publication]:
return await (
await prefetch_event_relations(Event.filter(mobilizon_id=event.mobilizon_id))
)[0].build_unsaved_publication_models()

View File

@ -1,4 +1,4 @@
from typing import Iterable, Optional, Dict, List from typing import Iterable, Optional
from uuid import UUID from uuid import UUID
from arrow import Arrow from arrow import Arrow
@ -8,18 +8,11 @@ from tortoise.transactions import atomic
from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus
from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.event import Event
from mobilizon_reshare.models.publication import Publication, PublicationStatus from mobilizon_reshare.models.publication import Publication, PublicationStatus
from mobilizon_reshare.publishers import get_active_publishers
from mobilizon_reshare.publishers.abstract import EventPublication
from mobilizon_reshare.storage.query import CONNECTION_NAME from mobilizon_reshare.storage.query import CONNECTION_NAME
async def get_mobilizon_event_publications(
event: MobilizonEvent,
) -> Iterable[Publication]:
models = await prefetch_event_relations(
Event.filter(mobilizon_id=event.mobilizon_id)
)
return models[0].publications
async def get_published_events( async def get_published_events(
from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None
) -> Iterable[MobilizonEvent]: ) -> Iterable[MobilizonEvent]:
@ -100,7 +93,7 @@ async def publications_with_status(
event_mobilizon_id: Optional[UUID] = None, event_mobilizon_id: Optional[UUID] = None,
from_date: Optional[Arrow] = None, from_date: Optional[Arrow] = None,
to_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
) -> Dict[UUID, Publication]: ) -> dict[UUID, Publication]:
query = Publication.filter(status=status) query = Publication.filter(status=status)
if event_mobilizon_id: if event_mobilizon_id:
@ -118,7 +111,7 @@ async def publications_with_status(
async def events_without_publications( async def events_without_publications(
from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None, from_date: Optional[Arrow] = None, to_date: Optional[Arrow] = None,
) -> List[MobilizonEvent]: ) -> list[MobilizonEvent]:
query = Event.filter(publications__id=None) query = Event.filter(publications__id=None)
events = await prefetch_event_relations( events = await prefetch_event_relations(
_add_date_window(query, "begin_datetime", from_date, to_date) _add_date_window(query, "begin_datetime", from_date, to_date)
@ -126,7 +119,7 @@ async def events_without_publications(
return list(map(MobilizonEvent.from_model, events)) return list(map(MobilizonEvent.from_model, events))
def _remove_duplicated_events(events: List[MobilizonEvent]): def _remove_duplicated_events(events: list[MobilizonEvent]) -> list[MobilizonEvent]:
"""Remove duplicates based on mobilizon_id""" """Remove duplicates based on mobilizon_id"""
result = [] result = []
seen_ids = set() seen_ids = set()
@ -139,7 +132,7 @@ def _remove_duplicated_events(events: List[MobilizonEvent]):
async def get_unpublished_events( async def get_unpublished_events(
unpublished_mobilizon_events: Iterable[MobilizonEvent], unpublished_mobilizon_events: Iterable[MobilizonEvent],
) -> List[MobilizonEvent]: ) -> list[MobilizonEvent]:
""" """
Returns all the unpublished events, removing duplicates that are present both in the DB and in the mobilizon query Returns all the unpublished events, removing duplicates that are present both in the DB and in the mobilizon query
""" """
@ -148,3 +141,13 @@ async def get_unpublished_events(
db_unpublished_events db_unpublished_events
) )
return _remove_duplicated_events(all_unpublished_events) return _remove_duplicated_events(all_unpublished_events)
@atomic(CONNECTION_NAME)
async def build_publications(event: MobilizonEvent) -> list[EventPublication]:
event_model = await Event.filter(mobilizon_id=event.mobilizon_id).first()
models = [
await event_model.build_publication_by_publisher_name(name)
for name in get_active_publishers()
]
return list(EventPublication.from_orm(m, event) for m in models)

View File

@ -1,53 +0,0 @@
import logging
from typing import List, Iterable, Optional
import arrow
from tortoise.transactions import atomic
from mobilizon_reshare.event.event import MobilizonEvent
from mobilizon_reshare.models.publication import Publication
from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers.coordinator import PublisherCoordinatorReport
from mobilizon_reshare.storage.query import CONNECTION_NAME
from mobilizon_reshare.storage.query.read_query import get_unpublished_events
@atomic(CONNECTION_NAME)
async def save_publication_report(
coordinator_report: PublisherCoordinatorReport,
publication_models: List[Publication],
) -> None:
publication_models = {m.id: m for m in publication_models}
for publication_report in coordinator_report.reports:
publication_id = publication_report.publication.id
publication_models[publication_id].status = publication_report.status
publication_models[publication_id].reason = publication_report.reason
publication_models[publication_id].timestamp = arrow.now().datetime
await publication_models[publication_id].save()
@atomic(CONNECTION_NAME)
async def create_unpublished_events(
unpublished_mobilizon_events: Iterable[MobilizonEvent],
) -> List[MobilizonEvent]:
# We store only new events, i.e. events whose mobilizon_id wasn't found in the DB.
unpublished_events = await get_unpublished_events(unpublished_mobilizon_events)
for event in unpublished_events:
await event.to_model().save()
return unpublished_events
async def create_publisher(name: str, account_ref: Optional[str] = None) -> None:
await Publisher.create(name=name, account_ref=account_ref)
@atomic(CONNECTION_NAME)
async def update_publishers(names: Iterable[str],) -> None:
names = set(names)
known_publisher_names = set(p.name for p in await Publisher.all())
for name in names.difference(known_publisher_names):
logging.info(f"Creating {name} publisher")
await create_publisher(name)

View File

@ -0,0 +1,79 @@
import logging
from typing import Iterable, Optional
import arrow
from tortoise.transactions import atomic
from mobilizon_reshare.event.event import MobilizonEvent
from mobilizon_reshare.models.event import Event
from mobilizon_reshare.models.publication import Publication
from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers.coordinator import PublisherCoordinatorReport
from mobilizon_reshare.storage.query import CONNECTION_NAME
from mobilizon_reshare.storage.query.read import events_without_publications
@atomic(CONNECTION_NAME)
async def save_publication_report(
coordinator_report: PublisherCoordinatorReport,
) -> None:
"""
Store a publication process outcome
"""
for publication_report in coordinator_report.reports:
event = await Event.filter(
mobilizon_id=publication_report.publication.event.mobilizon_id
).first()
publisher = await Publisher.filter(
name=publication_report.publication.publisher.name
).first()
await Publication.create(
id=publication_report.publication.id,
event_id=event.id,
publisher_id=publisher.id,
status=publication_report.status,
reason=publication_report.reason,
timestamp=arrow.now().datetime,
)
@atomic(CONNECTION_NAME)
async def create_unpublished_events(
events_from_mobilizon: Iterable[MobilizonEvent],
) -> list[MobilizonEvent]:
"""
Compute the difference between remote and local events and store it.
Returns the unpublished events merged state.
"""
# We store only new events, i.e. events whose mobilizon_id wasn't found in the DB.
unpublished_events = await events_without_publications()
known_event_mobilizon_ids = set(
map(lambda event: event.mobilizon_id, unpublished_events)
)
new_unpublished_events = list(
filter(
lambda event: event.mobilizon_id not in known_event_mobilizon_ids,
events_from_mobilizon,
)
)
for event in new_unpublished_events:
await event.to_model().save()
return await events_without_publications()
async def create_publisher(name: str, account_ref: Optional[str] = None) -> None:
await Publisher.create(name=name, account_ref=account_ref)
@atomic(CONNECTION_NAME)
async def update_publishers(
names: Iterable[str],
) -> None:
names = set(names)
known_publisher_names = set(p.name for p in await Publisher.all())
for name in names.difference(known_publisher_names):
logging.info(f"Creating {name} publisher")
await create_publisher(name)

View File

@ -0,0 +1,10 @@
from datetime import datetime, timezone, timedelta
today = datetime(
year=2021,
month=6,
day=6,
hour=5,
minute=0,
tzinfo=timezone(timedelta(hours=2)),
)

View File

@ -1,13 +1,15 @@
import uuid import uuid
import arrow
import pytest import pytest
from click.testing import CliRunner from click.testing import CliRunner
import mobilizon_reshare.publishers import mobilizon_reshare.publishers
from mobilizon_reshare.models import event import mobilizon_reshare.storage.query.read
from mobilizon_reshare.models.publisher import Publisher from mobilizon_reshare.models.publisher import Publisher
import mobilizon_reshare.main.recap import mobilizon_reshare.main.recap
from mobilizon_reshare.publishers import coordinator from mobilizon_reshare.publishers import coordinator
from tests import today
def simple_event_element(): def simple_event_element():
@ -30,16 +32,24 @@ def mobilizon_answer(elements):
return {"data": {"group": {"organizedEvents": {"elements": elements}}}} return {"data": {"group": {"organizedEvents": {"elements": elements}}}}
@pytest.fixture
async def mock_now(monkeypatch):
def _mock_now():
return arrow.get(today)
monkeypatch.setattr(mobilizon_reshare.main.recap, "now", _mock_now)
return arrow.get(today)
@pytest.fixture @pytest.fixture
async def mock_publisher_config(monkeypatch, publisher_class, mock_formatter_class): async def mock_publisher_config(monkeypatch, publisher_class, mock_formatter_class):
p = Publisher(name="test") # FIXME: This is subtly bound to the name field of publisher_class
p = Publisher(name="mock")
await p.save() await p.save()
p2 = Publisher(name="test2")
await p2.save()
def _mock_active_pub(): def _mock_active_pub():
return ["test", "test2"] return ["mock"]
def _mock_pub_class(name): def _mock_pub_class(name):
return publisher_class return publisher_class
@ -47,7 +57,6 @@ async def mock_publisher_config(monkeypatch, publisher_class, mock_formatter_cla
def _mock_format_class(name): def _mock_format_class(name):
return mock_formatter_class return mock_formatter_class
monkeypatch.setattr(event, "get_active_publishers", _mock_active_pub)
monkeypatch.setattr( monkeypatch.setattr(
mobilizon_reshare.publishers.platforms.platform_mapping, mobilizon_reshare.publishers.platforms.platform_mapping,
"get_publisher_class", "get_publisher_class",
@ -59,14 +68,22 @@ async def mock_publisher_config(monkeypatch, publisher_class, mock_formatter_cla
_mock_format_class, _mock_format_class,
) )
monkeypatch.setattr(
mobilizon_reshare.storage.query.read, "get_active_publishers", _mock_active_pub
)
monkeypatch.setattr( monkeypatch.setattr(
mobilizon_reshare.main.recap, "get_active_publishers", _mock_active_pub mobilizon_reshare.main.recap, "get_active_publishers", _mock_active_pub
) )
monkeypatch.setattr( monkeypatch.setattr(
mobilizon_reshare.main.recap, "get_publisher_class", _mock_pub_class, mobilizon_reshare.main.recap,
"get_publisher_class",
_mock_pub_class,
) )
monkeypatch.setattr( monkeypatch.setattr(
mobilizon_reshare.main.recap, "get_formatter_class", _mock_format_class, mobilizon_reshare.main.recap,
"get_formatter_class",
_mock_format_class,
) )
return p return p
@ -83,7 +100,9 @@ async def mock_notifier_config(monkeypatch, publisher_class, mock_formatter_clas
return mock_formatter_class return mock_formatter_class
monkeypatch.setattr( monkeypatch.setattr(
coordinator, "get_notifier_class", _mock_notifier_class, coordinator,
"get_notifier_class",
_mock_notifier_class,
) )
monkeypatch.setattr( monkeypatch.setattr(
mobilizon_reshare.publishers.platforms.platform_mapping, mobilizon_reshare.publishers.platforms.platform_mapping,

View File

@ -1,14 +1,20 @@
from logging import DEBUG from logging import DEBUG
from uuid import UUID
import arrow
import pytest import pytest
from mobilizon_reshare.main.recap import recap from mobilizon_reshare.main.recap import recap
from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.models.publication import PublicationStatus
from mobilizon_reshare.storage.query.model_creation import (
create_event_publication_models, spec = {
) # We need three events since recap will print only
# future events, but the 0th event happens at today + 0.
"event": 3,
"publications": [
{"event_idx": 1, "publisher_idx": 0, "status": PublicationStatus.COMPLETED},
{"event_idx": 2, "publisher_idx": 0, "status": PublicationStatus.COMPLETED},
],
"publisher": ["zulip"],
}
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -16,19 +22,9 @@ from mobilizon_reshare.storage.query.model_creation import (
) )
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_start_event_from_db( async def test_start_event_from_db(
caplog, mock_publisher_config, message_collector, event_generator, caplog, mock_publisher_config, mock_now, message_collector, generate_models
): ):
for i in range(2): await generate_models(spec)
event = event_generator(
mobilizon_id=UUID(int=i), begin_date=arrow.now().shift(days=2)
)
event_model = event.to_model()
await event_model.save()
publications = await create_event_publication_models(event_model)
for p in publications:
p.status = PublicationStatus.COMPLETED
await p.save()
with caplog.at_level(DEBUG): with caplog.at_level(DEBUG):
# calling the recap command # calling the recap command
@ -39,7 +35,7 @@ async def test_start_event_from_db(
recap_message = """Upcoming recap_message = """Upcoming
test event event_1
test event""" event_2"""
assert message_collector == [recap_message] * 2 # two publishers * 1 recap assert message_collector == [recap_message]

View File

@ -11,7 +11,8 @@ from mobilizon_reshare.models.publication import PublicationStatus
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize( @pytest.mark.parametrize(
"elements", [[]], "elements",
[[]],
) )
async def test_start_no_event( async def test_start_no_event(
mock_mobilizon_success_answer, mobilizon_answer, caplog, elements mock_mobilizon_success_answer, mobilizon_answer, caplog, elements
@ -47,7 +48,6 @@ async def test_start_new_event(
assert "Event to publish found" in caplog.text assert "Event to publish found" in caplog.text
assert message_collector == [ assert message_collector == [
"test event|Some description", "test event|Some description",
"test event|Some description",
] ]
all_events = ( all_events = (
@ -63,7 +63,7 @@ async def test_start_new_event(
# it should create a publication for each publisher # it should create a publication for each publisher
publications = all_events[0].publications publications = all_events[0].publications
assert len(publications) == 2, publications assert len(publications) == 1, publications
# all the other events should have no publication # all the other events should have no publication
for e in all_events[1:]: for e in all_events[1:]:
@ -85,7 +85,8 @@ async def test_start_new_event(
"publisher_class", [pytest.lazy_fixture("mock_publisher_class")] "publisher_class", [pytest.lazy_fixture("mock_publisher_class")]
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"elements", [[]], "elements",
[[]],
) )
@pytest.mark.parametrize("publication_window", [(0, 24)]) @pytest.mark.parametrize("publication_window", [(0, 24)])
async def test_start_event_from_db( async def test_start_event_from_db(
@ -109,16 +110,15 @@ async def test_start_event_from_db(
assert "Event to publish found" in caplog.text assert "Event to publish found" in caplog.text
assert message_collector == [ assert message_collector == [
"test event|description of the event", "test event|description of the event",
"test event|description of the event",
] ]
await event_model.fetch_related("publications", "publications__publisher") await event_model.fetch_related("publications", "publications__publisher")
# it should create a publication for each publisher # it should create a publication for each publisher
publications = event_model.publications publications = event_model.publications
assert len(publications) == 2, publications assert len(publications) == 1, publications
# all the publications for the first event should be saved as COMPLETED # all the publications for the first event should be saved as COMPLETED
for p in publications[1:]: for p in publications:
assert p.status == PublicationStatus.COMPLETED assert p.status == PublicationStatus.COMPLETED
# the derived status for the event should be COMPLETED # the derived status for the event should be COMPLETED
@ -133,7 +133,8 @@ async def test_start_event_from_db(
"publisher_class", [pytest.lazy_fixture("mock_publisher_invalid_class")] "publisher_class", [pytest.lazy_fixture("mock_publisher_invalid_class")]
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"elements", [[]], "elements",
[[]],
) )
@pytest.mark.parametrize("publication_window", [(0, 24)]) @pytest.mark.parametrize("publication_window", [(0, 24)])
async def test_start_publisher_failure( async def test_start_publisher_failure(
@ -159,7 +160,7 @@ async def test_start_publisher_failure(
await event_model.fetch_related("publications", "publications__publisher") await event_model.fetch_related("publications", "publications__publisher")
# it should create a publication for each publisher # it should create a publication for each publisher
publications = event_model.publications publications = event_model.publications
assert len(publications) == 2, publications assert len(publications) == 1, publications
# all the publications for event should be saved as FAILED # all the publications for event should be saved as FAILED
for p in publications: for p in publications:
@ -168,7 +169,7 @@ async def test_start_publisher_failure(
assert "Event to publish found" in caplog.text assert "Event to publish found" in caplog.text
assert message_collector == [ assert message_collector == [
f"Publication {p.id} failed with status: 1." f"Publication {p.id} failed with status: 0."
f"\nReason: credentials error\nPublisher: mock" f"\nReason: credentials error\nPublisher: mock"
for p in publications for p in publications
for _ in range(2) for _ in range(2)

View File

@ -2,6 +2,7 @@ import importlib.resources
import os import os
from collections import UserList from collections import UserList
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Union
from uuid import UUID from uuid import UUID
import arrow import arrow
@ -21,6 +22,7 @@ from mobilizon_reshare.publishers.abstract import (
AbstractEventFormatter, AbstractEventFormatter,
) )
from mobilizon_reshare.publishers.exceptions import PublisherError, InvalidResponse from mobilizon_reshare.publishers.exceptions import PublisherError, InvalidResponse
from tests import today
def generate_publication_status(published): def generate_publication_status(published):
@ -192,6 +194,64 @@ def notification_model_generator():
return _notification_model_generator return _notification_model_generator
async def _generate_publishers(specification):
publishers = []
for i, publisher_name in enumerate(specification["publisher"]):
publisher = Publisher(
id=UUID(int=i), name=publisher_name, account_ref=f"account_ref_{i}"
)
publishers.append(publisher)
await publisher.save()
return publishers
async def _generate_events(specification):
events = []
if "event" in specification.keys():
for i in range(specification["event"]):
begin_date = today + timedelta(days=i)
event = Event(
id=UUID(int=i),
name=f"event_{i}",
description=f"desc_{i}",
mobilizon_id=UUID(int=i),
mobilizon_link=f"moblink_{i}",
thumbnail_link=f"thumblink_{i}",
location=f"loc_{i}",
begin_datetime=begin_date,
end_datetime=begin_date + timedelta(hours=2),
)
events.append(event)
await event.save()
return events
async def _generate_publications(events, publishers, specification):
if "publications" in specification.keys():
for i, publication in enumerate(specification["publications"]):
status = publication.get("status", PublicationStatus.COMPLETED)
timestamp = publication.get("timestamp", today + timedelta(hours=i))
await Publication.create(
id=UUID(int=i),
status=status,
timestamp=timestamp,
event_id=events[publication["event_idx"]].id,
publisher_id=publishers[publication["publisher_idx"]].id,
)
@pytest.fixture(scope="module")
def generate_models():
async def _generate_models(specification: dict[str, Union[int, list]]):
publishers = await _generate_publishers(specification)
events = await _generate_events(specification)
await _generate_publications(events, publishers, specification)
return _generate_models
@pytest.fixture() @pytest.fixture()
def message_collector(): def message_collector():
class MessageCollector(UserList): class MessageCollector(UserList):

View File

@ -5,7 +5,9 @@ from uuid import UUID
import arrow import arrow
import pytest import pytest
import mobilizon_reshare.storage.query.read
from mobilizon_reshare.event.event import MobilizonEvent from mobilizon_reshare.event.event import MobilizonEvent
from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers.abstract import ( from mobilizon_reshare.publishers.abstract import (
AbstractPlatform, AbstractPlatform,
AbstractEventFormatter, AbstractEventFormatter,
@ -77,3 +79,20 @@ def mock_publisher_invalid_response(message_collector):
pass pass
return MockPublisher() return MockPublisher()
@pytest.fixture()
async def mock_active_publishers_config(monkeypatch):
p = Publisher(name="zulip")
await p.save()
def _mock_active_pub():
return ["zulip"]
monkeypatch.setattr(
mobilizon_reshare.storage.query.read,
"get_active_publishers",
_mock_active_pub
)
return p

View File

@ -89,13 +89,11 @@ async def mock_publications(
await publisher.save() await publisher.save()
publication = PublicationModel( publication = PublicationModel(
id=UUID(int=i + 1), id=UUID(int=i + 1),
status=PublicationStatus.UNSAVED,
event=event, event=event,
publisher=publisher, publisher=publisher,
timestamp=None, timestamp=None,
reason=None, reason=None,
) )
await publication.save()
publication = EventPublication.from_orm(publication, test_event) publication = EventPublication.from_orm(publication, test_event)
publication.publisher = mock_publisher_valid publication.publisher = mock_publisher_valid
publication.formatter = mock_formatter_valid publication.formatter = mock_formatter_valid

View File

@ -1,5 +1,3 @@
from functools import partial
import pytest import pytest
import requests import requests
import responses import responses
@ -7,8 +5,6 @@ import responses
from mobilizon_reshare.config.config import get_settings from mobilizon_reshare.config.config import get_settings
from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.models.publication import PublicationStatus
from mobilizon_reshare.models.publisher import Publisher from mobilizon_reshare.models.publisher import Publisher
from mobilizon_reshare.publishers import get_active_publishers
from mobilizon_reshare.publishers.abstract import EventPublication
from mobilizon_reshare.publishers.coordinator import PublisherCoordinator from mobilizon_reshare.publishers.coordinator import PublisherCoordinator
from mobilizon_reshare.publishers.exceptions import ( from mobilizon_reshare.publishers.exceptions import (
InvalidEvent, InvalidEvent,
@ -17,10 +13,7 @@ from mobilizon_reshare.publishers.exceptions import (
InvalidMessage, InvalidMessage,
) )
from mobilizon_reshare.publishers.platforms.zulip import ZulipFormatter, ZulipPublisher from mobilizon_reshare.publishers.platforms.zulip import ZulipFormatter, ZulipPublisher
from mobilizon_reshare.storage.query.save_query import update_publishers from mobilizon_reshare.storage.query.read import build_publications
from mobilizon_reshare.storage.query.model_creation import (
create_event_publication_models,
)
api_uri = "https://zulip.twc-italia.org/api/v1/" api_uri = "https://zulip.twc-italia.org/api/v1/"
users_me = { users_me = {
@ -88,19 +81,15 @@ def mocked_client_error_response():
@pytest.fixture @pytest.fixture
@pytest.mark.asyncio @pytest.mark.asyncio
async def setup_db(event_model_generator, publication_model_generator): async def setup_db(
mock_active_publishers_config, event_model_generator, publication_model_generator
):
settings = get_settings() settings = get_settings()
for publisher in get_active_publishers():
if publisher != "zulip":
settings["publisher"][publisher]["active"] = False
settings["publisher"]["zulip"][ settings["publisher"]["zulip"][
"bot_email" "bot_email"
] = "giacomotest2-bot@zulip.twc-italia.org" ] = "giacomotest2-bot@zulip.twc-italia.org"
settings["publisher"]["zulip"][ settings["publisher"]["zulip"]["instance"] = "https://zulip.twc-italia.org"
"instance"
] = "https://zulip.twc-italia.org"
await update_publishers(["zulip"])
publisher = await Publisher.filter(name="zulip").first() publisher = await Publisher.filter(name="zulip").first()
event = event_model_generator() event = event_model_generator()
await event.save() await event.save()
@ -112,42 +101,33 @@ async def setup_db(event_model_generator, publication_model_generator):
@pytest.fixture @pytest.fixture
@pytest.mark.asyncio @pytest.mark.asyncio
async def publication_models(event): async def unsaved_publications(event):
await event.to_model().save() await event.to_model().save()
publication_models = await create_event_publication_models(event) return await build_publications(event)
return publication_models
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_zulip_publisher( async def test_zulip_publisher(mocked_valid_response, setup_db, unsaved_publications):
mocked_valid_response, setup_db, event, publication_models
):
report = PublisherCoordinator( report = PublisherCoordinator(unsaved_publications).run()
list(map(partial(EventPublication.from_orm, event=event), publication_models,))
).run()
assert report.reports[0].status == PublicationStatus.COMPLETED assert report.reports[0].status == PublicationStatus.COMPLETED
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_zulip_publishr_failure_invalid_credentials( async def test_zulip_publishr_failure_invalid_credentials(
mocked_credential_error_response, setup_db, event, publication_models mocked_credential_error_response, setup_db, unsaved_publications
): ):
report = PublisherCoordinator( report = PublisherCoordinator(unsaved_publications).run()
list(map(partial(EventPublication.from_orm, event=event), publication_models))
).run()
assert report.reports[0].status == PublicationStatus.FAILED assert report.reports[0].status == PublicationStatus.FAILED
assert report.reports[0].reason == "403 Error - Your credentials are not valid!" assert report.reports[0].reason == "403 Error - Your credentials are not valid!"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_zulip_publisher_failure_client_error( async def test_zulip_publisher_failure_client_error(
mocked_client_error_response, setup_db, event, publication_models mocked_client_error_response, setup_db, unsaved_publications
): ):
report = PublisherCoordinator( report = PublisherCoordinator(unsaved_publications).run()
list(map(partial(EventPublication.from_orm, event=event), publication_models))
).run()
assert report.reports[0].status == PublicationStatus.FAILED assert report.reports[0].status == PublicationStatus.FAILED
assert report.reports[0].reason == "400 Error - Invalid request" assert report.reports[0].reason == "400 Error - Invalid request"

View File

@ -1,13 +1,9 @@
from datetime import datetime, timezone, timedelta from datetime import timedelta
from uuid import UUID from uuid import UUID
from mobilizon_reshare.models.publication import Publication from mobilizon_reshare.models.publication import Publication
from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.models.publication import PublicationStatus
from tests import today
today = datetime(
year=2021, month=6, day=6, hour=5, minute=0, tzinfo=timezone(timedelta(hours=2)),
)
complete_specification = { complete_specification = {
"event": 4, "event": 4,

View File

@ -1,68 +1,19 @@
from datetime import timedelta
from typing import Union
from uuid import UUID
import pytest import pytest
from mobilizon_reshare.models.event import Event import mobilizon_reshare.storage.query.read
from mobilizon_reshare.models.publication import PublicationStatus, Publication
from mobilizon_reshare.models.publisher import Publisher from mobilizon_reshare.models.publisher import Publisher
from tests.storage import today
async def _generate_publishers(specification): @pytest.fixture(scope="function")
async def mock_active_publishers(request, monkeypatch):
for name in request.param:
await Publisher.create(name=name)
publishers = [] def _mock_active_pub():
for i, publisher_name in enumerate(specification["publisher"]): return request.param
publisher = Publisher(
id=UUID(int=i), name=publisher_name, account_ref=f"account_ref_{i}"
)
publishers.append(publisher)
await publisher.save()
return publishers monkeypatch.setattr(
mobilizon_reshare.storage.query.read, "get_active_publishers", _mock_active_pub
async def _generate_events(specification):
events = []
if "event" in specification.keys():
for i in range(specification["event"]):
begin_date = today + timedelta(days=i)
event = Event(
id=UUID(int=i),
name=f"event_{i}",
description=f"desc_{i}",
mobilizon_id=UUID(int=i),
mobilizon_link=f"moblink_{i}",
thumbnail_link=f"thumblink_{i}",
location=f"loc_{i}",
begin_datetime=begin_date,
end_datetime=begin_date + timedelta(hours=2),
)
events.append(event)
await event.save()
return events
async def _generate_publications(events, publishers, specification):
if "publications" in specification.keys():
for i, publication in enumerate(specification["publications"]):
status = publication.get("status", PublicationStatus.COMPLETED)
timestamp = publication.get("timestamp", today + timedelta(hours=i))
await Publication.create(
id=UUID(int=i),
status=status,
timestamp=timestamp,
event_id=events[publication["event_idx"]].id,
publisher_id=publishers[publication["publisher_idx"]].id,
) )
return request.param
@pytest.fixture(scope="module")
def generate_models():
async def _generate_models(specification: dict[str, Union[int, list]]):
publishers = await _generate_publishers(specification)
events = await _generate_events(specification)
await _generate_publications(events, publishers, specification)
return _generate_models

View File

@ -7,17 +7,16 @@ import pytest
from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus from mobilizon_reshare.event.event import MobilizonEvent, EventPublicationStatus
from mobilizon_reshare.models.event import Event from mobilizon_reshare.models.event import Event
from mobilizon_reshare.models.publication import PublicationStatus from mobilizon_reshare.models.publication import PublicationStatus
from mobilizon_reshare.storage.query.read_query import ( from mobilizon_reshare.storage.query.read import (
get_mobilizon_event_publications,
get_published_events, get_published_events,
events_with_status, events_with_status,
prefetch_event_relations,
publications_with_status, publications_with_status,
events_without_publications, events_without_publications,
build_publications,
) )
from tests.storage import complete_specification from tests.storage import complete_specification
from tests.storage import result_publication from tests.storage import result_publication
from tests.storage import today from tests import today
event_0 = MobilizonEvent( event_0 = MobilizonEvent(
name="event_0", name="event_0",
@ -41,33 +40,6 @@ async def test_get_published_events(generate_models):
assert len(published_events) == 3 assert len(published_events) == 3
@pytest.mark.asyncio
async def test_get_mobilizon_event_publications(generate_models):
await generate_models(complete_specification)
models = await prefetch_event_relations(Event.filter(name="event_0"))
mobilizon_event = MobilizonEvent.from_model(models[0])
publications = list(await get_mobilizon_event_publications(mobilizon_event))
for pub in publications:
await pub.fetch_related("event")
await pub.fetch_related("publisher")
assert len(publications) == 3
assert publications[0].event.name == "event_0"
assert publications[0].publisher.name == "telegram"
assert publications[0].status == PublicationStatus.COMPLETED
assert publications[1].event.name == "event_0"
assert publications[1].publisher.name == "twitter"
assert publications[1].status == PublicationStatus.COMPLETED
assert publications[2].event.name == "event_0"
assert publications[2].publisher.name == "mastodon"
assert publications[2].status == PublicationStatus.COMPLETED
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize( @pytest.mark.parametrize(
"status,mobilizon_id,from_date,to_date,expected_result", "status,mobilizon_id,from_date,to_date,expected_result",
@ -96,7 +68,12 @@ async def test_get_mobilizon_event_publications(generate_models):
], ],
) )
async def test_publications_with_status( async def test_publications_with_status(
status, mobilizon_id, from_date, to_date, expected_result, generate_models, status,
mobilizon_id,
from_date,
to_date,
expected_result,
generate_models,
): ):
await generate_models(complete_specification) await generate_models(complete_specification)
publications = await publications_with_status( publications = await publications_with_status(
@ -186,6 +163,25 @@ async def test_event_with_status_window(
), ),
], ],
), ),
(
{
"event": 3,
"publications": [
{
"event_idx": 1,
"publisher_idx": 0,
"status": PublicationStatus.FAILED,
},
{
"event_idx": 2,
"publisher_idx": 0,
"status": PublicationStatus.COMPLETED,
},
],
"publisher": ["zulip"],
},
[event_0],
),
( (
complete_specification, complete_specification,
[ [
@ -212,3 +208,46 @@ async def test_events_without_publications(spec, expected_events, generate_model
unpublished_events = list(await events_without_publications()) unpublished_events = list(await events_without_publications())
assert len(unpublished_events) == len(expected_events) assert len(unpublished_events) == len(expected_events)
assert unpublished_events == expected_events assert unpublished_events == expected_events
@pytest.mark.asyncio
@pytest.mark.parametrize(
"mock_active_publishers, spec, event, n_publications",
[
(
[],
{"event": 2, "publications": [], "publisher": ["zulip"]},
event_0,
0,
),
(
["zulip"],
{"event": 2, "publications": [], "publisher": ["zulip"]},
event_0,
1,
),
(
["telegram", "zulip", "mastodon", "facebook"],
{
"event": 2,
"publications": [],
"publisher": ["telegram", "zulip", "mastodon", "facebook"],
},
event_0,
4,
),
],
indirect=["mock_active_publishers"],
)
async def test_build_publications(
mock_active_publishers, spec, event, n_publications, generate_models
):
await generate_models(spec)
publications = list(await build_publications(event))
assert len(publications) == n_publications
for p in publications:
assert p.event == event
assert p.publisher.name in mock_active_publishers

View File

@ -2,7 +2,7 @@ from uuid import UUID
import pytest import pytest
from mobilizon_reshare.storage.query.read_query import ( from mobilizon_reshare.storage.query.read import (
get_unpublished_events, get_unpublished_events,
get_all_events, get_all_events,
) )

View File

@ -12,8 +12,12 @@ from mobilizon_reshare.publishers.coordinator import (
PublisherCoordinatorReport, PublisherCoordinatorReport,
EventPublicationReport, EventPublicationReport,
) )
from mobilizon_reshare.storage.query.read_query import publications_with_status from mobilizon_reshare.publishers.platforms.telegram import (
from mobilizon_reshare.storage.query.save_query import ( TelegramFormatter,
TelegramPublisher,
)
from mobilizon_reshare.storage.query.read import publications_with_status
from mobilizon_reshare.storage.query.write import (
save_publication_report, save_publication_report,
update_publishers, update_publishers,
) )
@ -22,6 +26,18 @@ from tests.storage import today
two_publishers_specification = {"publisher": ["telegram", "twitter"]} two_publishers_specification = {"publisher": ["telegram", "twitter"]}
event_1 = MobilizonEvent(
name="event_1",
description="desc_1",
mobilizon_id=UUID(int=1),
mobilizon_link="moblink_1",
thumbnail_link="thumblink_1",
location="loc_1",
status=EventPublicationStatus.WAITING,
begin_datetime=arrow.get(today + timedelta(days=1)),
end_datetime=arrow.get(today + timedelta(days=1) + timedelta(hours=2)),
)
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -48,7 +64,10 @@ two_publishers_specification = {"publisher": ["telegram", "twitter"]}
], ],
) )
async def test_update_publishers( async def test_update_publishers(
specification, names, expected_result, generate_models, specification,
names,
expected_result,
generate_models,
): ):
await generate_models(specification) await generate_models(specification)
await update_publishers(names) await update_publishers(names)
@ -74,46 +93,41 @@ async def test_update_publishers(
status=PublicationStatus.COMPLETED, status=PublicationStatus.COMPLETED,
reason="", reason="",
publication=EventPublication( publication=EventPublication(
id=UUID(int=4), formatter=None, event=None, publisher=None id=UUID(int=6),
formatter=TelegramFormatter(),
event=event_1,
publisher=TelegramPublisher(),
), ),
), ),
], ],
), ),
MobilizonEvent( event_1,
name="event_1",
description="desc_1",
mobilizon_id=UUID(int=1),
mobilizon_link="moblink_1",
thumbnail_link="thumblink_1",
location="loc_1",
status=EventPublicationStatus.WAITING,
begin_datetime=arrow.get(today + timedelta(days=1)),
end_datetime=arrow.get(today + timedelta(days=1) + timedelta(hours=2)),
),
{ {
UUID(int=4): Publication( UUID(int=6): Publication(
id=UUID(int=4), status=PublicationStatus.COMPLETED, reason="" id=UUID(int=6), status=PublicationStatus.COMPLETED, reason=""
), ),
}, },
], ],
], ],
) )
async def test_save_publication_report( async def test_save_publication_report(
specification, report, event, expected_result, generate_models, specification,
report,
event,
expected_result,
generate_models,
): ):
await generate_models(specification) await generate_models(specification)
known_publication_ids = set(p.id for p in await Publication.all())
await save_publication_report(report)
publications = await publications_with_status(
status=PublicationStatus.COMPLETED, event_mobilizon_id=event.mobilizon_id,
)
await save_publication_report(report, list(publications.values()))
publication_ids = set(publications.keys())
publications = { publications = {
p_id: await Publication.filter(id=p_id).first() for p_id in publication_ids p.id: p for p in await Publication.filter(id__not_in=known_publication_ids)
} }
assert len(publications) == len(expected_result) assert len(publications) == len(expected_result)
for i in publication_ids: for i in publications.keys():
assert publications[i].status == expected_result[i].status assert publications[i].status == expected_result[i].status
assert publications[i].reason == expected_result[i].reason assert publications[i].reason == expected_result[i].reason
assert publications[i].timestamp assert publications[i].timestamp