Mobilizon
/
Mobilizon-Reshare-condividi-sui-social
mirror of https://github.com/Tech-Workers-Coalition-Italia/mobilizon-reshare.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

rework config (#121) 

* simplified config behavior

* temp

* removed redundant tests

* removed publication window

* removed settings_file cli option

* add pre_test code, in order to set environment variables

* Revert "add pre_test code, in order to set environment variables"

This reverts commit 0d25f9313a.

Co-authored-by: magowiz <magowiz@gmail.com>
Co-authored-by: Giacomo Leidi <goodoldpaul@autistici.org>
This commit is contained in:
Simone Robutti 2022-01-08 00:54:27 +01:00 committed by GitHub
parent 352c49ca94
commit a0a1d43fa0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 72 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
mobilizon_reshare/cli/__init__.py
View File

@ -3,7 +3,6 @@ import logging
import traceback
from logging.config import dictConfig
from pathlib import Path
import sys
from mobilizon_reshare.config.config import get_settings
from mobilizon_reshare.storage.db import tear_down, MoReDB
@ -16,23 +15,21 @@ async def graceful_exit(code):
exit(code)
async def init(settings_file):
settings = get_settings(settings_file)
async def init():
settings = get_settings()
dictConfig(settings["logging"])
db_path = Path(settings.db_path)
db = MoReDB(db_path)
db_setup = asyncio.create_task(db.setup())
_, _ = await asyncio.wait({db_setup},
return_when=asyncio.FIRST_EXCEPTION)
_, _ = await asyncio.wait({db_setup}, return_when=asyncio.FIRST_EXCEPTION)
if db_setup.exception():
logging.critical("exception during db setup")
raise db_setup.exception()
async def _safe_execution(f, settings_file):
init_task = asyncio.create_task(init(settings_file))
_, _ = await asyncio.wait({init_task},
return_when=asyncio.FIRST_EXCEPTION)
async def _safe_execution(f):
init_task = asyncio.create_task(init())
_, _ = await asyncio.wait({init_task}, return_when=asyncio.FIRST_EXCEPTION)
if init_task.exception():
logging.critical("exception during init")
# raise init_task.exception()
@ -50,5 +47,5 @@ async def _safe_execution(f, settings_file):
await graceful_exit(return_code)
def safe_execution(f, settings_file):
asyncio.run(_safe_execution(f, settings_file))
def safe_execution(f):
asyncio.run(_safe_execution(f))

42
mobilizon_reshare/cli/cli.py
View File

@ -10,11 +10,10 @@ from mobilizon_reshare.cli.commands.inspect.inspect_event import inspect_events
from mobilizon_reshare.cli.commands.inspect.inspect_publication import (
inspect_publications,
)
from mobilizon_reshare.cli.commands.start.main import main as start_main
from mobilizon_reshare.cli.commands.recap.main import main as recap_main
from mobilizon_reshare.cli.commands.start.main import main as start_main
from mobilizon_reshare.config.config import current_version
from mobilizon_reshare.config.publishers import publisher_names
from mobilizon_reshare.event.event import EventPublicationStatus
from mobilizon_reshare.models.publication import PublicationStatus
@ -32,14 +31,6 @@ status_name_to_enum = {
"all": None,
},
}
settings_file_option = click.option(
"-f",
"--settings-file",
type=click.Path(exists=True),
help="The path for the settings file. "
"Overrides the one specified in the environment variables.",
)
from_date_option = click.option(
"-b",
"--begin",
@ -89,17 +80,15 @@ def mobilizon_reshare(obj):
@mobilizon_reshare.command(help="Synchronize and publish events.")
@settings_file_option
@pass_context
def start(ctx, settings_file):
def start(ctx,):
ctx.ensure_object(dict)
safe_execution(start_main, settings_file=settings_file)
safe_execution(start_main,)
@mobilizon_reshare.command(help="Publish a recap of already published events.")
@settings_file_option
def recap(settings_file):
safe_execution(recap_main, settings_file=settings_file)
def recap():
safe_execution(recap_main,)
@mobilizon_reshare.group(help="List objects in the database with different criteria.")
@ -114,9 +103,10 @@ def inspect(ctx, begin, end):
@inspect.command(help="Query for events in the database.")
@event_status_option
@settings_file_option
@pass_context
def event(ctx, status, settings_file):
def event(
ctx, status,
):
ctx.ensure_object(dict)
safe_execution(
functools.partial(
@ -125,15 +115,15 @@ def event(ctx, status, settings_file):
frm=ctx.obj["begin"],
to=ctx.obj["end"],
),
settings_file,
)
@inspect.command(help="Query for publications in the database.")
@publication_status_option
@settings_file_option
@pass_context
def publication(ctx, status, settings_file):
def publication(
ctx, status,
):
ctx.ensure_object(dict)
safe_execution(
functools.partial(
@ -142,7 +132,6 @@ def publication(ctx, status, settings_file):
frm=ctx.obj["begin"],
to=ctx.obj["end"],
),
settings_file,
)
@ -152,11 +141,10 @@ def publication(ctx, status, settings_file):
)
@click.argument("event-id", type=click.UUID)
@click.argument("publisher", type=click.Choice(publisher_names))
@settings_file_option
def format(event_id, publisher, settings_file):
safe_execution(
functools.partial(format_event, event_id, publisher), settings_file,
)
def format(
event_id, publisher,
):
safe_execution(functools.partial(format_event, event_id, publisher),)
if __name__ == "__main__":

105
mobilizon_reshare/config/config.py
View File

@ -2,26 +2,18 @@ import importlib.resources
from pathlib import Path
from typing import Optional
import pkg_resources
from appdirs import AppDirs
from dynaconf import Dynaconf, Validator
import mobilizon_reshare
from mobilizon_reshare.config import strategies, publishers, notifiers
from mobilizon_reshare.config.notifiers import notifier_names
from mobilizon_reshare.config.publishers import publisher_names
base_validators = [
# strategy to decide events to publish
Validator("selection.strategy", must_exist=True, is_type_of=str),
Validator(
"publishing.window.begin",
must_exist=True,
is_type_of=int,
gte=0,
lte=24,
),
Validator("publishing.window.end", must_exist=True, is_type_of=int, gte=0, lte=24),
# url of the main Mobilizon instance to download events from
Validator("source.mobilizon.url", must_exist=True, is_type_of=str),
Validator("source.mobilizon.group", must_exist=True, is_type_of=str),
@ -41,44 +33,50 @@ def current_version() -> str:
return fp.read()
def build_settings(
settings_file: Optional[str] = None, validators: Optional[list[Validator]] = None
):
def get_settings_files_paths():
dirs = AppDirs(appname="mobilizon-reshare", version=current_version())
bundled_settings_path = pkg_resources.resource_filename(
"mobilizon_reshare", "settings.toml"
)
bundled_secrets_path = pkg_resources.resource_filename(
"mobilizon_reshare", ".secrets.toml"
)
return [
Path(dirs.user_config_dir, "mobilizon_reshare.toml").absolute(),
Path(dirs.site_config_dir, "mobilizon_reshare.toml").absolute(),
Path(dirs.site_config_dir, ".secrets.toml").absolute(),
Path(dirs.site_config_dir, ".secrets.toml").absolute(),
bundled_settings_path,
bundled_secrets_path,
]
def build_settings(validators: Optional[list[Validator]] = None):
"""
Creates a Dynaconf base object. Configuration files are checked in this order:
1. CLI argument
2. User configuration directory. On Linux that's `$XDG_CONFIG_HOME/mobilizon_reshare/<mobilizon-reshare-version>`;
3. System configuration directory. On Linux that's the first element of
1. User configuration directory. On Linux that's `$XDG_CONFIG_HOME/mobilizon_reshare/<mobilizon-reshare-version>`;
2. System configuration directory. On Linux that's the first element of
`$XDG_CONFIG_DIRS` + `/mobilizon_reshare/<mobilizon-reshare-version>`.
4. The default configuration distributed with the package.
3. The default configuration distributed with the package.
The first available configuration file will be loaded.
"""
dirs = AppDirs(appname="mobilizon-reshare", version=current_version())
with importlib.resources.path(
mobilizon_reshare, "settings.toml"
) as bundled_settings_path:
for f in [
settings_file,
Path(dirs.user_config_dir, "mobilizon_reshare.toml"),
Path(dirs.site_config_dir, "mobilizon_reshare.toml"),
bundled_settings_path,
]:
if f and Path(f).exists():
SETTINGS_FILE = f
break
ENVVAR_PREFIX = "MOBILIZON_RESHARE"
return Dynaconf(
config = Dynaconf(
environments=True,
envvar_prefix=ENVVAR_PREFIX,
settings_files=SETTINGS_FILE,
settings_files=get_settings_files_paths(),
validators=validators or [],
)
# TODO use validation control in dynaconf 3.2.0 once released
config.validators.validate()
return config
def build_and_validate_settings(settings_file: Optional[str] = None):
def build_and_validate_settings():
"""
Creates a settings object to be used in the application. It collects and apply generic validators and validators
specific for each publisher, notifier and publication strategy.
@ -86,9 +84,7 @@ def build_and_validate_settings(settings_file: Optional[str] = None):
# we first do a preliminary load of the settings without validation. We will later use them to determine which
# publishers, notifiers and strategy have been selected
raw_settings = build_settings(
settings_file=settings_file, validators=activeness_validators
)
raw_settings = build_settings(validators=activeness_validators)
# we retrieve validators that are conditional. Each module will analyze the settings and decide which validators
# need to be applied.
@ -98,14 +94,12 @@ def build_and_validate_settings(settings_file: Optional[str] = None):
# we rebuild the settings, providing all the selected validators.
settings = build_settings(
settings_file,
base_validators
+ strategy_validators
+ publisher_validators
+ notifier_validators,
)
# TODO use validation control in dynaconf 3.2.0 once released
settings.validators.validate()
return settings
@ -115,29 +109,22 @@ def build_and_validate_settings(settings_file: Optional[str] = None):
# The normal Dynaconf options to specify the settings files are also not a valid option because of the two steps
# validation that prevents us to employ their mechanism to specify settings files. This could probably be reworked
# better in the future.
class CustomConfig:
_instance = None
_settings_file = None
def __new__(cls, settings_file: Optional[str] = None):
if (
settings_file is None and cls._settings_file is not None
): # normal access, I don't want to reload
class CustomConfig(object):
@classmethod
def get_instance(cls):
if not hasattr(cls, "_instance") or cls._instance is None:
cls._instance = cls()
return cls._instance
if (
cls._instance is None and cls._settings_file is None
) or settings_file != cls._settings_file:
cls._settings_file = settings_file
cls._instance = super(CustomConfig, cls).__new__(cls)
cls.settings = build_and_validate_settings(settings_file)
def __init__(self):
self.settings = build_and_validate_settings()
return cls._instance
def update(self, settings_file: Optional[str] = None):
self.settings = build_and_validate_settings(settings_file)
@classmethod
def clear(cls):
cls._instance = None
def get_settings(settings_file: Optional[str] = None):
config = CustomConfig(settings_file)
return config.settings
def get_settings():
return CustomConfig.get_instance().settings

20
mobilizon_reshare/event/event_selection_strategies.py
View File

@ -16,26 +16,12 @@ class EventSelectionStrategy(ABC):
published_events: List[MobilizonEvent],
unpublished_events: List[MobilizonEvent],
) -> Optional[MobilizonEvent]:
if not self.is_in_publishing_window():
logger.info("Outside of publishing window, no event will be published.")
return None
selected = self._select(published_events, unpublished_events)
if selected:
return selected[0]
else:
return None
def is_in_publishing_window(self) -> bool:
settings = get_settings()
window_beginning = settings["publishing"]["window"]["begin"]
window_end = settings["publishing"]["window"]["end"]
now_hour = arrow.now().datetime.hour
if window_beginning <= window_end:
return window_beginning <= now_hour < window_end
else:
return now_hour >= window_beginning or now_hour < window_end
@abstractmethod
def _select(
self,
@ -96,8 +82,7 @@ STRATEGY_NAME_TO_STRATEGY_CLASS = {"next_event": SelectNextEventStrategy}
def select_unpublished_events(
published_events: List[MobilizonEvent],
unpublished_events: List[MobilizonEvent],
published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent],
):
strategy = STRATEGY_NAME_TO_STRATEGY_CLASS[
@ -108,8 +93,7 @@ def select_unpublished_events(
def select_event_to_publish(
published_events: List[MobilizonEvent],
unpublished_events: List[MobilizonEvent],
published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent],
):
strategy = STRATEGY_NAME_TO_STRATEGY_CLASS[

1
mobilizon_reshare/publishers/platforms/telegram.py
View File

@ -41,6 +41,7 @@ class TelegramFormatter(AbstractEventFormatter):
")",
">",
"<",
">",
"{",
"}",
]

3
mobilizon_reshare/settings.toml
View File

@ -13,9 +13,6 @@ group="my_group"
[default.selection]
strategy = "next_event"
[default.publishing.window]
begin=12
end=18
[default.selection.strategy_options]
break_between_events_in_minutes =60

8
tests/commands/test_start.py
View File

@ -32,13 +32,11 @@ async def test_start_no_event(
"elements",
[[simple_event_element()], [simple_event_element(), simple_event_element()]],
)
@pytest.mark.parametrize("publication_window", [(0, 24)])
async def test_start_new_event(
mock_mobilizon_success_answer,
mobilizon_answer,
caplog,
mock_publisher_config,
mock_publication_window,
message_collector,
elements,
):
@ -88,13 +86,11 @@ async def test_start_new_event(
@pytest.mark.parametrize(
"elements", [[]],
)
@pytest.mark.parametrize("publication_window", [(0, 24)])
async def test_start_event_from_db(
mock_mobilizon_success_answer,
mobilizon_answer,
caplog,
mock_publisher_config,
mock_publication_window,
message_collector,
event_generator,
):
@ -135,13 +131,11 @@ async def test_start_event_from_db(
@pytest.mark.parametrize(
"elements", [[]],
)
@pytest.mark.parametrize("publication_window", [(0, 24)])
async def test_start_publisher_failure(
mock_mobilizon_success_answer,
mobilizon_answer,
caplog,
mock_publisher_config,
mock_publication_window,
message_collector,
event_generator,
mock_notifier_config,
@ -217,13 +211,11 @@ def second_event_element():
@pytest.mark.parametrize(
"elements", [[second_event_element()]],
)
@pytest.mark.parametrize("publication_window", [(0, 24)])
async def test_start_second_execution(
mock_mobilizon_success_answer,
mobilizon_answer,
caplog,
mock_publisher_config,
mock_publication_window,
message_collector,
event_generator,
published_event,

33
tests/config/test_config_singleton.py
View File

@ -1,33 +0,0 @@
import pkg_resources
from mobilizon_reshare.config.config import get_settings
def test_singleton():
settings_file = pkg_resources.resource_filename(
"tests.resources.config", "test_singleton.toml"
)
config_1 = get_settings(settings_file)
config_2 = get_settings()
assert id(config_1) == id(config_2)
def test_same_file():
settings_file = pkg_resources.resource_filename(
"tests.resources.config", "test_singleton.toml"
)
config_1 = get_settings(settings_file)
config_2 = get_settings(settings_file)
assert id(config_1) == id(config_2)
def test_singleton_new_file():
settings_file = pkg_resources.resource_filename(
"tests.resources.config", "test_singleton.toml"
)
settings_file_2 = pkg_resources.resource_filename(
"tests.resources.config", "test_singleton_2.toml"
)
config_1 = get_settings(settings_file)
config_2 = get_settings(settings_file_2)
assert id(config_1) != id(config_2)

44
tests/config/test_validation.py
View File

@ -1,44 +0,0 @@
import os
import dynaconf
import pkg_resources
import pytest
from mobilizon_reshare.config.config import get_settings
@pytest.fixture
def invalid_settings_file(tmp_path, toml_content):
file = tmp_path / "tmp.toml"
file.write_text(toml_content)
return file
@pytest.mark.parametrize("toml_content", ["invalid toml["])
def test_get_settings_failure_invalid_toml(invalid_settings_file):
with pytest.raises(dynaconf.vendor.toml.decoder.TomlDecodeError):
get_settings(invalid_settings_file.absolute())
@pytest.mark.parametrize("toml_content", [""])
def test_get_settings_failure_invalid_preliminary_config(invalid_settings_file):
os.environ["SECRETS_FOR_DYNACONF"] = ""
with pytest.raises(dynaconf.validator.ValidationError):
get_settings(invalid_settings_file.absolute())
@pytest.mark.parametrize(
"invalid_toml,pattern_in_exception",
[["config_with_invalid_strategy.toml", "break_between_events_in_minutes"]],
)
def test_get_settings_failure_config_base_validators(
invalid_toml, pattern_in_exception
):
with pytest.raises(dynaconf.validator.ValidationError) as e:
get_settings(
pkg_resources.resource_filename("tests.resources.config", invalid_toml)
)
assert e.match(pattern_in_exception)

8
tests/conftest.py
View File

@ -309,14 +309,6 @@ def mock_mobilizon_success_answer(mobilizon_answer, mobilizon_url):
yield
@pytest.fixture
def mock_publication_window(publication_window):
begin, end = publication_window
get_settings().update(
{"publishing.window.begin": begin, "publishing.window.end": end}
)
@pytest.fixture
def mock_formatter_class():
class MockFormatter(AbstractEventFormatter):

36
tests/event/test_strategies.py
View File

@ -207,39 +207,3 @@ def mock_arrow_now(current_hour):
with patch("arrow.now", mock_now):
yield
@pytest.mark.parametrize("current_hour", [14, 15, 16, 18])
@pytest.mark.parametrize("publication_window", [(14, 19)])
def test_publishing_inner_window_true(mock_arrow_now, mock_publication_window):
"""
Testing that the window check correctly returns True when in an inner publishing window.
"""
assert SelectNextEventStrategy().is_in_publishing_window()
@pytest.mark.parametrize("current_hour", [2, 10, 11, 19])
@pytest.mark.parametrize("publication_window", [(14, 19)])
def test_publishing_inner_window_false(mock_arrow_now, mock_publication_window):
"""
Testing that the window check correctly returns False when not in an inner publishing window.
"""
assert not SelectNextEventStrategy().is_in_publishing_window()
@pytest.mark.parametrize("current_hour", [2, 10, 11, 19])
@pytest.mark.parametrize("publication_window", [(19, 14)])
def test_publishing_outer_window_true(mock_arrow_now, mock_publication_window):
"""
Testing that the window check correctly returns True when in an outer publishing window.
"""
assert SelectNextEventStrategy().is_in_publishing_window()
@pytest.mark.parametrize("current_hour", [14, 15, 16, 18])
@pytest.mark.parametrize("publication_window", [(19, 14)])
def test_publishing_outer_window_false(mock_arrow_now, mock_publication_window):
"""
Testing that the window check correctly returns False when not in an outer publishing window.
"""
assert not SelectNextEventStrategy().is_in_publishing_window()

3
tests/resources/config/config_with_invalid_strategy.toml
View File

@ -2,9 +2,6 @@
url="https://some_mobilizon"
group="my_group"
[testing.publishing.window]
begin=12
end=18
[testing.selection]
strategy = "next_event"

3
tests/resources/config/config_with_strategy.toml
View File

@ -2,9 +2,6 @@
url="https://some_mobilizon"
group="my_group"
[testing.publishing.window]
begin=12
end=18
[testing.selection]
strategy = "next_event"

3
tests/resources/config/test_singleton.toml
View File

@ -5,9 +5,6 @@ group="my_group"
[testing.selection]
strategy = "next_event"
[testing.publishing.window]
begin=12
end=18
[testing.selection.strategy_options]
break_between_events_in_minutes =60

3
tests/resources/config/test_singleton_2.toml
View File

@ -5,9 +5,6 @@ group="my_group"
[testing.selection]
strategy = "next_event"
[testing.publishing.window]
begin=12
end=18
[testing.selection.strategy_options]
break_between_events_in_minutes =60