rework config (#121)

* simplified config behavior

* temp

* removed redundant tests

* removed publication window

* removed settings_file cli option

* add pre_test code, in order to set environment variables

* Revert "add pre_test code, in order to set environment variables"

This reverts commit 0d25f9313a5a791b0c0ae5be63fa08aced69ba28.

Co-authored-by: magowiz <magowiz@gmail.com>
Co-authored-by: Giacomo Leidi <goodoldpaul@autistici.org>

mobilizon_reshare/cli/__init__.py

@@ -3,7 +3,6 @@ import logging
 import traceback
 from logging.config import dictConfig
 from pathlib import Path
-import sys
 
 from mobilizon_reshare.config.config import get_settings
 from mobilizon_reshare.storage.db import tear_down, MoReDB
@@ -16,23 +15,21 @@ async def graceful_exit(code):
     exit(code)
 
 
-async def init(settings_file):
+async def init():
-    settings = get_settings(settings_file)
+    settings = get_settings()
     dictConfig(settings["logging"])
     db_path = Path(settings.db_path)
     db = MoReDB(db_path)
     db_setup = asyncio.create_task(db.setup())
-    _, _ = await asyncio.wait({db_setup},
+    _, _ = await asyncio.wait({db_setup}, return_when=asyncio.FIRST_EXCEPTION)
-                              return_when=asyncio.FIRST_EXCEPTION)
     if db_setup.exception():
         logging.critical("exception during db setup")
         raise db_setup.exception()
 
 
-async def _safe_execution(f, settings_file):
+async def _safe_execution(f):
-    init_task = asyncio.create_task(init(settings_file))
+    init_task = asyncio.create_task(init())
-    _, _ = await asyncio.wait({init_task},
+    _, _ = await asyncio.wait({init_task}, return_when=asyncio.FIRST_EXCEPTION)
-                              return_when=asyncio.FIRST_EXCEPTION)
     if init_task.exception():
         logging.critical("exception during init")
         # raise init_task.exception()
@@ -50,5 +47,5 @@ async def _safe_execution(f, settings_file):
         await graceful_exit(return_code)
 
 
-def safe_execution(f, settings_file):
+def safe_execution(f):
-    asyncio.run(_safe_execution(f, settings_file))
+    asyncio.run(_safe_execution(f))

mobilizon_reshare/cli/cli.py

@@ -10,11 +10,10 @@ from mobilizon_reshare.cli.commands.inspect.inspect_event import inspect_events
 from mobilizon_reshare.cli.commands.inspect.inspect_publication import (
     inspect_publications,
 )
-from mobilizon_reshare.cli.commands.start.main import main as start_main
 from mobilizon_reshare.cli.commands.recap.main import main as recap_main
+from mobilizon_reshare.cli.commands.start.main import main as start_main
 from mobilizon_reshare.config.config import current_version
 from mobilizon_reshare.config.publishers import publisher_names
-
 from mobilizon_reshare.event.event import EventPublicationStatus
 from mobilizon_reshare.models.publication import PublicationStatus
 
@@ -32,14 +31,6 @@ status_name_to_enum = {
         "all": None,
     },
 }
-
-settings_file_option = click.option(
-    "-f",
-    "--settings-file",
-    type=click.Path(exists=True),
-    help="The path for the settings file. "
-    "Overrides the one specified in the environment variables.",
-)
 from_date_option = click.option(
     "-b",
     "--begin",
@@ -89,17 +80,15 @@ def mobilizon_reshare(obj):
 
 
 @mobilizon_reshare.command(help="Synchronize and publish events.")
-@settings_file_option
 @pass_context
-def start(ctx, settings_file):
+def start(ctx,):
     ctx.ensure_object(dict)
-    safe_execution(start_main, settings_file=settings_file)
+    safe_execution(start_main,)
 
 
 @mobilizon_reshare.command(help="Publish a recap of already published events.")
-@settings_file_option
+def recap():
-def recap(settings_file):
+    safe_execution(recap_main,)
-    safe_execution(recap_main, settings_file=settings_file)
 
 
 @mobilizon_reshare.group(help="List objects in the database with different criteria.")
@@ -114,9 +103,10 @@ def inspect(ctx, begin, end):
 
 @inspect.command(help="Query for events in the database.")
 @event_status_option
-@settings_file_option
 @pass_context
-def event(ctx, status, settings_file):
+def event(
+    ctx, status,
+):
     ctx.ensure_object(dict)
     safe_execution(
         functools.partial(
@@ -125,15 +115,15 @@ def event(ctx, status, settings_file):
             frm=ctx.obj["begin"],
             to=ctx.obj["end"],
         ),
-        settings_file,
     )
 
 
 @inspect.command(help="Query for publications in the database.")
 @publication_status_option
-@settings_file_option
 @pass_context
-def publication(ctx, status, settings_file):
+def publication(
+    ctx, status,
+):
     ctx.ensure_object(dict)
     safe_execution(
         functools.partial(
@@ -142,7 +132,6 @@ def publication(ctx, status, settings_file):
             frm=ctx.obj["begin"],
             to=ctx.obj["end"],
         ),
-        settings_file,
     )
 
 
@@ -152,11 +141,10 @@ def publication(ctx, status, settings_file):
 )
 @click.argument("event-id", type=click.UUID)
 @click.argument("publisher", type=click.Choice(publisher_names))
-@settings_file_option
+def format(
-def format(event_id, publisher, settings_file):
+    event_id, publisher,
-    safe_execution(
+):
-        functools.partial(format_event, event_id, publisher), settings_file,
+    safe_execution(functools.partial(format_event, event_id, publisher),)
-    )
 
 
 if __name__ == "__main__":

mobilizon_reshare/config/config.py

@@ -2,26 +2,18 @@ import importlib.resources
 from pathlib import Path
 from typing import Optional
 
+import pkg_resources
 from appdirs import AppDirs
 from dynaconf import Dynaconf, Validator
 
 import mobilizon_reshare
 from mobilizon_reshare.config import strategies, publishers, notifiers
 from mobilizon_reshare.config.notifiers import notifier_names
-
 from mobilizon_reshare.config.publishers import publisher_names
 
 base_validators = [
     # strategy to decide events to publish
     Validator("selection.strategy", must_exist=True, is_type_of=str),
-    Validator(
-        "publishing.window.begin",
-        must_exist=True,
-        is_type_of=int,
-        gte=0,
-        lte=24,
-    ),
-    Validator("publishing.window.end", must_exist=True, is_type_of=int, gte=0, lte=24),
     # url of the main Mobilizon instance to download events from
     Validator("source.mobilizon.url", must_exist=True, is_type_of=str),
     Validator("source.mobilizon.group", must_exist=True, is_type_of=str),
@@ -41,44 +33,50 @@ def current_version() -> str:
         return fp.read()
 
 
-def build_settings(
+def get_settings_files_paths():
-    settings_file: Optional[str] = None, validators: Optional[list[Validator]] = None
+
-):
+    dirs = AppDirs(appname="mobilizon-reshare", version=current_version())
+    bundled_settings_path = pkg_resources.resource_filename(
+        "mobilizon_reshare", "settings.toml"
+    )
+    bundled_secrets_path = pkg_resources.resource_filename(
+        "mobilizon_reshare", ".secrets.toml"
+    )
+    return [
+        Path(dirs.user_config_dir, "mobilizon_reshare.toml").absolute(),
+        Path(dirs.site_config_dir, "mobilizon_reshare.toml").absolute(),
+        Path(dirs.site_config_dir, ".secrets.toml").absolute(),
+        Path(dirs.site_config_dir, ".secrets.toml").absolute(),
+        bundled_settings_path,
+        bundled_secrets_path,
+    ]
+
+
+def build_settings(validators: Optional[list[Validator]] = None):
     """
     Creates a Dynaconf base object. Configuration files are checked in this order:
 
-      1. CLI argument
+      1. User configuration directory. On Linux that's `$XDG_CONFIG_HOME/mobilizon_reshare/<mobilizon-reshare-version>`;
-      2. User configuration directory. On Linux that's `$XDG_CONFIG_HOME/mobilizon_reshare/<mobilizon-reshare-version>`;
+      2. System configuration directory. On Linux that's the first element of
-      3. System configuration directory. On Linux that's the first element of
          `$XDG_CONFIG_DIRS` + `/mobilizon_reshare/<mobilizon-reshare-version>`.
-      4. The default configuration distributed with the package.
+      3. The default configuration distributed with the package.
 
     The first available configuration file will be loaded.
     """
-    dirs = AppDirs(appname="mobilizon-reshare", version=current_version())
-    with importlib.resources.path(
-        mobilizon_reshare, "settings.toml"
-    ) as bundled_settings_path:
-        for f in [
-            settings_file,
-            Path(dirs.user_config_dir, "mobilizon_reshare.toml"),
-            Path(dirs.site_config_dir, "mobilizon_reshare.toml"),
-            bundled_settings_path,
-        ]:
-            if f and Path(f).exists():
-                SETTINGS_FILE = f
-                break
-
     ENVVAR_PREFIX = "MOBILIZON_RESHARE"
-    return Dynaconf(
+    config = Dynaconf(
         environments=True,
         envvar_prefix=ENVVAR_PREFIX,
-        settings_files=SETTINGS_FILE,
+        settings_files=get_settings_files_paths(),
         validators=validators or [],
     )
 
+    # TODO use validation control in dynaconf 3.2.0 once released
+    config.validators.validate()
+    return config
 
-def build_and_validate_settings(settings_file: Optional[str] = None):
+
+def build_and_validate_settings():
     """
     Creates a settings object to be used in the application. It collects and apply generic validators and validators
     specific for each publisher, notifier and publication strategy.
@@ -86,9 +84,7 @@ def build_and_validate_settings(settings_file: Optional[str] = None):
 
     # we first do a preliminary load of the settings without validation. We will later use them to determine which
     # publishers, notifiers and strategy have been selected
-    raw_settings = build_settings(
+    raw_settings = build_settings(validators=activeness_validators)
-        settings_file=settings_file, validators=activeness_validators
-    )
 
     # we retrieve validators that are conditional. Each module will analyze the settings and decide which validators
     # need to be applied.
@@ -98,14 +94,12 @@ def build_and_validate_settings(settings_file: Optional[str] = None):
 
     # we rebuild the settings, providing all the selected validators.
     settings = build_settings(
-        settings_file,
         base_validators
         + strategy_validators
         + publisher_validators
         + notifier_validators,
     )
-    # TODO use validation control in dynaconf 3.2.0 once released
+
-    settings.validators.validate()
     return settings
 
 
@@ -115,29 +109,22 @@ def build_and_validate_settings(settings_file: Optional[str] = None):
 # The normal Dynaconf options to specify the settings files are also not a valid option because of the two steps
 # validation that prevents us to employ their mechanism to specify settings files. This could probably be reworked
 # better in the future.
-class CustomConfig:
-    _instance = None
-    _settings_file = None
 
-    def __new__(cls, settings_file: Optional[str] = None):
-        if (
-            settings_file is None and cls._settings_file is not None
-        ):  # normal access, I don't want to reload
-            return cls._instance
-
-        if (
-            cls._instance is None and cls._settings_file is None
-        ) or settings_file != cls._settings_file:
-            cls._settings_file = settings_file
-            cls._instance = super(CustomConfig, cls).__new__(cls)
-            cls.settings = build_and_validate_settings(settings_file)
 
+class CustomConfig(object):
+    @classmethod
+    def get_instance(cls):
+        if not hasattr(cls, "_instance") or cls._instance is None:
+            cls._instance = cls()
         return cls._instance
 
-    def update(self, settings_file: Optional[str] = None):
+    def __init__(self):
-        self.settings = build_and_validate_settings(settings_file)
+        self.settings = build_and_validate_settings()
+
+    @classmethod
+    def clear(cls):
+        cls._instance = None
 
 
-def get_settings(settings_file: Optional[str] = None):
+def get_settings():
-    config = CustomConfig(settings_file)
+    return CustomConfig.get_instance().settings
-    return config.settings

mobilizon_reshare/event/event_selection_strategies.py

@@ -16,26 +16,12 @@ class EventSelectionStrategy(ABC):
         published_events: List[MobilizonEvent],
         unpublished_events: List[MobilizonEvent],
     ) -> Optional[MobilizonEvent]:
-
-        if not self.is_in_publishing_window():
-            logger.info("Outside of publishing window, no event will be published.")
-            return None
         selected = self._select(published_events, unpublished_events)
         if selected:
             return selected[0]
         else:
             return None
 
-    def is_in_publishing_window(self) -> bool:
-        settings = get_settings()
-        window_beginning = settings["publishing"]["window"]["begin"]
-        window_end = settings["publishing"]["window"]["end"]
-        now_hour = arrow.now().datetime.hour
-        if window_beginning <= window_end:
-            return window_beginning <= now_hour < window_end
-        else:
-            return now_hour >= window_beginning or now_hour < window_end
-
     @abstractmethod
     def _select(
         self,
@@ -96,8 +82,7 @@ STRATEGY_NAME_TO_STRATEGY_CLASS = {"next_event": SelectNextEventStrategy}
 
 
 def select_unpublished_events(
-    published_events: List[MobilizonEvent],
+    published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent],
-    unpublished_events: List[MobilizonEvent],
 ):
 
     strategy = STRATEGY_NAME_TO_STRATEGY_CLASS[
@@ -108,8 +93,7 @@ def select_unpublished_events(
 
 
 def select_event_to_publish(
-    published_events: List[MobilizonEvent],
+    published_events: List[MobilizonEvent], unpublished_events: List[MobilizonEvent],
-    unpublished_events: List[MobilizonEvent],
 ):
 
     strategy = STRATEGY_NAME_TO_STRATEGY_CLASS[

mobilizon_reshare/publishers/platforms/telegram.py

@@ -41,6 +41,7 @@ class TelegramFormatter(AbstractEventFormatter):
         ")",
         ">",
         "<",
+        ">",
         "{",
         "}",
     ]

mobilizon_reshare/settings.toml

@@ -13,9 +13,6 @@ group="my_group"
 [default.selection]
 strategy = "next_event"
 
-[default.publishing.window]
-begin=12
-end=18
 
 [default.selection.strategy_options]
 break_between_events_in_minutes =60

tests/commands/test_start.py

@@ -32,13 +32,11 @@ async def test_start_no_event(
     "elements",
     [[simple_event_element()], [simple_event_element(), simple_event_element()]],
 )
-@pytest.mark.parametrize("publication_window", [(0, 24)])
 async def test_start_new_event(
     mock_mobilizon_success_answer,
     mobilizon_answer,
     caplog,
     mock_publisher_config,
-    mock_publication_window,
     message_collector,
     elements,
 ):
@@ -88,13 +86,11 @@ async def test_start_new_event(
 @pytest.mark.parametrize(
     "elements", [[]],
 )
-@pytest.mark.parametrize("publication_window", [(0, 24)])
 async def test_start_event_from_db(
     mock_mobilizon_success_answer,
     mobilizon_answer,
     caplog,
     mock_publisher_config,
-    mock_publication_window,
     message_collector,
     event_generator,
 ):
@@ -135,13 +131,11 @@ async def test_start_event_from_db(
 @pytest.mark.parametrize(
     "elements", [[]],
 )
-@pytest.mark.parametrize("publication_window", [(0, 24)])
 async def test_start_publisher_failure(
     mock_mobilizon_success_answer,
     mobilizon_answer,
     caplog,
     mock_publisher_config,
-    mock_publication_window,
     message_collector,
     event_generator,
     mock_notifier_config,
@@ -217,13 +211,11 @@ def second_event_element():
 @pytest.mark.parametrize(
     "elements", [[second_event_element()]],
 )
-@pytest.mark.parametrize("publication_window", [(0, 24)])
 async def test_start_second_execution(
     mock_mobilizon_success_answer,
     mobilizon_answer,
     caplog,
     mock_publisher_config,
-    mock_publication_window,
     message_collector,
     event_generator,
     published_event,

tests/config/test_config_singleton.py

@@ -1,33 +0,0 @@
-import pkg_resources
-
-from mobilizon_reshare.config.config import get_settings
-
-
-def test_singleton():
-    settings_file = pkg_resources.resource_filename(
-        "tests.resources.config", "test_singleton.toml"
-    )
-    config_1 = get_settings(settings_file)
-    config_2 = get_settings()
-    assert id(config_1) == id(config_2)
-
-
-def test_same_file():
-    settings_file = pkg_resources.resource_filename(
-        "tests.resources.config", "test_singleton.toml"
-    )
-    config_1 = get_settings(settings_file)
-    config_2 = get_settings(settings_file)
-    assert id(config_1) == id(config_2)
-
-
-def test_singleton_new_file():
-    settings_file = pkg_resources.resource_filename(
-        "tests.resources.config", "test_singleton.toml"
-    )
-    settings_file_2 = pkg_resources.resource_filename(
-        "tests.resources.config", "test_singleton_2.toml"
-    )
-    config_1 = get_settings(settings_file)
-    config_2 = get_settings(settings_file_2)
-    assert id(config_1) != id(config_2)

tests/config/test_validation.py

@@ -1,44 +0,0 @@
-import os
-
-import dynaconf
-import pkg_resources
-import pytest
-
-from mobilizon_reshare.config.config import get_settings
-
-
-@pytest.fixture
-def invalid_settings_file(tmp_path, toml_content):
-    file = tmp_path / "tmp.toml"
-    file.write_text(toml_content)
-    return file
-
-
-@pytest.mark.parametrize("toml_content", ["invalid toml["])
-def test_get_settings_failure_invalid_toml(invalid_settings_file):
-    with pytest.raises(dynaconf.vendor.toml.decoder.TomlDecodeError):
-        get_settings(invalid_settings_file.absolute())
-
-
-@pytest.mark.parametrize("toml_content", [""])
-def test_get_settings_failure_invalid_preliminary_config(invalid_settings_file):
-    os.environ["SECRETS_FOR_DYNACONF"] = ""
-
-    with pytest.raises(dynaconf.validator.ValidationError):
-        get_settings(invalid_settings_file.absolute())
-
-
-@pytest.mark.parametrize(
-    "invalid_toml,pattern_in_exception",
-    [["config_with_invalid_strategy.toml", "break_between_events_in_minutes"]],
-)
-def test_get_settings_failure_config_base_validators(
-    invalid_toml, pattern_in_exception
-):
-
-    with pytest.raises(dynaconf.validator.ValidationError) as e:
-        get_settings(
-            pkg_resources.resource_filename("tests.resources.config", invalid_toml)
-        )
-
-    assert e.match(pattern_in_exception)

tests/conftest.py

@@ -309,14 +309,6 @@ def mock_mobilizon_success_answer(mobilizon_answer, mobilizon_url):
         yield
 
 
-@pytest.fixture
-def mock_publication_window(publication_window):
-    begin, end = publication_window
-    get_settings().update(
-        {"publishing.window.begin": begin, "publishing.window.end": end}
-    )
-
-
 @pytest.fixture
 def mock_formatter_class():
     class MockFormatter(AbstractEventFormatter):

tests/event/test_strategies.py

@@ -207,39 +207,3 @@ def mock_arrow_now(current_hour):
 
     with patch("arrow.now", mock_now):
         yield
-
-
-@pytest.mark.parametrize("current_hour", [14, 15, 16, 18])
-@pytest.mark.parametrize("publication_window", [(14, 19)])
-def test_publishing_inner_window_true(mock_arrow_now, mock_publication_window):
-    """
-    Testing that the window check correctly returns True when in an inner publishing window.
-    """
-    assert SelectNextEventStrategy().is_in_publishing_window()
-
-
-@pytest.mark.parametrize("current_hour", [2, 10, 11, 19])
-@pytest.mark.parametrize("publication_window", [(14, 19)])
-def test_publishing_inner_window_false(mock_arrow_now, mock_publication_window):
-    """
-    Testing that the window check correctly returns False when not in an inner publishing window.
-    """
-    assert not SelectNextEventStrategy().is_in_publishing_window()
-
-
-@pytest.mark.parametrize("current_hour", [2, 10, 11, 19])
-@pytest.mark.parametrize("publication_window", [(19, 14)])
-def test_publishing_outer_window_true(mock_arrow_now, mock_publication_window):
-    """
-    Testing that the window check correctly returns True when in an outer publishing window.
-    """
-    assert SelectNextEventStrategy().is_in_publishing_window()
-
-
-@pytest.mark.parametrize("current_hour", [14, 15, 16, 18])
-@pytest.mark.parametrize("publication_window", [(19, 14)])
-def test_publishing_outer_window_false(mock_arrow_now, mock_publication_window):
-    """
-    Testing that the window check correctly returns False when not in an outer publishing window.
-    """
-    assert not SelectNextEventStrategy().is_in_publishing_window()

tests/resources/config/config_with_invalid_strategy.toml

@@ -2,9 +2,6 @@
 url="https://some_mobilizon"
 group="my_group"
 
-[testing.publishing.window]
-begin=12
-end=18
 
 [testing.selection]
 strategy = "next_event"

tests/resources/config/config_with_strategy.toml

@@ -2,9 +2,6 @@
 url="https://some_mobilizon"
 group="my_group"
 
-[testing.publishing.window]
-begin=12
-end=18
 
 [testing.selection]
 strategy = "next_event"

tests/resources/config/test_singleton.toml

@@ -5,9 +5,6 @@ group="my_group"
 [testing.selection]
 strategy = "next_event"
 
-[testing.publishing.window]
-begin=12
-end=18
 
 [testing.selection.strategy_options]
 break_between_events_in_minutes =60

tests/resources/config/test_singleton_2.toml

@@ -5,9 +5,6 @@ group="my_group"
 [testing.selection]
 strategy = "next_event"
 
-[testing.publishing.window]
-begin=12
-end=18
 
 [testing.selection.strategy_options]
 break_between_events_in_minutes =60