diff --git a/channels-lock.scm b/channels-lock.scm index ac25cd9..3345467 100644 --- a/channels-lock.scm +++ b/channels-lock.scm @@ -10,7 +10,7 @@ (name 'guix) (url "https://git.savannah.gnu.org/git/guix.git") (commit - "42d1fa6fc4faabe9a4c327014c7289dda08ad8a2") + "79a3cd34c0318928186a04b6481c4d22c0051d04") (introduction (make-channel-introduction "afb9f2752315f131e4ddd44eba02eed403365085" diff --git a/guix.scm b/guix.scm index 09536cc..cb89e7c 100644 --- a/guix.scm +++ b/guix.scm @@ -47,7 +47,7 @@ (base32 "066r7mimlpb5q1fr2f1z59l4jc89kv4h2kgkcifyqav6544w8ncq")))))) -(define-public mobilizon-reshare.git +(define _mobilizon-reshare.git (let ((source-version (with-input-from-file (string-append %source-dir "/mobilizon_reshare/VERSION") @@ -63,21 +63,35 @@ (substitute-keyword-arguments (package-arguments mobilizon-reshare) ((#:phases phases) #~(modify-phases #$phases + (add-after 'unpack 'patch-version + (lambda _ + (with-output-to-file "mobilizon_reshare/VERSION" + (lambda _ + (display #$version))))) (delete 'patch-pyproject.toml))))) (native-inputs (modify-inputs (package-native-inputs mobilizon-reshare) - (prepend python-httpx python-fastapi python-fastapi-pagination))) + (prepend python-httpx))) (propagated-inputs (modify-inputs (package-propagated-inputs mobilizon-reshare) - (prepend python-asyncpg python-uvicorn) + (prepend python-asyncpg + python-uvicorn + python-fastapi + python-fastapi-pagination) (replace "python-tweepy" python-tweepy-4.13) (replace "dynaconf" dynaconf-3.1.11) (replace "python-markdownify" - python-markdownify) - (replace "python-tortoise-orm" - python-tortoise-orm)))))) + python-markdownify)))))) + +(define-public patch-for-mobilizon-reshare-0.3.3 + (package-input-rewriting/spec `(("python-oauthlib" . ,(const python-oauthlib-3.2)) + ("python-beautifulsoup4" . ,(const python-beautifulsoup4)) + ("python-tortoise-orm" . ,(const python-tortoise-orm))))) + +(define-public mobilizon-reshare.git + (patch-for-mobilizon-reshare-0.3.3 _mobilizon-reshare.git)) (define-public mobilizon-reshare-scheduler (package (inherit mobilizon-reshare.git) @@ -101,7 +115,4 @@ (description "This script is intended to start a scheduler running @code{mobilizon-reshare}."))) -(define-public patch-for-mobilizon-reshare-0.3.3 - (package-input-rewriting/spec `(("python-oauthlib". ,(const python-oauthlib-3.2))))) - -(patch-for-mobilizon-reshare-0.3.3 mobilizon-reshare.git) +mobilizon-reshare.git diff --git a/mobilizon_reshare/main/publish.py b/mobilizon_reshare/main/publish.py index d6fa7d6..ea318ec 100644 --- a/mobilizon_reshare/main/publish.py +++ b/mobilizon_reshare/main/publish.py @@ -35,7 +35,7 @@ async def publish_publications( await save_publication_report(report) for publication_report in report.reports: - if not publication_report.succesful: + if not publication_report.successful: PublicationFailureNotifiersCoordinator(publication_report,).notify_failure() return report diff --git a/mobilizon_reshare/publishers/coordinators/__init__.py b/mobilizon_reshare/publishers/coordinators/__init__.py index 6c3a838..ee54ee6 100644 --- a/mobilizon_reshare/publishers/coordinators/__init__.py +++ b/mobilizon_reshare/publishers/coordinators/__init__.py @@ -11,7 +11,7 @@ class BasePublicationReport: reason: Optional[str] @property - def succesful(self): + def successful(self): return self.status == PublicationStatus.COMPLETED def get_failure_message(self): diff --git a/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py b/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py index 1aae03a..454c323 100644 --- a/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py +++ b/mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py @@ -1,7 +1,7 @@ import dataclasses import logging from dataclasses import dataclass -from typing import List, Optional +from typing import List, Optional, Sequence from mobilizon_reshare.dataclasses.publication import _EventPublication from mobilizon_reshare.models.publication import PublicationStatus @@ -38,7 +38,7 @@ class BaseEventPublishingCoordinator: except Exception as e: return reasons + [str(e)] - def _validate(self): + def _validate(self) -> List[EventPublicationReport]: errors = [] for publication in self.publications: @@ -60,3 +60,7 @@ class BaseEventPublishingCoordinator: ) return errors + + def _filter_publications(self, errors: Sequence[EventPublicationReport]) -> List[_EventPublication]: + publishers_with_errors = set(e.publication.publisher for e in errors) + return [p for p in self.publications if p.publisher not in publishers_with_errors] diff --git a/mobilizon_reshare/publishers/coordinators/event_publishing/dry_run.py b/mobilizon_reshare/publishers/coordinators/event_publishing/dry_run.py index 517d005..4d9308b 100644 --- a/mobilizon_reshare/publishers/coordinators/event_publishing/dry_run.py +++ b/mobilizon_reshare/publishers/coordinators/event_publishing/dry_run.py @@ -1,38 +1,27 @@ +from typing import List, Sequence + +from mobilizon_reshare.dataclasses import _EventPublication from mobilizon_reshare.models.publication import PublicationStatus -from mobilizon_reshare.publishers.coordinators.event_publishing import ( - BaseEventPublishingCoordinator, -) from mobilizon_reshare.publishers.coordinators.event_publishing.publish import ( - PublisherCoordinatorReport, + PublisherCoordinator, EventPublicationReport, ) -class DryRunPublisherCoordinator(BaseEventPublishingCoordinator): +class DryRunPublisherCoordinator(PublisherCoordinator): """ Coordinator to perform a dry-run on the event publication """ - def run(self) -> PublisherCoordinatorReport: - errors = self._validate() - if errors: - coord_report = PublisherCoordinatorReport( - reports=errors, publications=self.publications + def _publish(self, publications: Sequence[_EventPublication]) -> List[EventPublicationReport]: + return [ + EventPublicationReport( + status=PublicationStatus.COMPLETED, + publication=publication, + reason=None, + published_content=publication.formatter.get_message_from_event( + publication.event + ), ) - else: - reports = [ - EventPublicationReport( - status=PublicationStatus.COMPLETED, - publication=publication, - reason=None, - published_content=publication.formatter.get_message_from_event( - publication.event - ), - ) - for publication in self.publications - ] - coord_report = PublisherCoordinatorReport( - publications=self.publications, reports=reports - ) - - return coord_report + for publication in publications + ] diff --git a/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py b/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py index 58514b2..bd02057 100644 --- a/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py +++ b/mobilizon_reshare/publishers/coordinators/event_publishing/publish.py @@ -1,7 +1,7 @@ import dataclasses import logging from dataclasses import dataclass -from typing import Sequence +from typing import Sequence, List from mobilizon_reshare.dataclasses.publication import _EventPublication from mobilizon_reshare.models.publication import PublicationStatus @@ -39,18 +39,18 @@ class PublisherCoordinator(BaseEventPublishingCoordinator): """ def run(self) -> PublisherCoordinatorReport: - errors = self._validate() - if errors: - return PublisherCoordinatorReport( - reports=errors, publications=self.publications - ) + validation_reports = self._validate() + valid_publications = self._filter_publications(validation_reports) + publishing_reports = self._publish(valid_publications) + return PublisherCoordinatorReport( + publications=self.publications, + reports=validation_reports + publishing_reports + ) - return self._publish() - - def _publish(self) -> PublisherCoordinatorReport: + def _publish(self, publications: Sequence[_EventPublication]) -> List[EventPublicationReport]: reports = [] - for publication in self.publications: + for publication in publications: try: publication_report = self._publish_publication(publication) @@ -65,9 +65,7 @@ class PublisherCoordinator(BaseEventPublishingCoordinator): ) ) - return PublisherCoordinatorReport( - publications=self.publications, reports=reports - ) + return reports @staticmethod def _publish_publication(publication): diff --git a/mobilizon_reshare/storage/query/write.py b/mobilizon_reshare/storage/query/write.py index ef662d1..dbcb862 100644 --- a/mobilizon_reshare/storage/query/write.py +++ b/mobilizon_reshare/storage/query/write.py @@ -22,7 +22,7 @@ from mobilizon_reshare.storage.query.read import get_event @atomic() async def upsert_publication( - publication_report: EventPublicationReport, event: MobilizonEvent + publication_report: EventPublicationReport, event: Event ): publisher_model = await ( diff --git a/poetry.lock b/poetry.lock index fe8a4cb..2813997 100644 --- a/poetry.lock +++ b/poetry.lock @@ -137,11 +137,11 @@ python-versions = ">=3.7" [[package]] name = "beautifulsoup4" -version = "4.10.0" +version = "4.11.2" description = "Screen-scraping library" category = "main" optional = false -python-versions = ">3.0.0" +python-versions = ">=3.6.0" [package.dependencies] soupsieve = ">1.2" @@ -152,7 +152,7 @@ lxml = ["lxml"] [[package]] name = "certifi" -version = "2022.12.7" +version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -187,7 +187,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7 [[package]] name = "coverage" -version = "7.2.3" +version = "7.2.5" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -368,7 +368,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "6.4.1" +version = "6.6.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -378,9 +378,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -testing = ["packaging", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -515,7 +515,7 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" -version = "2.15.0" +version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false @@ -911,7 +911,7 @@ python-versions = ">=3.7" [[package]] name = "tomlkit" -version = "0.11.7" +version = "0.11.8" description = "Style preserving TOML library" category = "main" optional = false @@ -938,7 +938,7 @@ asyncmy = ["asyncmy (>=0.2.5,<0.3.0)"] asyncodbc = ["asyncodbc (>=0.1.1,<0.2.0)"] asyncpg = ["asyncpg"] accel = ["ciso8601", "orjson", "uvloop"] -psycopg = ["psycopg[binary,pool] (==3.0.12)"] +psycopg = ["psycopg[pool,binary] (==3.0.12)"] [[package]] name = "tweepy" @@ -1020,7 +1020,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-co [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "8ea9b23f8f89fba762ef813aca3eb5ab2d8ecf81197895e954db750f3c06f388" +content-hash = "bfc1512cd6f94fdc013dbebcf70c0077093b2bc3126c8573c35a3569445f948d" [metadata.files] aerich = [] diff --git a/pyproject.toml b/pyproject.toml index 910657a..c3af406 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ Jinja2 = "~3.1" requests = "~2.28" arrow = "~1.1" click = "~8.1" -beautifulsoup4 = "~4.10" +beautifulsoup4 = "~4.11" markdownify = "~0.10" appdirs = "~1.4" tweepy = "~4.13" diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh index 8bec820..9e3cd67 100755 --- a/scripts/build_docker_image.sh +++ b/scripts/build_docker_image.sh @@ -3,4 +3,4 @@ set -eu guix time-machine -C channels-lock.scm -- build -f guix.scm -guix time-machine -C channels-lock.scm -- pack -L . -f docker --save-provenance --root=docker-image.tar.gz --entry-point=bin/scheduler.py mobilizon-reshare-scheduler +guix time-machine -C channels-lock.scm -- pack -L . -f docker -S /opt/bin=bin --save-provenance --root=docker-image.tar.gz --entry-point=bin/scheduler.py mobilizon-reshare-scheduler diff --git a/settings.toml b/settings.toml deleted file mode 100755 index e69de29..0000000 diff --git a/tests/publishers/test_coordinator.py b/tests/publishers/test_coordinator.py index 5a1008e..a305807 100644 --- a/tests/publishers/test_coordinator.py +++ b/tests/publishers/test_coordinator.py @@ -138,6 +138,24 @@ async def test_publication_coordinator_run_failure( assert list(report.reports)[0].reason == "credentials error, Invalid event error" +@pytest.mark.parametrize("num_publications", [2]) +@pytest.mark.asyncio +async def test_publication_coordinator_run_partial_failure( + mock_publications, mock_publisher_invalid, mock_formatter_invalid +): + + mock_publications[0].publisher = mock_publisher_invalid + mock_publications[0].formatter = mock_formatter_invalid + coordinator = PublisherCoordinator(mock_publications) + + report = coordinator.run() + assert len(report.reports) == 2 + assert not list(report.reports)[0].successful + assert list(report.reports)[0].reason == "credentials error, Invalid event error" + assert list(report.reports)[1].successful + assert list(report.reports)[1].reason is None + + @pytest.mark.parametrize("num_publications", [1]) @pytest.mark.asyncio async def test_publication_coordinator_run_failure_response(