Failures (#183)

* Update beautifulsoup4 to 4.11.

* Update docker image.

* Add type annotations.

* Publish where you can and notify errors.

* Add tests for partial publishing.

channels-lock.scm

@@ -10,7 +10,7 @@
   (name 'guix)
   (url "https://git.savannah.gnu.org/git/guix.git")
   (commit
-   "42d1fa6fc4faabe9a4c327014c7289dda08ad8a2")
+   "79a3cd34c0318928186a04b6481c4d22c0051d04")
   (introduction
    (make-channel-introduction
     "afb9f2752315f131e4ddd44eba02eed403365085"

guix.scm

@@ -47,7 +47,7 @@
               (base32
                "066r7mimlpb5q1fr2f1z59l4jc89kv4h2kgkcifyqav6544w8ncq"))))))
 
-(define-public mobilizon-reshare.git
+(define _mobilizon-reshare.git
   (let ((source-version (with-input-from-file
                             (string-append %source-dir
                                            "/mobilizon_reshare/VERSION")
@@ -63,21 +63,35 @@
        (substitute-keyword-arguments (package-arguments mobilizon-reshare)
          ((#:phases phases)
           #~(modify-phases #$phases
+              (add-after 'unpack 'patch-version
+                (lambda _
+                  (with-output-to-file "mobilizon_reshare/VERSION"
+                   (lambda _
+                     (display #$version)))))
               (delete 'patch-pyproject.toml)))))
       (native-inputs
        (modify-inputs (package-native-inputs mobilizon-reshare)
-         (prepend python-httpx python-fastapi python-fastapi-pagination)))
+         (prepend python-httpx)))
       (propagated-inputs
        (modify-inputs (package-propagated-inputs mobilizon-reshare)
-         (prepend python-asyncpg python-uvicorn)
+         (prepend python-asyncpg
+                  python-uvicorn
+                  python-fastapi
+                  python-fastapi-pagination)
          (replace "python-tweepy"
                    python-tweepy-4.13)
          (replace "dynaconf"
                    dynaconf-3.1.11)
          (replace "python-markdownify"
-                   python-markdownify)
-         (replace "python-tortoise-orm"
-                   python-tortoise-orm))))))
+                   python-markdownify))))))
+
+(define-public patch-for-mobilizon-reshare-0.3.3
+  (package-input-rewriting/spec `(("python-oauthlib" . ,(const python-oauthlib-3.2))
+                                  ("python-beautifulsoup4" . ,(const python-beautifulsoup4))
+                                  ("python-tortoise-orm" . ,(const python-tortoise-orm)))))
+
+(define-public mobilizon-reshare.git
+  (patch-for-mobilizon-reshare-0.3.3 _mobilizon-reshare.git))
 
 (define-public mobilizon-reshare-scheduler
  (package (inherit mobilizon-reshare.git)
@@ -101,7 +115,4 @@
    (description "This script is intended to start a scheduler
 running @code{mobilizon-reshare}.")))
 
-(define-public patch-for-mobilizon-reshare-0.3.3
-  (package-input-rewriting/spec `(("python-oauthlib". ,(const python-oauthlib-3.2)))))
-
-(patch-for-mobilizon-reshare-0.3.3 mobilizon-reshare.git)
+mobilizon-reshare.git

mobilizon_reshare/main/publish.py

@@ -35,7 +35,7 @@ async def publish_publications(
 
     await save_publication_report(report)
     for publication_report in report.reports:
-        if not publication_report.succesful:
+        if not publication_report.successful:
             PublicationFailureNotifiersCoordinator(publication_report,).notify_failure()
 
     return report

mobilizon_reshare/publishers/coordinators/__init__.py

@@ -11,7 +11,7 @@ class BasePublicationReport:
     reason: Optional[str]
 
     @property
-    def succesful(self):
+    def successful(self):
         return self.status == PublicationStatus.COMPLETED
 
     def get_failure_message(self):

mobilizon_reshare/publishers/coordinators/event_publishing/__init__.py

@@ -1,7 +1,7 @@
 import dataclasses
 import logging
 from dataclasses import dataclass
-from typing import List, Optional
+from typing import List, Optional, Sequence
 
 from mobilizon_reshare.dataclasses.publication import _EventPublication
 from mobilizon_reshare.models.publication import PublicationStatus
@@ -38,7 +38,7 @@ class BaseEventPublishingCoordinator:
         except Exception as e:
             return reasons + [str(e)]
 
-    def _validate(self):
+    def _validate(self) -> List[EventPublicationReport]:
         errors = []
 
         for publication in self.publications:
@@ -60,3 +60,7 @@ class BaseEventPublishingCoordinator:
                 )
 
         return errors
+
+    def _filter_publications(self, errors: Sequence[EventPublicationReport]) -> List[_EventPublication]:
+        publishers_with_errors = set(e.publication.publisher for e in errors)
+        return [p for p in self.publications if p.publisher not in publishers_with_errors]

mobilizon_reshare/publishers/coordinators/event_publishing/dry_run.py

@@ -1,26 +1,20 @@
+from typing import List, Sequence
+
+from mobilizon_reshare.dataclasses import _EventPublication
 from mobilizon_reshare.models.publication import PublicationStatus
-from mobilizon_reshare.publishers.coordinators.event_publishing import (
-    BaseEventPublishingCoordinator,
-)
 from mobilizon_reshare.publishers.coordinators.event_publishing.publish import (
-    PublisherCoordinatorReport,
+    PublisherCoordinator,
     EventPublicationReport,
 )
 
 
-class DryRunPublisherCoordinator(BaseEventPublishingCoordinator):
+class DryRunPublisherCoordinator(PublisherCoordinator):
     """
     Coordinator to perform a dry-run on the event publication
     """
 
-    def run(self) -> PublisherCoordinatorReport:
-        errors = self._validate()
-        if errors:
-            coord_report = PublisherCoordinatorReport(
-                reports=errors, publications=self.publications
-            )
-        else:
-            reports = [
+    def _publish(self, publications: Sequence[_EventPublication]) -> List[EventPublicationReport]:
+        return [
             EventPublicationReport(
                 status=PublicationStatus.COMPLETED,
                 publication=publication,
@@ -29,10 +23,5 @@ class DryRunPublisherCoordinator(BaseEventPublishingCoordinator):
                     publication.event
                 ),
             )
-                for publication in self.publications
+            for publication in publications
         ]
-            coord_report = PublisherCoordinatorReport(
-                publications=self.publications, reports=reports
-            )
-
-        return coord_report

mobilizon_reshare/publishers/coordinators/event_publishing/publish.py

@@ -1,7 +1,7 @@
 import dataclasses
 import logging
 from dataclasses import dataclass
-from typing import Sequence
+from typing import Sequence, List
 
 from mobilizon_reshare.dataclasses.publication import _EventPublication
 from mobilizon_reshare.models.publication import PublicationStatus
@@ -39,18 +39,18 @@ class PublisherCoordinator(BaseEventPublishingCoordinator):
     """
 
     def run(self) -> PublisherCoordinatorReport:
-        errors = self._validate()
-        if errors:
+        validation_reports = self._validate()
+        valid_publications = self._filter_publications(validation_reports)
+        publishing_reports = self._publish(valid_publications)
         return PublisherCoordinatorReport(
-                reports=errors, publications=self.publications
+            publications=self.publications,
+            reports=validation_reports + publishing_reports
         )
 
-        return self._publish()
-
-    def _publish(self) -> PublisherCoordinatorReport:
+    def _publish(self, publications: Sequence[_EventPublication]) -> List[EventPublicationReport]:
         reports = []
 
-        for publication in self.publications:
+        for publication in publications:
 
             try:
                 publication_report = self._publish_publication(publication)
@@ -65,9 +65,7 @@ class PublisherCoordinator(BaseEventPublishingCoordinator):
                     )
                 )
 
-        return PublisherCoordinatorReport(
-            publications=self.publications, reports=reports
-        )
+        return reports
 
     @staticmethod
     def _publish_publication(publication):

mobilizon_reshare/storage/query/write.py

@@ -22,7 +22,7 @@ from mobilizon_reshare.storage.query.read import get_event
 
 @atomic()
 async def upsert_publication(
-    publication_report: EventPublicationReport, event: MobilizonEvent
+    publication_report: EventPublicationReport, event: Event
 ):
 
     publisher_model = await (

poetry.lock

@@ -137,11 +137,11 @@ python-versions = ">=3.7"
 
 [[package]]
 name = "beautifulsoup4"
-version = "4.10.0"
+version = "4.11.2"
 description = "Screen-scraping library"
 category = "main"
 optional = false
-python-versions = ">3.0.0"
+python-versions = ">=3.6.0"
 
 [package.dependencies]
 soupsieve = ">1.2"
@@ -152,7 +152,7 @@ lxml = ["lxml"]
 
 [[package]]
 name = "certifi"
-version = "2022.12.7"
+version = "2023.5.7"
 description = "Python package for providing Mozilla's CA Bundle."
 category = "main"
 optional = false
@@ -187,7 +187,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7
 
 [[package]]
 name = "coverage"
-version = "7.2.3"
+version = "7.2.5"
 description = "Code coverage measurement for Python"
 category = "dev"
 optional = false
@@ -368,7 +368,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
 [[package]]
 name = "importlib-metadata"
-version = "6.4.1"
+version = "6.6.0"
 description = "Read metadata from Python packages"
 category = "dev"
 optional = false
@@ -378,9 +378,9 @@ python-versions = ">=3.7"
 zipp = ">=0.5"
 
 [package.extras]
-testing = ["packaging", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"]
 docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
 perf = ["ipython"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"]
 
 [[package]]
 name = "iniconfig"
@@ -515,7 +515,7 @@ email = ["email-validator (>=1.0.3)"]
 
 [[package]]
 name = "pygments"
-version = "2.15.0"
+version = "2.15.1"
 description = "Pygments is a syntax highlighting package written in Python."
 category = "dev"
 optional = false
@@ -911,7 +911,7 @@ python-versions = ">=3.7"
 
 [[package]]
 name = "tomlkit"
-version = "0.11.7"
+version = "0.11.8"
 description = "Style preserving TOML library"
 category = "main"
 optional = false
@@ -938,7 +938,7 @@ asyncmy = ["asyncmy (>=0.2.5,<0.3.0)"]
 asyncodbc = ["asyncodbc (>=0.1.1,<0.2.0)"]
 asyncpg = ["asyncpg"]
 accel = ["ciso8601", "orjson", "uvloop"]
-psycopg = ["psycopg[binary,pool] (==3.0.12)"]
+psycopg = ["psycopg[pool,binary] (==3.0.12)"]
 
 [[package]]
 name = "tweepy"
@@ -1020,7 +1020,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-co
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "8ea9b23f8f89fba762ef813aca3eb5ab2d8ecf81197895e954db750f3c06f388"
+content-hash = "bfc1512cd6f94fdc013dbebcf70c0077093b2bc3126c8573c35a3569445f948d"
 
 [metadata.files]
 aerich = []

pyproject.toml

@@ -17,7 +17,7 @@ Jinja2 = "~3.1"
 requests = "~2.28"
 arrow = "~1.1"
 click = "~8.1"
-beautifulsoup4 = "~4.10"
+beautifulsoup4 = "~4.11"
 markdownify = "~0.10"
 appdirs = "~1.4"
 tweepy = "~4.13"

scripts/build_docker_image.sh

@@ -3,4 +3,4 @@ set -eu
 
 guix time-machine -C channels-lock.scm -- build -f guix.scm
 
-guix time-machine -C channels-lock.scm -- pack -L . -f docker --save-provenance --root=docker-image.tar.gz --entry-point=bin/scheduler.py mobilizon-reshare-scheduler
+guix time-machine -C channels-lock.scm -- pack -L . -f docker -S /opt/bin=bin --save-provenance --root=docker-image.tar.gz --entry-point=bin/scheduler.py mobilizon-reshare-scheduler

tests/publishers/test_coordinator.py

@@ -138,6 +138,24 @@ async def test_publication_coordinator_run_failure(
     assert list(report.reports)[0].reason == "credentials error, Invalid event error"
 
 
+@pytest.mark.parametrize("num_publications", [2])
+@pytest.mark.asyncio
+async def test_publication_coordinator_run_partial_failure(
+    mock_publications, mock_publisher_invalid, mock_formatter_invalid
+):
+
+    mock_publications[0].publisher = mock_publisher_invalid
+    mock_publications[0].formatter = mock_formatter_invalid
+    coordinator = PublisherCoordinator(mock_publications)
+
+    report = coordinator.run()
+    assert len(report.reports) == 2
+    assert not list(report.reports)[0].successful
+    assert list(report.reports)[0].reason == "credentials error, Invalid event error"
+    assert list(report.reports)[1].successful
+    assert list(report.reports)[1].reason is None
+
+
 @pytest.mark.parametrize("num_publications", [1])
 @pytest.mark.asyncio
 async def test_publication_coordinator_run_failure_response(