diff --git a/channels-lock.scm b/channels-lock.scm index 2a03d6a..7047e2e 100644 --- a/channels-lock.scm +++ b/channels-lock.scm @@ -8,6 +8,6 @@ "1121fa432f0e6422d5f9ebb96fb0014c4d5231f5") (introduction (make-channel-introduction - "41000d16c5c1586482a76d856c3152a6b8fcce8a" + "afb9f2752315f131e4ddd44eba02eed403365085" (openpgp-fingerprint "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA"))))) diff --git a/doc/dependency-hell.md b/doc/dependency-hell.md new file mode 100644 index 0000000..b9ee2d9 --- /dev/null +++ b/doc/dependency-hell.md @@ -0,0 +1,76 @@ +# Beating dependency hell with GNU Guix + +`mobilizon-reshare`'s distribution process relies quite a bit upon GNU Guix. It's involved in our CI pipeline, and it builds the [OCI compliant](https://opencontainers.org/) container image available on Docker Hub. It provides us with [inspectable](https://hpc.guix.info/blog/2021/10/when-docker-images-become-fixed-point/), [bit-for-bit reproducible](https://reproducible-builds.org/) and [fully bootstrappable](https://bootstrappable.org) images which in turns allows for strong control on what code is actually bundled within the image and it should prevent entire classes of supply-chain attacks starting from the [Trusting Trust problem](https://www.cs.cmu.edu/~rdriley/487/papers/Thompson_1984_ReflectionsonTrustingTrust.pdf) up until the many recent [attacks](https://www.sonatype.com/resources/state-of-the-software-supply-chain-2021) to many FOSS software registries. + +To allow for interoperability with the Python ecosystem, we also ship a `pyproject.toml` that we handle with [Poetry](https://python-poetry.org/). The next paragraph will elaborate on the interactions between Poetry and Guix. + +## Update the dependency graph of mobilizon-reshare + +> **Beware!** - Dependency updates are better delivered to master as a single commit, to avoid confusing the CI. + +### Python dependencies + +We **must** keep Poetry and Guix version as much aligned as possible, to prevent unpredictable behavior. All the following content assumes this invariant. + +Everything starts from `pyproject.toml`: usually your IDE warns you about outdated dependency, so let's assume you want to bump the version of a Python package. First keep in mind that Poetry's [tilde requirements](https://python-poetry.org/docs/dependency-specification/#tilde-requirements) are SemVer compatible but stricter than caret requirements so they should make matching Guix version easier. Then it's time to actually edit `pyproject.toml` and bump the version of a package. + +To update Python dependencies and test your changes the steps are: + +```shell +$ poetry update +$ guix time-machine -C channels-lock.scm -- build -L . mobilizon-reshare.git +$ scripts/build_docker_image.sh +``` + +If these steps succeed you can safely commit your changes. If Guix fails you have to examine the output of the command that failed and figure out the problem. 99% of the times it'll be a version mismatch, as Guix's [`python-build-system`](https://guix.gnu.org/en/manual/devel/en/guix.html#index-python_002dbuild_002dsystem) has a `sanity-check` phase that'll try to instantiate the entry point generated by Poetry that, among other things, checks for runtime dependencies versions and errors out if it finds a mismatch between the version actually available in the runtime environment and the version defined in `pyproject.toml`. + +You now have two alternatives: + +1. You try to follow the next step about system dependencies. `channels-lock.scm` locks everything: as long as the Guix commit specified in that file does not change, `guix time-machine` will look for the exact same package graph. This means that every time we build the image we get the same exact dependencies we ask for, but this semantics is slightly different from Poetry's lock-file which instead tracks the **latest version** (within the constraints) available on Pypi. Having a more updated Guix version may allow for a more updated mapping of Pypi. +2. You find the package (or packages) responsible for the mismatch and try to manipulate it to follow Poetry's constraints. This requires some basic Scheme understanding but nothing complex. There are many ways a Guix package can be programmatically manipulated as it's just a structured Scheme record, you can start by looking into [package variants](https://guix.gnu.org/en/manual/devel/en/guix.html#Defining-Package-Variants) or also directly at `docker/mobilizon-reshare.scm`. + +### System dependencies + +Python's own dependencies are dependencies too! Guix freezes the whole dependency graph of an artifact with [channels specifications](https://guix.gnu.org/en/manual/devel/en/guix.html#Replicating-Guix) so to update "system" dependencies you need to follow these steps. + +First let's update our Guix version to the latest commit: + +```shell +$ guix pull +Updating channel 'guix' from Git repository at 'https://git.savannah.gnu.org/git/guix.git'... +Authenticating channel 'guix', commits 9edb3f6 to d41c82b (162 new commits)... +Building from these channels: + guix https://git.savannah.gnu.org/git/guix.git d41c82b +substitute: updating substitutes from 'https://ci.guix.gnu.org'... 100.0%\ + +[...] + +building package cache... +building profile with 3 packages... +$ +``` + +Channels specification define the Guix commit that should be used to fetch the right dependency graph, so what we want to do is replace the commit in `channels-lock.scm` with the one we just pulled: + +```shell +$ guix describe +Generation 31 Mar 12 2022 12:35:00 (current) + guix d41c82b + repository URL: https://git.savannah.gnu.org/git/guix.git + branch: master + commit: d41c82b481fd0f5c7d45d6e2629fdf9d2085205b + +$ vim channels-lock.scm +``` + +To test our change we can run: + +```shell +$ guix time-machine -C channels-lock.scm -- build -L . mobilizon-reshare.git +``` + +But a better test would be to build the Docker image, as that actually bundles all required runtime dependencies: + +```shell +$ scripts/build_docker_image.sh +``` \ No newline at end of file diff --git a/docker/mobilizon-reshare.scm b/docker/mobilizon-reshare.scm index 5e075bc..c8ae3be 100644 --- a/docker/mobilizon-reshare.scm +++ b/docker/mobilizon-reshare.scm @@ -30,38 +30,6 @@ "https://wiki.coopcycle.org/en:license" "Coopyleft License"))) -(define-public python-tweepy - (package - (name "python-tweepy") - (version "4.1.0") - (source - (origin - (method git-fetch) - (uri - (git-reference - (url "https://github.com/tweepy/tweepy") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1c0paxc38i5jq8i20f9xwv966sap4nnhgnbdxg3611pllnzg5wdv")))) - (build-system python-build-system) - (arguments - `(#:phases - (modify-phases %standard-phases - (replace 'check - (lambda* (#:key tests? #:allow-other-keys) - (when tests? - (invoke "python" "-m" "unittest"))))))) - (propagated-inputs - (list python-aiohttp python-requests python-requests-oauthlib)) - (native-inputs - (list python-coveralls python-tox python-vcrpy)) - (home-page "https://www.tweepy.org/") - (synopsis "Twitter library for Python") - (description "Twitter library for Python") - (license license:expat))) - (define-public python-facebook-sdk (package (name "python-facebook-sdk") @@ -290,16 +258,6 @@ simplify testing of asynchronous tornado applications.") (description "We have made you a wrapper you can't refuse") (license #f))) -(define-public python-requests-2.25 - (package (inherit python-requests) - (version "2.25.1") - (source - (origin - (method url-fetch) - (uri (pypi-uri "requests" version)) - (sha256 - (base32 "015qflyqsgsz09gnar69s6ga74ivq5kch69s4qxz3904m7a3v5r7")))))) - (define-public python-click-8.0 (package (inherit python-click) (version "8.0.3") @@ -313,9 +271,6 @@ simplify testing of asynchronous tornado applications.") (define click-8-instead-of-click-7 (package-input-rewriting/spec `(("python-click" . ,(const python-click-8.0))))) -(define requests-2.25-instead-of-requests-2.26 - (package-input-rewriting/spec `(("python-requests" . ,(const python-requests-2.25))))) - (define-public mobilizon-reshare.git (let ((source-version (with-input-from-file (string-append %source-dir @@ -370,12 +325,12 @@ simplify testing of asynchronous tornado applications.") python-beautifulsoup4 python-click-8.0 (click-8-instead-of-click-7 dynaconf) - (requests-2.25-instead-of-requests-2.26 python-facebook-sdk) + python-facebook-sdk python-jinja2 python-markdownify - python-requests-2.25 + python-requests python-telegram-bot - (requests-2.25-instead-of-requests-2.26 python-tweepy) + python-tweepy python-tortoise-orm-0.18.1)) (home-page "https://github.com/Tech-Workers-Coalition-Italia/mobilizon-reshare") diff --git a/poetry.lock b/poetry.lock index cb5556b..308b886 100644 --- a/poetry.lock +++ b/poetry.lock @@ -154,12 +154,15 @@ optional = false python-versions = "*" [[package]] -name = "chardet" -version = "4.0.0" -description = "Universal encoding detector for Python 2 and 3" +name = "charset-normalizer" +version = "2.0.12" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] [[package]] name = "click" @@ -262,11 +265,11 @@ requests = "*" [[package]] name = "idna" -version = "2.10" +version = "3.3" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.5" [[package]] name = "imagesize" @@ -278,7 +281,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.11.1" +version = "4.11.2" description = "Read metadata from Python packages" category = "dev" optional = false @@ -288,7 +291,7 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] @@ -336,20 +339,6 @@ html5 = ["html5lib"] htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] -[[package]] -name = "mako" -version = "1.1.6" -description = "A super-fast templating language that borrows the best ideas from the existing templating languages." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html5 = ["html5lib"] -htmlsoup = ["beautifulsoup4"] -source = ["Cython (>=0.29.7)"] - [[package]] name = "markdownify" version = "0.10.3" @@ -526,7 +515,7 @@ six = ">=1.5" [[package]] name = "python-slugify" -version = "6.1.0" +version = "6.1.1" description = "A Python slugify application that also handles Unicode" category = "dev" optional = false @@ -580,21 +569,21 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} [[package]] name = "requests" -version = "2.25.1" +version = "2.26.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<5" -idna = ">=2.5,<3" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "requests-oauthlib" @@ -853,7 +842,7 @@ accel = ["ciso8601", "orjson", "uvloop"] [[package]] name = "tweepy" -version = "4.1.0" +version = "4.4.0" description = "Twitter library for Python" category = "main" optional = false @@ -937,7 +926,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "cf98937ea4203a9ddd996c17373db4a357e3936ac163ccb4eaef86b9d40e3fe1" +content-hash = "a0c4bd103adac3bffbec1b01205c6b339eea499755731851ddd4f296bee1aab8" [metadata.files] aerich = [ @@ -992,9 +981,9 @@ certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, ] -chardet = [ - {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, - {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, +charset-normalizer = [ + {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, + {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, ] click = [ {file = "click-8.0.4-py3-none-any.whl", hash = "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1"}, @@ -1073,16 +1062,16 @@ facebook-sdk = [ {file = "facebook_sdk-3.1.0-py2.py3-none-any.whl", hash = "sha256:2e987b3e0f466a6f4ee77b935eb023dba1384134f004a2af21f1cfff7fe0806e"}, ] idna = [ - {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, - {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] imagesize = [ {file = "imagesize-1.3.0-py2.py3-none-any.whl", hash = "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c"}, {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.11.1-py3-none-any.whl", hash = "sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094"}, - {file = "importlib_metadata-4.11.1.tar.gz", hash = "sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c"}, + {file = "importlib_metadata-4.11.2-py3-none-any.whl", hash = "sha256:d16e8c1deb60de41b8e8ed21c1a7b947b0bc62fab7e1d470bcdf331cea2e6735"}, + {file = "importlib_metadata-4.11.2.tar.gz", hash = "sha256:b36ffa925fe3139b2f6ff11d6925ffd4fa7bc47870165e3ac260ac7b4f91e6ac"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1159,14 +1148,6 @@ lxml = [ {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8b99ec73073b37f9ebe8caf399001848fced9c08064effdbfc4da2b5a8d07b93"}, {file = "lxml-4.8.0.tar.gz", hash = "sha256:f63f62fc60e6228a4ca9abae28228f35e1bd3ce675013d1dfb828688d50c6e23"}, ] -mako = [ - {file = "Mako-1.1.6-py2.py3-none-any.whl", hash = "sha256:afaf8e515d075b22fad7d7b8b30e4a1c90624ff2f3733a06ec125f5a5f043a57"}, - {file = "Mako-1.1.6.tar.gz", hash = "sha256:4e9e345a41924a954251b95b4b28e14a301145b544901332e658907a7464b6b2"}, -] -markdown = [ - {file = "Markdown-3.3.6-py3-none-any.whl", hash = "sha256:9923332318f843411e9932237530df53162e29dc7a4e2b91e35764583c46c9a3"}, - {file = "Markdown-3.3.6.tar.gz", hash = "sha256:76df8ae32294ec39dcf89340382882dfa12975f87f45c3ed1ecdb1e8cefc7006"}, -] markdownify = [ {file = "markdownify-0.10.3-py3-none-any.whl", hash = "sha256:edad0ad3896ec7460d05537ad804bbb3614877c6cd0df27b56dee218236d9ce2"}, {file = "markdownify-0.10.3.tar.gz", hash = "sha256:782e310390cd5e4bde7543ceb644598c78b9824ee9f8d7ef9f9f4f8782e46974"}, @@ -1266,8 +1247,8 @@ python-dateutil = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] python-slugify = [ - {file = "python-slugify-6.1.0.tar.gz", hash = "sha256:eff190e4dfac97d2f8c1890ee682709ecd23650742361687db82d95e1e5e25f5"}, - {file = "python_slugify-6.1.0-py2.py3-none-any.whl", hash = "sha256:2e3fad0bf38b11514f8de911ea04e7a6c6a08bb1bac18abd96d9566c34404d56"}, + {file = "python-slugify-6.1.1.tar.gz", hash = "sha256:00003397f4e31414e922ce567b3a4da28cf1436a53d332c9aeeb51c7d8c469fd"}, + {file = "python_slugify-6.1.1-py2.py3-none-any.whl", hash = "sha256:8c0016b2d74503eb64761821612d58fcfc729493634b1eb0575d8f5b4aa1fbcf"}, ] python-telegram-bot = [ {file = "python-telegram-bot-13.10.tar.gz", hash = "sha256:d2c555431821f4ace0c1b7ce12af41999f01b793b275dee131f1034d08c01e3e"}, @@ -1282,8 +1263,8 @@ pytz-deprecation-shim = [ {file = "pytz_deprecation_shim-0.1.0.post0.tar.gz", hash = "sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d"}, ] requests = [ - {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, - {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, ] requests-oauthlib = [ {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, @@ -1405,8 +1386,8 @@ tortoise-orm = [ {file = "tortoise_orm-0.18.1-py3-none-any.whl", hash = "sha256:edc9f3b49635b1dd74f73de38f54e031377e4f02b3698322502047f2e031af8b"}, ] tweepy = [ - {file = "tweepy-4.1.0-py2.py3-none-any.whl", hash = "sha256:42c63f5ee2210a8afc7178c74a6d800ef5911b007ad19e774d75dec4b777993e"}, - {file = "tweepy-4.1.0.tar.gz", hash = "sha256:88e2938de5ac7043c9ba8b8358996fbc5806059d63c96269d22527a40ca7d511"}, + {file = "tweepy-4.4.0-py2.py3-none-any.whl", hash = "sha256:cf02c4fbbd027fbc7172c24d03f53f061329ac040b22d201e59592a1cff86364"}, + {file = "tweepy-4.4.0.tar.gz", hash = "sha256:8d4b4520271b796fa7efc4c5d5ef3228af4d79f6a4d3ace3900b2778ed8f6f1c"}, ] typing-extensions = [ {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, diff --git a/pyproject.toml b/pyproject.toml index c816bf4..1cd9988 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,13 +14,13 @@ dynaconf = "~3.1" tortoise-orm = "~0.18" aiosqlite = "~0.17" Jinja2 = "~3.0" -requests = "~2.25" +requests = "~2.26" arrow = "~1.1" click = "~8.0" beautifulsoup4 = "~4.10" markdownify = "~0.10" appdirs = "~1.4" -tweepy = "~4.1" +tweepy = "~4.4" facebook-sdk = "~3.1" aerich = "~0.6" python-telegram-bot = "~13.10"