From a80eca8a010a6ef3c879305c268589e33f0a1a10 Mon Sep 17 00:00:00 2001 From: Simone Robutti Date: Sun, 25 Sep 2022 23:39:24 +0200 Subject: [PATCH] added check for sqlite db --- mobilizon_reshare/config/config.py | 2 +- mobilizon_reshare/settings.toml | 3 +- mobilizon_reshare/storage/db.py | 60 ++++++++++----------------- mobilizon_reshare/web/backend/main.py | 19 ++++++++- poetry.lock | 17 +++++++- pyproject.toml | 2 +- tests/web/__init__.py | 0 tests/web/test_web_db.py | 37 +++++++++++++++++ 8 files changed, 96 insertions(+), 44 deletions(-) create mode 100644 tests/web/__init__.py create mode 100644 tests/web/test_web_db.py diff --git a/mobilizon_reshare/config/config.py b/mobilizon_reshare/config/config.py index 301d59b..e2a397f 100644 --- a/mobilizon_reshare/config/config.py +++ b/mobilizon_reshare/config/config.py @@ -20,7 +20,7 @@ base_validators = [ # url of the main Mobilizon instance to download events from Validator("source.mobilizon.url", must_exist=True, is_type_of=str), Validator("source.mobilizon.group", must_exist=True, is_type_of=str), - Validator("db_path", must_exist=True, is_type_of=str), + Validator("db_url", must_exist=True, is_type_of=str), Validator("locale", must_exist=True, is_type_of=str, default="en-us"), ] diff --git a/mobilizon_reshare/settings.toml b/mobilizon_reshare/settings.toml index eeda85f..89f9645 100644 --- a/mobilizon_reshare/settings.toml +++ b/mobilizon_reshare/settings.toml @@ -1,7 +1,6 @@ [default] local_state_dir = "/var/mobilizon_reshare" -db_name = "events.db" -db_path = "@format {this.local_state_dir}/{this.db_name}" +db_url = "sqlite:///var/mobilizon_reshare/events.db" locale= "en-us" [default.source.mobilizon] diff --git a/mobilizon_reshare/storage/db.py b/mobilizon_reshare/storage/db.py index 2996f1d..6415b45 100644 --- a/mobilizon_reshare/storage/db.py +++ b/mobilizon_reshare/storage/db.py @@ -3,9 +3,9 @@ from logging.config import dictConfig from pathlib import Path import pkg_resources +import urllib3.util from aerich import Command from tortoise import Tortoise -from tortoise.contrib.fastapi import register_tortoise from mobilizon_reshare.config.config import get_settings from mobilizon_reshare.config.publishers import publisher_names @@ -14,21 +14,13 @@ from mobilizon_reshare.storage.query.write import update_publishers logger = logging.getLogger(__name__) -def get_db_url(): - """gets db url from settings - - Returns: - str : db url - """ - settings = get_settings() - db_path = Path(settings.db_path) - db_url = f"sqlite:///{db_path}" - return db_url +def get_db_url() -> urllib3.util.Url: + return urllib3.util.parse_url(get_settings().db_url) def get_tortoise_orm(): return { - "connections": {"default": get_db_url()}, + "connections": {"default": get_db_url().url}, "apps": { "models": { "models": [ @@ -50,19 +42,12 @@ TORTOISE_ORM = get_tortoise_orm() class MoReDB: - def __init__(self, path: Path): - self.path = path - # TODO: Check if DB is openable/"queriable" - self.is_init = self.path.exists() and (not self.path.is_dir()) - if not self.is_init: - self.path.parent.mkdir(parents=True, exist_ok=True) - async def _implement_db_changes(self): migration_queries_location = pkg_resources.resource_filename( "mobilizon_reshare", "migrations" ) command = Command( - tortoise_config=TORTOISE_ORM, + tortoise_config=get_tortoise_orm(), app="models", location=migration_queries_location, ) @@ -73,23 +58,18 @@ class MoReDB: async def setup(self): await self._implement_db_changes() - await Tortoise.init(config=TORTOISE_ORM,) - if not self.is_init: - await Tortoise.generate_schemas() - self.is_init = True - logger.info(f"Successfully initialized database at {self.path}") - + await Tortoise.init(config=get_tortoise_orm(),) + await Tortoise.generate_schemas() await update_publishers(publisher_names) - @staticmethod - def register_app(app): - orm_data = get_tortoise_orm() - register_tortoise( - app, - db_url=orm_data["connections"]["default"], - modules=orm_data["apps"], - generate_schemas=True, - ) + +class MoReSQLiteDB(MoReDB): + def __init__(self): + self.path = Path(get_db_url().path) + # TODO: Check if DB is openable/"queriable" + self.is_init = self.path.exists() and (not self.path.is_dir()) + if not self.is_init: + self.path.parent.mkdir(parents=True, exist_ok=True) async def tear_down(): @@ -97,8 +77,14 @@ async def tear_down(): async def init(): + # init logging settings = get_settings() dictConfig(settings["logging"]) - db_path = Path(settings.db_path) - db = MoReDB(db_path) + + # init storage + url = get_db_url() + if url.scheme == "sqlite": + db = MoReSQLiteDB() + else: + db = MoReDB() await db.setup() diff --git a/mobilizon_reshare/web/backend/main.py b/mobilizon_reshare/web/backend/main.py index 1e358c2..f61dc89 100644 --- a/mobilizon_reshare/web/backend/main.py +++ b/mobilizon_reshare/web/backend/main.py @@ -1,13 +1,27 @@ +import logging + from fastapi import FastAPI from tortoise.contrib.pydantic import pydantic_model_creator from mobilizon_reshare.models.event import Event -from mobilizon_reshare.storage.db import init +from mobilizon_reshare.storage.db import init as init_db, get_db_url app = FastAPI() event_pydantic = pydantic_model_creator(Event) +logger = logging.getLogger(__name__) + + +def check_database(): + url = get_db_url() + if url.scheme == "sqlite": + logger.warning( + "Database is SQLite. This might create issues when running the web application. Please use a " + "PostgreSQL or MariaDB backend." + ) + + def register_endpoints(app): @app.get("/events", status_code=200) async def get_event(): @@ -17,6 +31,7 @@ def register_endpoints(app): @app.on_event("startup") async def init_app(): - await init() + check_database() + await init_db() register_endpoints(app) return app diff --git a/poetry.lock b/poetry.lock index 91b2788..43f29e6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,6 +72,19 @@ python-versions = ">=3.6" [package.dependencies] python-dateutil = ">=2.7.0" +[[package]] +name = "asyncpg" +version = "0.26.0" +description = "An asyncio PostgreSQL driver" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.extras] +dev = ["Cython (>=0.29.24,<0.30.0)", "pytest (>=6.0)", "Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)"] +test = ["pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"] + [[package]] name = "asynctest" version = "0.13.0" @@ -839,6 +852,7 @@ python-versions = ">=3.7,<4.0" [package.dependencies] aiosqlite = ">=0.16.0,<0.18.0" +asyncpg = {version = "*", optional = true, markers = "extra == \"asyncpg\""} iso8601 = ">=1.0.2,<2.0.0" pypika-tortoise = ">=0.1.6,<0.2.0" pytz = "*" @@ -928,7 +942,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "426776ae7c2dc58809693c231cb1f0bb110cbef889131c4718b5a6275e4dfaa6" +content-hash = "199b3be9e7cf5adb6ddff51200181f03c9386bcd92dc09bf2c899bb6baa090ce" [metadata.files] aerich = [] @@ -946,6 +960,7 @@ arrow = [ {file = "arrow-1.1.1-py3-none-any.whl", hash = "sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510"}, {file = "arrow-1.1.1.tar.gz", hash = "sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a"}, ] +asyncpg = [] asynctest = [ {file = "asynctest-0.13.0-py3-none-any.whl", hash = "sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676"}, {file = "asynctest-0.13.0.tar.gz", hash = "sha256:c27862842d15d83e6a34eb0b2866c323880eb3a75e4485b079ea11748fd77fac"}, diff --git a/pyproject.toml b/pyproject.toml index e74e595..f8803aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ license = "Coopyleft" [tool.poetry.dependencies] python = "^3.9" dynaconf = "~3.1" -tortoise-orm = "~0.19" +tortoise-orm = {extras = ["asyncpg"], version = "^0.19.2"} aiosqlite = "~0.17" Jinja2 = "~3.1" requests = "~2.27" diff --git a/tests/web/__init__.py b/tests/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/web/test_web_db.py b/tests/web/test_web_db.py new file mode 100644 index 0000000..6c0e3cb --- /dev/null +++ b/tests/web/test_web_db.py @@ -0,0 +1,37 @@ +import logging + +import pytest +import urllib3.util + +from mobilizon_reshare.web.backend import main +from mobilizon_reshare.web.backend.main import check_database, init_app + + +def test_check_database_sqlite(caplog): + with caplog.at_level(logging.WARNING): + check_database() + assert caplog.messages == [ + "Database is SQLite. This might create issues when running the web application. " + "Please use a PostgreSQL or MariaDB backend." + ] + + +@pytest.mark.asyncio +async def test_check_database_cli(caplog): + with caplog.at_level(logging.WARNING): + await init_app() + assert caplog.messages == [ + "Database is SQLite. This might create issues when running the web application. " + "Please use a PostgreSQL or MariaDB backend." + ] + + +@pytest.mark.asyncio +async def test_check_database_postgres(caplog, monkeypatch): + def get_url(): + return urllib3.util.parse_url("postgres://someone@something.it") + + monkeypatch.setattr(main, "get_db_url", get_url) + with caplog.at_level(logging.WARNING): + check_database() + assert caplog.messages == []