From de9c7afcf7a587d57418e9c1d702a60ca26b4c65 Mon Sep 17 00:00:00 2001 From: Gabriele De Rosa Date: Wed, 6 Jan 2021 17:57:35 +0100 Subject: [PATCH] Init --- .dockerignore | 158 +++++++++++++++++++++ .env.sample | 4 + .gitignore | 158 +++++++++++++++++++++ Dockerfile | 19 +++ bot.py | 346 +++++++++++++++++++++++++++++++++++++++++++++ db/.env.sample | 3 + db/web/.env.sample | 8 ++ docker-compose.yml | 49 +++++++ requirements.txt | 10 ++ template.html | 77 ++++++++++ 10 files changed, 832 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.sample create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 bot.py create mode 100644 db/.env.sample create mode 100644 db/web/.env.sample create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 template.html diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2944ecf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,158 @@ +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +### vscode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### DATA ### + +# Config +.env + +# Database +db + +# Tmp data +out \ No newline at end of file diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..4feb68d --- /dev/null +++ b/.env.sample @@ -0,0 +1,4 @@ +TELEGRAM_BOT_USERNAME = CHANGE_ME +TELEGRAM_BOT_TOKEN = CHANGE_ME +MONGODB_USER = root +MONGODB_PASS = CHANGE_ME diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..257057f --- /dev/null +++ b/.gitignore @@ -0,0 +1,158 @@ +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +### vscode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### DATA ### + +# Configs +.env + +# Database +db/data + +# Tmp data +out \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..29d2198 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3 + +# Set working space +WORKDIR /usr/src/app + +# Install requirements +RUN ln -snf /usr/share/zoneinfo/Europe/Rome /etc/localtime && \ + apt-get update && apt-get install -y wkhtmltopdf +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Copy app +COPY . . + +# Create folder +RUN mkdir out + +# Start the bot +CMD python -u bot.py \ No newline at end of file diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..40f80f6 --- /dev/null +++ b/bot.py @@ -0,0 +1,346 @@ +import os +import logging +import imgkit +import secrets +import schedule +import time +from dotenv import load_dotenv +from telegram.ext import Updater +from telegram.ext import CommandHandler +from pymongo import MongoClient + +import io +from datetime import timedelta as td + +import matplotlib.dates as mdates +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import requests + +import warnings + + +# Constants +DATA_URL = "https://raw.githubusercontent.com/italia/covid19-opendata-vaccini/master/dati/somministrazioni-vaccini" \ + "-summary-latest.csv" +ITALIAN_POPULATION = 60_360_000 +HIT = ITALIAN_POPULATION / 100 * 80 # We need 80% of population vaccined for herd immunity + +# Exclude warnings +warnings.filterwarnings("ignore") + +# Init environment variables +load_dotenv() + +# Init logging +logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) + +# Get telegram token +telegram_bot_token = os.environ.get('TELEGRAM_BOT_TOKEN') +# Check telegram token +if telegram_bot_token is None: + print('No telegram token.') + exit() +# Init telegram bot +updater = Updater(token=telegram_bot_token, use_context=True) + +# Get mongodb auth +mongodb_user = os.environ.get('MONGODB_USER') +mongodb_pass = os.environ.get('MONGODB_PASS') +# Check mongodb auth +if mongodb_user is None or mongodb_pass is None: + print('No mongodb auth.') + exit() +mongodb_uri = 'mongodb://' + mongodb_user + ':' + mongodb_pass + '@db:27017', +# Init mongodb database +client = MongoClient(mongodb_uri) + +# Get bot database +db = client['bot'] + + + + +# Function to get data +def download(): + + r = requests.get(DATA_URL) + df = pd.read_csv( + io.StringIO(r.text), + index_col="data_somministrazione", + ) + df.index = pd.to_datetime( + df.index, + format="%Y-%m-%d", + ) + df = df.loc[df["area"] == "ITA"] + df["totale"] = pd.to_numeric(df["totale"]) + # df = df[:-1] # Ignore the last day because it's often incomplete + + lastWeekData = df.loc[df.index > df.index[-1] - td(days=7)] + vaccinesPerDayAverage = sum(lastWeekData["totale"]) / 7 + remainingDays = HIT / vaccinesPerDayAverage + hitDate = df.index[-1] + td(days=remainingDays) + + # Generate plot + plt.ylabel("Vaccini al giorno") + plt.xlabel("Ultima settimana") + plt.grid(True) + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + plt.gca().xaxis.set_major_locator(mdates.AutoDateLocator()) + plt.gcf().autofmt_xdate() + plt.bar(lastWeekData.index, height=lastWeekData["totale"]) + # Trendline + z = np.polyfit(range(0, 7), lastWeekData["totale"], 2) + p = np.poly1d(z) + plt.plot(lastWeekData.index, p(range(0, 7)), "r--") + # Secret 4 filenames + sf = secrets.token_hex(16) + # Generate plot filename + plot_filename = 'plot_' + sf + '.png' + # Create plot image/png + plt.savefig('out/' + plot_filename, dpi=300, bbox_inches='tight') + # Generate tmp webpage/html filename + webpage_filename = 'tmp_' + sf + '.html' + # Generate template + with open('template.html', 'r+') as f: + with open('out/' + webpage_filename, 'w+') as wf: + for line in f.read().splitlines(): + if "" in line: + line = f"{sum(df['totale'])}" + elif "" in line: + line = f"{int(vaccinesPerDayAverage*7)}" + elif "" in line: + line = f"{int(vaccinesPerDayAverage)}" + elif "" in line: + line = f"{hitDate.strftime('%d/%m/%Y')}" + elif "" in line: + line = f"{hitDate.strftime('%H:%M:%S')}" + elif "" in line: + line = f"{int(remainingDays)}" + elif "plot.png" in line: + line = line.replace('plot.png', plot_filename) + wf.write("\n" + line) + # Generate plot filename + results_filename = 'results_' + sf + '.png' + # Create results image/png + imgkit.from_file('out/' + webpage_filename, 'out/' + results_filename) + + # Return out data + return { + 'plot': 'out/' + plot_filename, + 'results': 'out/' + results_filename, + 'webpage': 'out/' + webpage_filename + } + + +# Help command +def help(update, context): + + # Help msg + help_msg = "_Lista dei comandi_ \n\n" + help_msg += "/start - Avvia il bot \n" + help_msg += "/get - Ricevi i dati aggiornati \n" + help_msg += "/news `ON`/`OFF` - Attiva o Disattiva le notifiche giornaliere \n" + help_msg += "/help - Messaggio di aiuto \n" + help_msg += "/info - Informazioni su questo bot \n" + help_msg += "/stop - Disattiva il bot \n" + + # Send welcome message + context.bot.send_message(chat_id=update.effective_chat.id, text=help_msg, parse_mode='Markdown') + + + + +# Info command +def info(update, context): + + # Info msg + info_msg = "_Informazioni utili sul bot_\n\n" + info_msg += "Il bot è stato sviluppato da @derogab e il codice sorgente è pubblicamente disponibile su Github. \n\n" + info_msg += "I dati mostrati sono scaricati dagli [Open Data ufficiali](https://github.com/italia/covid19-opendata-vaccini) sui vaccini in Italia. \n\n" + info_msg += "I grafici sono automaticamente generati mediante il codice della [repository pubblica](https://github.com/MarcoBuster/quanto-manca) di @MarcoBuster. \n\n" + + # Send welcome message + context.bot.send_message(chat_id=update.effective_chat.id, text=info_msg, parse_mode='Markdown', disable_web_page_preview=True) + + +# Start command +def start(update, context): + global db + + # Insert user to db + myquery = { + "_id": update.effective_chat.id + } + newvalues = { + "$set": { + "_id": update.effective_chat.id, + "username": update.effective_chat.username, + "first_name": update.effective_chat.first_name, + "last_name": update.effective_chat.last_name, + "active": True, + "news": True + } + } + db.users.update_one(myquery, newvalues, upsert=True) + + # welcome msg + welcome_msg = "Benvenuto su 🇮🇹 *ITA vs. COVID* 🦠 !\n\n" + welcome_msg += "Il bot che ti aggiorna sulla battaglia contro il Covid in Italia." + + # Send welcome message + context.bot.send_message(chat_id=update.effective_chat.id, text=welcome_msg, parse_mode='Markdown') + # Send help message + help(update, context) + # Send help message + info(update, context) + + +# Stop command +def stop(update, context): + global db + + # Set active = false on db + myquery = { + "_id": update.effective_chat.id + } + newvalues = { + "$set": { + "active": False + } + } + db.users.update_one(myquery, newvalues, upsert=True) + + # stop msg + stop_msg = "Il bot è stato spento.\n\n" + stop_msg += "Quando vorrai riavviarlo ti basterà cliccare su /start" + + # Send welcome message + context.bot.send_message(chat_id=update.effective_chat.id, text=stop_msg, parse_mode='Markdown') + +# News command +def news(update, context): + global db + + msg = update.message.text + news = msg.replace('/news', '').strip().lower() + + if news == "on": + to_set = True + set_msg = "Le notifiche giornaliere sono state correttamente abiitate.\n\n" + set_msg += "Riceverai i dati sul progresso della battaglia contro il Covid in Italia ogni giorno alle 10:00." + + elif news == "off": + to_set = False + set_msg = "Le notifiche giornaliere sono state correttamente disabilitate." + + else: + # Invalid param msg + invalid_param_msg = "Il parametro inserito è errato. \n\n" + invalid_param_msg += "/news ON - Per attivare le notifiche giornaliere\n" + invalid_param_msg += "/news OFF - Per disattivare le notifiche giornaliere\n\n" + # Send message + context.bot.send_message(chat_id=update.effective_chat.id, text=invalid_param_msg, parse_mode='Markdown') + # Exit + return + + # Set news on db + myquery = { + "_id": update.effective_chat.id + } + newvalues = { + "$set": { + "news": to_set + } + } + db.users.update_one(myquery, newvalues, upsert=True) + + # Send welcome message + context.bot.send_message(chat_id=update.effective_chat.id, text=set_msg, parse_mode='Markdown') + +# Get data +def get(update, context): + + # Download data + data = download() + + # Send photo + context.bot.send_photo(chat_id=update.effective_chat.id, photo=open(data['results'], 'rb'), caption="") + + # Remove tmp files + os.remove(data['plot']) + os.remove(data['webpage']) + os.remove(data['results']) + + + +# Cron job +def job(): + global db + + # Job running... + print('Job running...') + + # Download data + data = download() + + # Get active user with daily news + users = db.users.find({ + "active": True, + "news": True + }) + # Send news to all active user + for user in users: + + # Send photo + updater.bot.send_photo(chat_id=user['_id'], photo=open(data['results'], 'rb'), caption="") + + + # Remove tmp files + os.remove(data['plot']) + os.remove(data['webpage']) + os.remove(data['results']) + + + + + +# Telegram dispatcher +dispatcher = updater.dispatcher + +# Add /help command +start_handler = CommandHandler('help', help) +dispatcher.add_handler(start_handler) +# Add /info command +start_handler = CommandHandler('info', info) +dispatcher.add_handler(start_handler) +# Add /start command +start_handler = CommandHandler('start', start) +dispatcher.add_handler(start_handler) +# Add /stop command +start_handler = CommandHandler('stop', stop) +dispatcher.add_handler(start_handler) +# Add /get command +start_handler = CommandHandler('get', get) +dispatcher.add_handler(start_handler) +# Add /news command +start_handler = CommandHandler('news', news) +dispatcher.add_handler(start_handler) + +# Setup cron +schedule.every().day.at("10:30").do(job) + +# Start bot +updater.start_polling() + +# Run schedule and check bot +while True: + + if not updater.running: + # Bot is down. + exit() # and auto restart + + schedule.run_pending() + time.sleep(1) diff --git a/db/.env.sample b/db/.env.sample new file mode 100644 index 0000000..a0ee30f --- /dev/null +++ b/db/.env.sample @@ -0,0 +1,3 @@ +MONGO_INITDB_ROOT_USERNAME = root +MONGO_INITDB_ROOT_PASSWORD = CHANGE_ME +MONGO_INITDB_DATABASE = bot diff --git a/db/web/.env.sample b/db/web/.env.sample new file mode 100644 index 0000000..5f112b2 --- /dev/null +++ b/db/web/.env.sample @@ -0,0 +1,8 @@ +ME_CONFIG_MONGODB_SERVER = db +ME_CONFIG_MONGODB_PORT = 27017 +ME_CONFIG_MONGODB_ENABLE_ADMIN = true +ME_CONFIG_MONGODB_AUTH_DATABASE = admin +ME_CONFIG_MONGODB_ADMINUSERNAME = root +ME_CONFIG_MONGODB_ADMINPASSWORD = CHANGE_ME +ME_CONFIG_BASICAUTH_USERNAME = CHANGE_ME +ME_CONFIG_BASICAUTH_PASSWORD = CHANGE_ME diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..09acf9e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +version: "3.5" + +networks: + itavscovidbot.network: + name: itavscovidbot.network + + +services: + + db: + image: mongo + container_name: itavscovidbot.db + restart: unless-stopped + env_file: + - ./db/.env + volumes: + - ./db/data:/data/db + networks: + - itavscovidbot.network + + db.web: + depends_on: + - db + image: mongo-express + container_name: itavscovidbot.db.web + restart: unless-stopped + links: + - db + env_file: + - ./db/web/.env + ports: + - 8081:8081 + networks: + - itavscovidbot.network + + app: + depends_on: + - db + - db.web + build: + context: . + container_name: itavscovidbot.app + restart: unless-stopped + links: + - db + env_file: + - ./.env + networks: + - itavscovidbot.network diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7de795a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +python-telegram-bot +python-dotenv +pandas +matplotlib +requests +numpy +imgkit +pdfkit +pymongo +schedule diff --git a/template.html b/template.html new file mode 100644 index 0000000..9a458d4 --- /dev/null +++ b/template.html @@ -0,0 +1,77 @@ + + + + + Quanto manca? + + + + + + +
+

Quando torneremo alla normalità?

+

+ La domanda non ha una risposta precisa. + Non è ancora chiaro + quale è la percentuale di popolazione che deve essere vaccinata per avere la cosidetta + immunità di gregge o di massa dal virus SARS-Cov-2. + Realisticamente, questo numero si aggira intorno all'80% della popolazione + ovvero circa 48 milioni di persone. +

+

Analisi dei dati della campagna vaccinale

+

+ La campagna vaccinale in Italia è appena iniziata e chiaramente non è ancora a regime, per via di vari fattori + politici e non che questo sito non ha intenzione di trattare o dibattere.
+

+
+ In Italia ci sono
+ + + persone vaccinate.
+ Ne abbiamo vaccinate
+ + + nell'ultima settimana,
+ con un ritmo di
+ + + + + vaccini al giorno.
+ Continuando di questo passo, raggiungeremo l'immunità di gregge il
+ + + + + +
+ ovvero fra
+ + + giorni. + +
+ Grafico vaccini ultima settimana +

+ I dati non comprendono quelli del giorno attuale perché solitamente incompleti. +

+
+ + \ No newline at end of file