From 007aec7529297c7bcfa6a35a3346aa82a4b59197 Mon Sep 17 00:00:00 2001 From: codl Date: Tue, 29 Aug 2017 14:46:32 +0200 Subject: [PATCH] flakes8 --- app.py | 19 ++++-- dodo.py | 33 +++++++--- forget.py | 4 +- lib/auth.py | 6 +- lib/brotli.py | 20 ++++-- lib/cachebust.py | 8 ++- lib/interval.py | 29 +-------- lib/mastodon.py | 70 ++++++++++++--------- lib/scales.py | 29 +++++++++ lib/session.py | 6 +- lib/twitter.py | 80 ++++++++++++++++-------- lib/version.py | 2 +- model.py | 128 +++++++++++++++++++++++++++----------- routes.py | 138 ++++++++++++++++++++++++++++------------- tasks.py | 126 ++++++++++++++++++++++++------------- tools/write-version.sh | 2 +- version.py | 2 +- 17 files changed, 472 insertions(+), 230 deletions(-) create mode 100644 lib/scales.py diff --git a/app.py b/app.py index dc36fa0..31f782d 100644 --- a/app.py +++ b/app.py @@ -6,7 +6,6 @@ from flask_migrate import Migrate import version from lib import cachebust from flask_limiter import Limiter -from flask_limiter.util import get_remote_address from lib import get_viewer import os import mimetypes @@ -29,7 +28,7 @@ app.config.update(default_config) app.config.from_pyfile('config.py', True) -metadata = MetaData(naming_convention = { +metadata = MetaData(naming_convention={ "ix": 'ix_%(column_0_label)s', "uq": "uq_%(table_name)s_%(column_0_name)s", "ck": "ck_%(table_name)s_%(constraint_name)s", @@ -55,12 +54,14 @@ if 'SENTRY_DSN' in app.config: url_for = cachebust(app) + @app.context_processor def inject_static(): def static(filename, **kwargs): return url_for('static', filename=filename, **kwargs) return {'st': static} + def rate_limit_key(): viewer = get_viewer() if viewer: @@ -71,16 +72,25 @@ def rate_limit_key(): return address return request.remote_addr + limiter = Limiter(app, key_func=rate_limit_key) + @app.after_request def install_security_headers(resp): - csp = "default-src 'none'; img-src 'self' https:; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'" + csp = ("default-src 'none';" + "img-src 'self' https:;" + "script-src 'self';" + "style-src 'self' 'unsafe-inline';" + "connect-src 'self';" + "frame-ancestors 'none';" + ) if 'CSP_REPORT_URI' in app.config: csp += "; report-uri " + app.config.get('CSP_REPORT_URI') if app.config.get('HTTPS'): - resp.headers.set('strict-transport-security', 'max-age={}'.format(60*60*24*365)) + resp.headers.set('strict-transport-security', + 'max-age={}'.format(60*60*24*365)) csp += "; upgrade-insecure-requests" resp.headers.set('Content-Security-Policy', csp) @@ -91,4 +101,5 @@ def install_security_headers(resp): return resp + mimetypes.add_type('image/webp', '.webp') diff --git a/dodo.py b/dodo.py index 8210bab..916633a 100644 --- a/dodo.py +++ b/dodo.py @@ -1,10 +1,12 @@ from doit import create_after + def reltouch(source_filename, dest_filename): from os import stat, utime stat_res = stat(source_filename) utime(dest_filename, ns=(stat_res.st_atime_ns, stat_res.st_mtime_ns)) + def resize_image(basename, width, format): from PIL import Image with Image.open('assets/{}.png'.format(basename)) as im: @@ -13,23 +15,25 @@ def resize_image(basename, width, format): else: im = im.convert('RGB') height = im.height * width // im.width - new = im.resize((width,height), resample=Image.LANCZOS) + new = im.resize((width, height), resample=Image.LANCZOS) if format == 'jpeg': kwargs = dict( - optimize = True, - progressive = True, - quality = 80, + optimize=True, + progressive=True, + quality=80, ) elif format == 'webp': kwargs = dict( - quality = 79, + quality=79, ) elif format == 'png': kwargs = dict( - optimize = True, + optimize=True, ) new.save('static/{}-{}.{}'.format(basename, width, format), **kwargs) - reltouch('assets/{}.png'.format(basename), 'static/{}-{}.{}'.format(basename, width, format)) + reltouch('assets/{}.png'.format(basename), + 'static/{}-{}.{}'.format(basename, width, format)) + def task_logotype(): """resize and convert logotype""" @@ -45,9 +49,10 @@ def task_logotype(): clean=True, ) + def task_service_icon(): """resize and convert service icons""" - widths = (20,40,80) + widths = (20, 40, 80) formats = ('webp', 'png') for width in widths: for format in formats: @@ -55,11 +60,13 @@ def task_service_icon(): yield dict( name='{}-{}.{}'.format(basename, width, format), actions=[(resize_image, (basename, width, format))], - targets=['static/{}-{}.{}'.format(basename,width,format)], + targets=[ + 'static/{}-{}.{}'.format(basename, width, format)], file_dep=['assets/{}.png'.format(basename)], clean=True, ) + def task_copy(): "copy assets verbatim" @@ -81,6 +88,7 @@ def task_copy(): clean=True, ) + def task_minify_css(): """minify css file with csscompressor""" @@ -99,12 +107,16 @@ def task_minify_css(): clean=True, ) + @create_after('logotype') @create_after('service_icon') @create_after('copy') @create_after('minify_css') def task_compress(): - "make gzip and brotli compressed versions of each static file for the server to lazily serve" + """ + make gzip and brotli compressed versions of each + static file for the server to lazily serve + """ from glob import glob from itertools import chain @@ -146,6 +158,7 @@ def task_compress(): clean=True, ) + if __name__ == '__main__': import doit doit.run(globals()) diff --git a/forget.py b/forget.py index 97d4c56..d5cb5ad 100644 --- a/forget.py +++ b/forget.py @@ -1,2 +1,2 @@ -from app import app -import routes +from app import app # noqa: F401 +import routes # noqa: F401 diff --git a/lib/auth.py b/lib/auth.py index 0ba2327..a45aea1 100644 --- a/lib/auth.py +++ b/lib/auth.py @@ -1,6 +1,7 @@ from flask import g, redirect, jsonify, make_response, abort, request from functools import wraps + def require_auth(fun): @wraps(fun) def wrapper(*args, **kwargs): @@ -9,11 +10,14 @@ def require_auth(fun): return fun(*args, **kwargs) return wrapper + def require_auth_api(fun): @wraps(fun) def wrapper(*args, **kwargs): if not g.viewer: - return make_response((jsonify(status='error', error='not logged in'), 403)) + return make_response(( + jsonify(status='error', error='not logged in'), + 403)) return fun(*args, **kwargs) return wrapper diff --git a/lib/brotli.py b/lib/brotli.py index c6fd97b..5cabf08 100644 --- a/lib/brotli.py +++ b/lib/brotli.py @@ -6,6 +6,7 @@ import redis import os.path import mimetypes + class BrotliCache(object): def __init__(self, redis_kwargs={}, max_wait=0.020, expire=60*60*6): self.redis = redis.StrictRedis(**redis_kwargs) @@ -32,8 +33,13 @@ class BrotliCache(object): response.headers.set('x-brotli-cache', 'MISS') lock_key = 'brotlicache:lock:{}'.format(digest) if self.redis.set(lock_key, 1, nx=True, ex=10): - mode = brotli_.MODE_TEXT if response.content_type.startswith('text/') else brotli_.MODE_GENERIC - t = Thread(target=self.compress, args=(cache_key, lock_key, body, mode)) + mode = ( + brotli_.MODE_TEXT + if response.content_type.startswith('text/') + else brotli_.MODE_GENERIC) + t = Thread( + target=self.compress, + args=(cache_key, lock_key, body, mode)) t.start() if self.max_wait > 0: t.join(self.max_wait) @@ -50,8 +56,10 @@ class BrotliCache(object): return response -def brotli(app, static = True, dynamic = True): + +def brotli(app, static=True, dynamic=True): original_static = app.view_functions['static'] + def static_maybe_gzip_brotli(filename=None): path = os.path.join(app.static_folder, filename) for encoding, extension in (('br', '.br'), ('gzip', '.gz')): @@ -59,10 +67,12 @@ def brotli(app, static = True, dynamic = True): continue encpath = path + extension if os.path.isfile(encpath): - resp = make_response(original_static(filename=filename + extension)) + resp = make_response( + original_static(filename=filename + extension)) resp.headers.set('content-encoding', encoding) resp.headers.set('vary', 'accept-encoding') - mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + mimetype = (mimetypes.guess_type(filename)[0] + or 'application/octet-stream') resp.headers.set('content-type', mimetype) return resp return original_static(filename=filename) diff --git a/lib/cachebust.py b/lib/cachebust.py index f017e3c..aff4262 100644 --- a/lib/cachebust.py +++ b/lib/cachebust.py @@ -1,5 +1,7 @@ from flask import url_for, abort import os + + def cachebust(app): @app.route('/static-cb//') def static_cachebust(timestamp, filename): @@ -12,14 +14,16 @@ def cachebust(app): abort(404) else: resp = app.view_functions['static'](filename=filename) - resp.headers.set('cache-control', 'public, immutable, max-age=%s' % (60*60*24*365,)) + resp.headers.set( + 'cache-control', + 'public, immutable, max-age={}'.format(60*60*24*365)) if 'expires' in resp.headers: resp.headers.remove('expires') return resp @app.context_processor def replace_url_for(): - return dict(url_for = cachebust_url_for) + return dict(url_for=cachebust_url_for) def cachebust_url_for(endpoint, **kwargs): if endpoint == 'static': diff --git a/lib/interval.py b/lib/interval.py index e6a8c32..0c7d148 100644 --- a/lib/interval.py +++ b/lib/interval.py @@ -1,30 +1,6 @@ from datetime import timedelta, datetime -from statistics import mean +from scales import SCALES -SCALES = [ - ('minutes', timedelta(minutes=1)), - ('hours', timedelta(hours=1)), - ('days', timedelta(days=1)), - ('weeks', timedelta(days=7)), - ('months', timedelta(days= - # you, a fool: a month is 30 days - # me, wise: - mean((31, - mean((29 if year % 400 == 0 - or (year % 100 != 0 and year % 4 == 0) - else 28 - for year in range(400))) - ,31,30,31,30,31,31,30,31,30,31)) - )), - ('years', timedelta(days= - # you, a fool: ok. a year is 365.25 days. happy? - # me, wise: absolutely not - mean((366 if year % 400 == 0 - or (year % 100 != 0 and year % 4 == 0) - else 365 - for year in range(400))) - )), -] def decompose_interval(attrname): scales = [scale[1] for scale in SCALES] @@ -69,7 +45,6 @@ def decompose_interval(attrname): raise ValueError("Incorrect time interval", e) setattr(self, attrname, value * getattr(self, scl_name)) - setattr(cls, scl_name, scale) setattr(cls, sig_name, significand) @@ -77,6 +52,7 @@ def decompose_interval(attrname): return decorator + def relative(interval): # special cases if interval > timedelta(seconds=-15) and interval < timedelta(0): @@ -99,5 +75,6 @@ def relative(interval): else: return '{} ago'.format(output) + def relnow(time): return relative(time - datetime.now()) diff --git a/lib/mastodon.py b/lib/mastodon.py index 82f3c7d..fcdd8c6 100644 --- a/lib/mastodon.py +++ b/lib/mastodon.py @@ -1,4 +1,3 @@ -import mastodon from mastodon import Mastodon from mastodon.Mastodon import MastodonAPIError from model import MastodonApp, Account, OAuthToken, Post @@ -7,6 +6,7 @@ from app import db from math import inf import iso8601 + def get_or_create_app(instance_url, callback, website): instance_url = instance_url app = MastodonApp.query.get(instance_url) @@ -18,7 +18,8 @@ def get_or_create_app(instance_url, callback, website): proto = 'http' if not app: - client_id, client_secret = Mastodon.create_app('forget', + client_id, client_secret = Mastodon.create_app( + 'forget', scopes=('read', 'write'), api_base_url='{}://{}'.format(proto, instance_url), redirect_uris=callback, @@ -31,18 +32,22 @@ def get_or_create_app(instance_url, callback, website): app.protocol = proto return app + def anonymous_api(app): - return Mastodon(app.client_id, - client_secret = app.client_secret, + return Mastodon( + app.client_id, + client_secret=app.client_secret, api_base_url='{}://{}'.format(app.protocol, app.instance), ) + def login_url(app, callback): return anonymous_api(app).auth_request_url( redirect_uris=callback, scopes=('read', 'write',) ) + def receive_code(code, app, callback): api = anonymous_api(app) access_token = api.log_in( @@ -54,7 +59,7 @@ def receive_code(code, app, callback): remote_acc = api.account_verify_credentials() acc = account_from_api_object(remote_acc, app.instance) acc = db.session.merge(acc) - token = OAuthToken(token = access_token) + token = OAuthToken(token=access_token) token = db.session.merge(token) token.account = acc @@ -64,12 +69,12 @@ def receive_code(code, app, callback): def get_api_for_acc(account): app = MastodonApp.query.get(account.mastodon_instance) for token in account.tokens: - api = Mastodon(app.client_id, - client_secret = app.client_secret, - api_base_url = '{}://{}'.format(app.protocol, app.instance), - access_token = token.token, - ratelimit_method = 'throw', - #debug_requests = True, + api = Mastodon( + app.client_id, + client_secret=app.client_secret, + api_base_url='{}://{}'.format(app.protocol, app.instance), + access_token=token.token, + ratelimit_method='throw', ) # api.verify_credentials() @@ -91,15 +96,18 @@ def fetch_acc(acc, cursor=None): print('no access, aborting') return None - newacc = account_from_api_object(api.account_verify_credentials(), acc.mastodon_instance) + newacc = account_from_api_object( + api.account_verify_credentials(), acc.mastodon_instance) acc = db.session.merge(newacc) - kwargs = dict(limit = 40) + kwargs = dict(limit=40) if cursor: kwargs.update(cursor) if 'max_id' not in kwargs: - most_recent_post = Post.query.with_parent(acc).order_by(db.desc(Post.created_at)).first() + most_recent_post = ( + Post.query.with_parent(acc) + .order_by(db.desc(Post.created_at)).first()) if most_recent_post: kwargs['since_id'] = most_recent_post.mastodon_id @@ -120,27 +128,31 @@ def fetch_acc(acc, cursor=None): return kwargs + def post_from_api_object(obj, instance): return Post( - mastodon_instance = instance, - mastodon_id = obj['id'], - favourite = obj['favourited'], - has_media = 'media_attachments' in obj and bool(obj['media_attachments']), - created_at = iso8601.parse_date(obj['created_at']), - author_id = account_from_api_object(obj['account'], instance).id, - direct = obj['visibility'] == 'direct', + mastodon_instance=instance, + mastodon_id=obj['id'], + favourite=obj['favourited'], + has_media=('media_attachments' in obj + and bool(obj['media_attachments'])), + created_at=iso8601.parse_date(obj['created_at']), + author_id=account_from_api_object(obj['account'], instance).id, + direct=obj['visibility'] == 'direct', ) + def account_from_api_object(obj, instance): return Account( - mastodon_instance = instance, - mastodon_id = obj['id'], - screen_name = obj['username'], - display_name = obj['display_name'], - avatar_url = obj['avatar'], - reported_post_count = obj['statuses_count'], + mastodon_instance=instance, + mastodon_id=obj['id'], + screen_name=obj['username'], + display_name=obj['display_name'], + avatar_url=obj['avatar'], + reported_post_count=obj['statuses_count'], ) + def refresh_posts(posts): acc = posts[0].author api = get_api_for_acc(acc) @@ -151,7 +163,8 @@ def refresh_posts(posts): for post in posts: try: status = api.status(post.mastodon_id) - new_post = db.session.merge(post_from_api_object(status, post.mastodon_instance)) + new_post = db.session.merge( + post_from_api_object(status, post.mastodon_instance)) new_posts.append(new_post) except MastodonAPIError as e: if str(e) == 'Endpoint not found.': @@ -161,6 +174,7 @@ def refresh_posts(posts): return new_posts + def delete(post): api = get_api_for_acc(post.author) api.status_delete(post.mastodon_id) diff --git a/lib/scales.py b/lib/scales.py new file mode 100644 index 0000000..b8b0fed --- /dev/null +++ b/lib/scales.py @@ -0,0 +1,29 @@ +# flake8: noqa +from datetime import timedelta +from statistics import mean + +SCALES = [ + ('minutes', timedelta(minutes=1)), + ('hours', timedelta(hours=1)), + ('days', timedelta(days=1)), + ('weeks', timedelta(days=7)), + ('months', timedelta(days= + # you, a fool: a month is 30 days + # me, wise: + mean((31, + mean((29 if year % 400 == 0 + or (year % 100 != 0 and year % 4 == 0) + else 28 + for year in range(400))) + ,31,30,31,30,31,31,30,31,30,31)) + )), + ('years', timedelta(days= + # you, a fool: ok. a year is 365.25 days. happy? + # me, wise: absolutely not + mean((366 if year % 400 == 0 + or (year % 100 != 0 and year % 4 == 0) + else 365 + for year in range(400))) + )), +] + diff --git a/lib/session.py b/lib/session.py index 05bc89f..3d6f86a 100644 --- a/lib/session.py +++ b/lib/session.py @@ -1,17 +1,21 @@ from flask import request + def set_session_cookie(session, response, secure=True): - response.set_cookie('forget_sid', session.id, + response.set_cookie( + 'forget_sid', session.id, max_age=60*60*48, httponly=True, secure=secure) + def get_viewer_session(): from model import Session sid = request.cookies.get('forget_sid', None) if sid: return Session.query.get(sid) + def get_viewer(): session = get_viewer_session() if session: diff --git a/lib/twitter.py b/lib/twitter.py index 1c58650..aae95f1 100644 --- a/lib/twitter.py +++ b/lib/twitter.py @@ -8,6 +8,7 @@ import locale from zipfile import ZipFile from io import BytesIO + def get_login_url(callback='oob', consumer_key=None, consumer_secret=None): twitter = Twitter( auth=OAuth('', '', consumer_key, consumer_secret), @@ -16,33 +17,42 @@ def get_login_url(callback='oob', consumer_key=None, consumer_secret=None): oauth_token = resp['oauth_token'] oauth_token_secret = resp['oauth_token_secret'] - token = OAuthToken(token = oauth_token, token_secret = oauth_token_secret) + token = OAuthToken(token=oauth_token, token_secret=oauth_token_secret) db.session.merge(token) db.session.commit() - return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,) + return ( + "https://api.twitter.com/oauth/authenticate?oauth_token=%s" + % (oauth_token,)) + def account_from_api_user_object(obj): return Account( - twitter_id = obj['id_str'], - display_name = obj['name'], - screen_name = obj['screen_name'], - avatar_url = obj['profile_image_url_https'], - reported_post_count = obj['statuses_count']) + twitter_id=obj['id_str'], + display_name=obj['name'], + screen_name=obj['screen_name'], + avatar_url=obj['profile_image_url_https'], + reported_post_count=obj['statuses_count']) -def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None): + +def receive_verifier(oauth_token, oauth_verifier, + consumer_key=None, consumer_secret=None): temp_token = OAuthToken.query.get(oauth_token) if not temp_token: raise Exception("OAuth token has expired") twitter = Twitter( - auth=OAuth(temp_token.token, temp_token.token_secret, consumer_key, consumer_secret), + auth=OAuth(temp_token.token, temp_token.token_secret, + consumer_key, consumer_secret), format='', api_version=None) - resp = url_decode(twitter.oauth.access_token(oauth_verifier = oauth_verifier)) + resp = url_decode( + twitter.oauth.access_token(oauth_verifier=oauth_verifier)) db.session.delete(temp_token) - new_token = OAuthToken(token = resp['oauth_token'], token_secret = resp['oauth_token_secret']) + new_token = OAuthToken(token=resp['oauth_token'], + token_secret=resp['oauth_token_secret']) new_token = db.session.merge(new_token) new_twitter = Twitter( - auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret)) + auth=OAuth(new_token.token, new_token.token_secret, + consumer_key, consumer_secret)) remote_acct = new_twitter.account.verify_credentials() acct = account_from_api_user_object(remote_acct) acct = db.session.merge(acct) @@ -52,15 +62,17 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se return new_token -def get_twitter_for_acc(account): +def get_twitter_for_acc(account): consumer_key = app.config['TWITTER_CONSUMER_KEY'] consumer_secret = app.config['TWITTER_CONSUMER_SECRET'] - tokens = OAuthToken.query.with_parent(account).order_by(db.desc(OAuthToken.created_at)).all() + tokens = (OAuthToken.query.with_parent(account) + .order_by(db.desc(OAuthToken.created_at)).all()) for token in tokens: t = Twitter( - auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret)) + auth=OAuth(token.token, token.token_secret, + consumer_key, consumer_secret)) try: t.account.verify_credentials() return t @@ -79,24 +91,30 @@ def get_twitter_for_acc(account): account.force_log_out() return None + locale.setlocale(locale.LC_TIME, 'C') + def post_from_api_tweet_object(tweet, post=None): if not post: post = Post() post.twitter_id = tweet['id_str'] try: - post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y') + post.created_at = datetime.strptime( + tweet['created_at'], '%a %b %d %H:%M:%S %z %Y') except ValueError: - post.created_at = datetime.strptime(tweet['created_at'], '%Y-%m-%d %H:%M:%S %z') - #whyyy + post.created_at = datetime.strptime( + tweet['created_at'], '%Y-%m-%d %H:%M:%S %z') + # whyyy post.author_id = 'twitter:{}'.format(tweet['user']['id_str']) if 'favorited' in tweet: post.favourite = tweet['favorited'] if 'entities' in tweet: - post.has_media = bool('media' in tweet['entities'] and tweet['entities']['media']) + post.has_media = bool( + 'media' in tweet['entities'] and tweet['entities']['media']) return post + def fetch_acc(account, cursor): t = get_twitter_for_acc(account) if not t: @@ -106,12 +124,19 @@ def fetch_acc(account, cursor): user = t.account.verify_credentials() db.session.merge(account_from_api_user_object(user)) - kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' } + kwargs = { + 'user_id': account.twitter_id, + 'count': 200, + 'trim_user': True, + 'tweet_mode': 'extended', + } if cursor: kwargs.update(cursor) if 'max_id' not in kwargs: - most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first() + most_recent_post = ( + Post.query.order_by(db.desc(Post.created_at)) + .filter(Post.author_id == account.id).first()) if most_recent_post: kwargs['since_id'] = most_recent_post.twitter_id @@ -142,11 +167,14 @@ def refresh_posts(posts): t = get_twitter_for_acc(posts[0].author) if not t: raise Exception('shit idk. twitter says no') - tweets = t.statuses.lookup(_id=",".join((post.twitter_id for post in posts)), - trim_user = True, tweet_mode = 'extended') + tweets = t.statuses.lookup( + _id=",".join((post.twitter_id for post in posts)), + trim_user=True, tweet_mode='extended') refreshed_posts = list() for post in posts: - tweet = next((tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), None) + tweet = next( + (tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), + None) if not tweet: db.session.delete(post) else: @@ -166,7 +194,9 @@ def chunk_twitter_archive(archive_id): ta = TwitterArchive.query.get(archive_id) with ZipFile(BytesIO(ta.body), 'r') as zipfile: - files = [filename for filename in zipfile.namelist() if filename.startswith('data/js/tweets/') and filename.endswith('.js')] + files = [filename for filename in zipfile.namelist() + if filename.startswith('data/js/tweets/') + and filename.endswith('.js')] files.sort() diff --git a/lib/version.py b/lib/version.py index 56ee70b..ecaca50 100644 --- a/lib/version.py +++ b/lib/version.py @@ -3,9 +3,9 @@ import re version_re = re.compile('(?P.+)-(?P[0-9]+)-g(?P[0-9a-f]+)') + def url_for_version(ver): match = version_re.match(ver) if not match: return app.config['REPO_URL'] return app.config['COMMIT_URL'].format(**match.groupdict()) - diff --git a/model.py b/model.py index 93a06b1..e9a0905 100644 --- a/model.py +++ b/model.py @@ -4,12 +4,16 @@ from app import db import secrets from lib import decompose_interval + class TimestampMixin(object): - created_at = db.Column(db.DateTime, server_default=db.func.now(), nullable=False) - updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now(), nullable=False) + created_at = db.Column(db.DateTime, server_default=db.func.now(), + nullable=False) + updated_at = db.Column(db.DateTime, server_default=db.func.now(), + onupdate=db.func.now(), nullable=False) def touch(self): - self.updated_at=db.func.now() + self.updated_at = db.func.now() + class RemoteIDMixin(object): @property @@ -23,7 +27,9 @@ class RemoteIDMixin(object): if not self.id: return None if self.service != "twitter": - raise Exception("tried to get twitter id for a {} {}".format(self.service, type(self))) + raise Exception( + "tried to get twitter id for a {} {}" + .format(self.service, type(self))) return self.id.split(":")[1] @twitter_id.setter @@ -35,7 +41,9 @@ class RemoteIDMixin(object): if not self.id: return None if self.service != "mastodon": - raise Exception("tried to get mastodon instance for a {} {}".format(self.service, type(self))) + raise Exception( + "tried to get mastodon instance for a {} {}" + .format(self.service, type(self))) return self.id.split(":", 1)[1].split('@')[1] @mastodon_instance.setter @@ -47,7 +55,9 @@ class RemoteIDMixin(object): if not self.id: return None if self.service != "mastodon": - raise Exception("tried to get mastodon id for a {} {}".format(self.service, type(self))) + raise Exception( + "tried to get mastodon id for a {} {}" + .format(self.service, type(self))) return self.id.split(":", 1)[1].split('@')[0] @mastodon_id.setter @@ -61,13 +71,20 @@ class Account(TimestampMixin, RemoteIDMixin): __tablename__ = 'accounts' id = db.Column(db.String, primary_key=True) - policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False) - policy_keep_latest = db.Column(db.Integer, server_default='100', nullable=False) - policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False) - policy_keep_media = db.Column(db.Boolean, server_default='FALSE', nullable=False) - policy_delete_every = db.Column(db.Interval, server_default='30 minutes', nullable=False) - policy_keep_younger = db.Column(db.Interval, server_default='365 days', nullable=False) - policy_keep_direct = db.Column(db.Boolean, server_default='TRUE', nullable=False) + policy_enabled = db.Column(db.Boolean, server_default='FALSE', + nullable=False) + policy_keep_latest = db.Column(db.Integer, server_default='100', + nullable=False) + policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', + nullable=False) + policy_keep_media = db.Column(db.Boolean, server_default='FALSE', + nullable=False) + policy_delete_every = db.Column(db.Interval, server_default='30 minutes', + nullable=False) + policy_keep_younger = db.Column(db.Interval, server_default='365 days', + nullable=False) + policy_keep_direct = db.Column(db.Boolean, server_default='TRUE', + nullable=False) display_name = db.Column(db.String) screen_name = db.Column(db.String) @@ -96,7 +113,8 @@ class Account(TimestampMixin, RemoteIDMixin): def validate_intervals(self, key, value): if not (value == timedelta(0) or value >= timedelta(minutes=1)): value = timedelta(minutes=1) - if key == 'policy_delete_every' and datetime.now() + value < self.next_delete: + if key == 'policy_delete_every' and \ + datetime.now() + value < self.next_delete: # make sure that next delete is not in the far future self.next_delete = datetime.now() + value return value @@ -107,7 +125,6 @@ class Account(TimestampMixin, RemoteIDMixin): return 0 return value - # backref: tokens # backref: twitter_archives # backref: posts @@ -121,20 +138,24 @@ class Account(TimestampMixin, RemoteIDMixin): def estimate_eligible_for_delete(self): """ - this is an estimation because we do not know if favourite status has changed since last time a post was refreshed - and it is unfeasible to refresh every single post every time we need to know how many posts are eligible to delete + this is an estimation because we do not know if favourite status has + changed since last time a post was refreshed and it is unfeasible to + refresh every single post every time we need to know how many posts are + eligible to delete """ - latest_n_posts = Post.query.with_parent(self).order_by(db.desc(Post.created_at)).limit(self.policy_keep_latest) - query = Post.query.with_parent(self).\ - filter(Post.created_at <= db.func.now() - self.policy_keep_younger).\ - except_(latest_n_posts) + latest_n_posts = (Post.query.with_parent(self) + .order_by(db.desc(Post.created_at)) + .limit(self.policy_keep_latest)) + query = (Post.query.with_parent(self) + .filter(Post.created_at <= + db.func.now() - self.policy_keep_younger) + .except_(latest_n_posts)) if(self.policy_keep_favourites): - query = query.filter_by(favourite = False) + query = query.filter_by(favourite=False) if(self.policy_keep_media): - query = query.filter_by(has_media = False) + query = query.filter_by(has_media=False) return query.count() - def force_log_out(self): Session.query.with_parent(self).delete() db.session.commit() @@ -150,22 +171,36 @@ class OAuthToken(db.Model, TimestampMixin): token = db.Column(db.String, primary_key=True) token_secret = db.Column(db.String, nullable=True) - account_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=True, index=True) - account = db.relationship(Account, backref=db.backref('tokens', order_by=lambda: db.desc(OAuthToken.created_at))) + account_id = db.Column(db.String, + db.ForeignKey('accounts.id', ondelete='CASCADE', + onupdate='CASCADE'), + nullable=True, index=True) + account = db.relationship( + Account, + backref=db.backref('tokens', + order_by=lambda: db.desc(OAuthToken.created_at)) + ) - # note: account_id is nullable here because we don't know what account a token is for - # until we call /account/verify_credentials with it + # note: account_id is nullable here because we don't know what account a + # token is for until we call /account/verify_credentials with it class Session(db.Model, TimestampMixin): __tablename__ = 'sessions' - id = db.Column(db.String, primary_key=True, default=lambda: secrets.token_urlsafe()) + id = db.Column(db.String, primary_key=True, + default=lambda: secrets.token_urlsafe()) - account_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False, index=True) + account_id = db.Column( + db.String, + db.ForeignKey('accounts.id', + ondelete='CASCADE', onupdate='CASCADE'), + nullable=False, index=True) account = db.relationship(Account, lazy='joined', backref='sessions') - csrf_token = db.Column(db.String, default=lambda: secrets.token_urlsafe(), nullable=False) + csrf_token = db.Column(db.String, + default=lambda: secrets.token_urlsafe(), + nullable=False) class Post(db.Model, TimestampMixin, RemoteIDMixin): @@ -173,9 +208,15 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin): id = db.Column(db.String, primary_key=True) - author_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False) - author = db.relationship(Account, - backref=db.backref('posts', order_by=lambda: db.desc(Post.created_at))) + author_id = db.Column( + db.String, + db.ForeignKey('accounts.id', + ondelete='CASCADE', onupdate='CASCADE'), + nullable=False) + author = db.relationship( + Account, + backref=db.backref('posts', + order_by=lambda: db.desc(Post.created_at))) favourite = db.Column(db.Boolean, server_default='FALSE', nullable=False) has_media = db.Column(db.Boolean, server_default='FALSE', nullable=False) @@ -184,17 +225,27 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin): def __repr__(self): return ''.format(self.id, self.author_id) + db.Index('ix_posts_author_id_created_at', Post.author_id, Post.created_at) + class TwitterArchive(db.Model, TimestampMixin): __tablename__ = 'twitter_archives' id = db.Column(db.Integer, primary_key=True) - account_id = db.Column(db.String, db.ForeignKey('accounts.id', onupdate='CASCADE', ondelete='CASCADE'), nullable=False) - account = db.relationship(Account, backref=db.backref('twitter_archives', order_by=lambda: db.desc(TwitterArchive.id))) + account_id = db.Column( + db.String, + db.ForeignKey('accounts.id', + onupdate='CASCADE', ondelete='CASCADE'), + nullable=False) + account = db.relationship( + Account, + backref=db.backref('twitter_archives', + order_by=lambda: db.desc(TwitterArchive.id))) body = db.deferred(db.Column(db.LargeBinary, nullable=False)) chunks = db.Column(db.Integer) - chunks_successful = db.Column(db.Integer, server_default='0', nullable=False) + chunks_successful = db.Column(db.Integer, + server_default='0', nullable=False) chunks_failed = db.Column(db.Integer, server_default='0', nullable=False) def status(self): @@ -204,8 +255,10 @@ class TwitterArchive(db.Model, TimestampMixin): return 'successful' return 'pending' + ProtoEnum = db.Enum('http', 'https', name='enum_protocol') + class MastodonApp(db.Model, TimestampMixin): __tablename__ = 'mastodon_apps' @@ -214,6 +267,7 @@ class MastodonApp(db.Model, TimestampMixin): client_secret = db.Column(db.String, nullable=False) protocol = db.Column(ProtoEnum, nullable=False) + class MastodonInstance(db.Model): """ this is for the autocomplete in the mastodon login form diff --git a/routes.py b/routes.py index 796addd..913843b 100644 --- a/routes.py +++ b/routes.py @@ -1,4 +1,5 @@ -from flask import render_template, url_for, redirect, request, g, Response, jsonify +from flask import render_template, url_for, redirect, request, g, Response,\ + jsonify from datetime import datetime, timedelta import lib.twitter import lib.mastodon @@ -6,7 +7,7 @@ import lib from lib.auth import require_auth, require_auth_api, csrf from lib import set_session_cookie from lib import get_viewer_session, get_viewer -from model import Account, Session, Post, TwitterArchive, MastodonApp, MastodonInstance +from model import Session, TwitterArchive, MastodonApp, MastodonInstance from app import app, db, sentry, limiter import tasks from zipfile import BadZipFile @@ -15,6 +16,7 @@ from urllib.error import URLError import version import lib.version + @app.before_request def load_viewer(): g.viewer = get_viewer_session() @@ -25,6 +27,7 @@ def load_viewer(): 'service': g.viewer.account.service }) + @app.context_processor def inject_version(): return dict( @@ -32,6 +35,7 @@ def inject_version(): repo_url=lib.version.url_for_version(version.version), ) + @app.context_processor def inject_sentry(): if sentry: @@ -41,6 +45,7 @@ def inject_sentry(): return dict(sentry_dsn=client_dsn) return dict() + @app.after_request def touch_viewer(resp): if 'viewer' in g and g.viewer: @@ -52,29 +57,40 @@ def touch_viewer(resp): lib.brotli.brotli(app) + @app.route('/') def index(): if g.viewer: - return render_template('logged_in.html', scales=lib.interval.SCALES, - tweet_archive_failed = 'tweet_archive_failed' in request.args, - settings_error = 'settings_error' in request.args - ) + return render_template( + 'logged_in.html', + scales=lib.interval.SCALES, + tweet_archive_failed='tweet_archive_failed' in request.args, + settings_error='settings_error' in request.args) else: - instances = MastodonInstance.query.filter(MastodonInstance.popularity > 13).order_by(db.desc(MastodonInstance.popularity), MastodonInstance.instance).limit(5) - return render_template('index.html', - mastodon_instances = instances, - twitter_login_error = 'twitter_login_error' in request.args) + instances = ( + MastodonInstance.query + .filter(MastodonInstance.popularity > 13) + .order_by(db.desc(MastodonInstance.popularity), + MastodonInstance.instance) + .limit(5)) + return render_template( + 'index.html', + mastodon_instances=instances, + twitter_login_error='twitter_login_error' in request.args) + @app.route('/login/twitter') @limiter.limit('3/minute') def twitter_login_step1(): try: return redirect(lib.twitter.get_login_url( - callback = url_for('twitter_login_step2', _external=True), + callback=url_for('twitter_login_step2', _external=True), **app.config.get_namespace("TWITTER_") )) except (TwitterError, URLError): - return redirect(url_for('index', twitter_login_error='', _anchor='log_in')) + return redirect( + url_for('index', twitter_login_error='', _anchor='log_in')) + @app.route('/login/twitter/callback') @limiter.limit('3/minute') @@ -82,9 +98,11 @@ def twitter_login_step2(): try: oauth_token = request.args['oauth_token'] oauth_verifier = request.args['oauth_verifier'] - token = lib.twitter.receive_verifier(oauth_token, oauth_verifier, **app.config.get_namespace("TWITTER_")) + token = lib.twitter.receive_verifier( + oauth_token, oauth_verifier, + **app.config.get_namespace("TWITTER_")) - session = Session(account_id = token.account_id) + session = Session(account_id=token.account_id) db.session.add(session) db.session.commit() @@ -94,17 +112,21 @@ def twitter_login_step2(): set_session_cookie(session, resp, app.config.get('HTTPS')) return resp except (TwitterError, URLError): - return redirect(url_for('index', twitter_login_error='', _anchor='log_in')) + return redirect( + url_for('index', twitter_login_error='', _anchor='log_in')) + class TweetArchiveEmptyException(Exception): pass + @app.route('/upload_tweet_archive', methods=('POST',)) @limiter.limit('10/10 minutes') @require_auth def upload_tweet_archive(): - ta = TwitterArchive(account = g.viewer.account, - body = request.files['file'].read()) + ta = TwitterArchive( + account=g.viewer.account, + body=request.files['file'].read()) db.session.add(ta) db.session.commit() @@ -120,10 +142,12 @@ def upload_tweet_archive(): for filename in files: tasks.import_twitter_archive_month.s(ta.id, filename).apply_async() - return redirect(url_for('index', _anchor='recent_archives')) except (BadZipFile, TweetArchiveEmptyException): - return redirect(url_for('index', tweet_archive_failed='', _anchor='tweet_archive_import')) + return redirect( + url_for('index', tweet_archive_failed='', + _anchor='tweet_archive_import')) + @app.route('/settings', methods=('POST',)) @csrf @@ -138,9 +162,9 @@ def settings(): except ValueError: return 400 - return redirect(url_for('index', settings_saved='')) + @app.route('/disable', methods=('POST',)) @csrf @require_auth @@ -150,24 +174,37 @@ def disable(): return redirect(url_for('index')) + @app.route('/enable', methods=('POST',)) @csrf @require_auth def enable(): - - risky = False - if not 'confirm' in request.form and not g.viewer.account.policy_enabled: + if 'confirm' not in request.form and not g.viewer.account.policy_enabled: if g.viewer.account.policy_delete_every == timedelta(0): approx = g.viewer.account.estimate_eligible_for_delete() - return render_template('warn.html', message=f"""You've set the time between deleting posts to 0. Every post that matches your expiration rules will be deleted within minutes. - { ("That's about " + str(approx) + " posts.") if approx > 0 else "" } - Go ahead?""") + return render_template( + 'warn.html', + message=f""" + You've set the time between deleting posts to 0. Every post + that matches your expiration rules will be deleted within + minutes. + { ("That's about " + str(approx) + " posts.") if approx > 0 + else "" } + Go ahead? + """) if g.viewer.account.next_delete < datetime.now() - timedelta(days=365): - return render_template('warn.html', message="""Once you enable Forget, posts that match your expiration rules will be deleted permanently. We can't bring them back. Make sure that you won't miss them.""") - + return render_template( + 'warn.html', + message=""" + Once you enable Forget, posts that match your + expiration rules will be deleted permanently. + We can't bring them back. Make sure that you won't + miss them. + """) if not g.viewer.account.policy_enabled: - g.viewer.account.next_delete = datetime.now() + g.viewer.account.policy_delete_every + g.viewer.account.next_delete = ( + datetime.now() + g.viewer.account.policy_delete_every) g.viewer.account.policy_enabled = True db.session.commit() @@ -184,6 +221,7 @@ def logout(): g.viewer = None return redirect(url_for('index')) + @app.route('/api/settings', methods=('PUT',)) @require_auth_api def api_settings_put(): @@ -197,6 +235,7 @@ def api_settings_put(): db.session.commit() return jsonify(status='success', updated=updated) + @app.route('/api/viewer') @require_auth_api def api_viewer(): @@ -211,6 +250,7 @@ def api_viewer(): service=viewer.service, ) + @app.route('/api/viewer/timers') @require_auth_api def api_viewer_timers(): @@ -224,23 +264,33 @@ def api_viewer_timers(): next_delete_rel=lib.interval.relnow(viewer.next_delete), ) + @app.route('/login/mastodon', methods=('GET', 'POST')) def mastodon_login_step1(instance=None): - instances = MastodonInstance.query.filter(MastodonInstance.popularity > 1).order_by(db.desc(MastodonInstance.popularity), MastodonInstance.instance).limit(30) + instances = ( + MastodonInstance + .query.filter(MastodonInstance.popularity > 1) + .order_by(db.desc(MastodonInstance.popularity), + MastodonInstance.instance) + .limit(30)) - instance_url = request.args.get('instance_url', None) or request.form.get('instance_url', None) + instance_url = (request.args.get('instance_url', None) + or request.form.get('instance_url', None)) if not instance_url: - return render_template('mastodon_login.html', instances=instances, - address_error = request.method == 'POST', - generic_error = 'error' in request.args + return render_template( + 'mastodon_login.html', instances=instances, + address_error=request.method == 'POST', + generic_error='error' in request.args ) instance_url = instance_url.split("@")[-1].lower() - callback = url_for('mastodon_login_step2', instance=instance_url, _external=True) + callback = url_for('mastodon_login_step2', + instance=instance_url, _external=True) - app = lib.mastodon.get_or_create_app(instance_url, + app = lib.mastodon.get_or_create_app( + instance_url, callback, url_for('index', _external=True)) db.session.merge(app) @@ -249,24 +299,26 @@ def mastodon_login_step1(instance=None): return redirect(lib.mastodon.login_url(app, callback)) + @app.route('/login/mastodon/callback/') -def mastodon_login_step2(instance): +def mastodon_login_step2(instance_url): code = request.args.get('code', None) - app = MastodonApp.query.get(instance) + app = MastodonApp.query.get(instance_url) if not code or not app: return redirect('mastodon_login_step1', error=True) - callback = url_for('mastodon_login_step2', instance=instance, _external=True) + callback = url_for('mastodon_login_step2', + instance=instance_url, _external=True) token = lib.mastodon.receive_code(code, app, callback) account = token.account - sess = Session(account = account) + sess = Session(account=account) db.session.add(sess) - i=MastodonInstance(instance=instance) - i=db.session.merge(i) - i.bump() + instance = MastodonInstance(instance=instance_url) + instance = db.session.merge(instance) + instance.bump() db.session.commit() diff --git a/tasks.py b/tasks.py index ca892df..6e8f3d6 100644 --- a/tasks.py +++ b/tasks.py @@ -1,14 +1,14 @@ from celery import Celery, Task - from app import app as flaskapp from app import db -from model import Session, Account, TwitterArchive, Post, OAuthToken, MastodonInstance +from model import Session, Account, TwitterArchive, Post, OAuthToken,\ + MastodonInstance import lib.twitter import lib.mastodon from mastodon.Mastodon import MastodonRatelimitError from twitter import TwitterError from urllib.error import URLError -from datetime import timedelta, datetime +from datetime import timedelta from zipfile import ZipFile from io import BytesIO, TextIOWrapper import json @@ -16,7 +16,9 @@ from kombu import Queue import random import version -app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle') + +app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], + task_serializer='pickle') app.conf.task_queues = ( Queue('default', routing_key='celery'), Queue('high_prio', routing_key='high'), @@ -41,14 +43,20 @@ class DBTask(Task): finally: db.session.close() + app.Task = DBTask + +def noop(*args, **kwargs): + pass + + @app.task(autoretry_for=(TwitterError, URLError, MastodonRatelimitError)) def fetch_acc(id, cursor=None): acc = Account.query.get(id) print(f'fetching {acc}') try: - action = lambda acc, cursor: None + action = noop if(acc.service == 'twitter'): action = lib.twitter.fetch_acc elif(acc.service == 'mastodon'): @@ -61,8 +69,10 @@ def fetch_acc(id, cursor=None): acc.touch_fetch() db.session.commit() + @app.task -def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limit=20): +def queue_fetch_for_most_stale_accounts( + min_staleness=timedelta(minutes=5), limit=20): accs = Account.query\ .join(Account.tokens).group_by(Account)\ .filter(Account.last_fetch < db.func.now() - min_staleness)\ @@ -70,7 +80,6 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi .limit(limit) for acc in accs: fetch_acc.s(acc.id).delay() - #acc.touch_fetch() db.session.commit() @@ -92,8 +101,8 @@ def import_twitter_archive_month(archive_id, month_path): post = lib.twitter.post_from_api_tweet_object(tweet) existing_post = db.session.query(Post).get(post.id) - if post.author_id != ta.account_id \ - or existing_post and existing_post.author_id != ta.account_id: + if post.author_id != ta.account_id or\ + existing_post and existing_post.author_id != ta.account_id: raise Exception("Shenanigans!") post = db.session.merge(post) @@ -111,81 +120,104 @@ def import_twitter_archive_month(archive_id, month_path): @app.task def periodic_cleanup(): # delete sessions after 48 hours - Session.query.filter(Session.updated_at < (db.func.now() - timedelta(hours=48))).\ - delete(synchronize_session=False) + (Session.query + .filter(Session.updated_at < (db.func.now() - timedelta(hours=48))) + .delete(synchronize_session=False)) # delete twitter archives after 3 days - TwitterArchive.query.filter(TwitterArchive.updated_at < (db.func.now() - timedelta(days=3))).\ - delete(synchronize_session=False) + (TwitterArchive.query + .filter(TwitterArchive.updated_at < (db.func.now() - timedelta(days=3))) + .delete(synchronize_session=False)) # delete anonymous oauth tokens after 1 day - OAuthToken.query.filter(OAuthToken.updated_at < (db.func.now() - timedelta(days=1)))\ - .filter(OAuthToken.account_id == None)\ - .delete(synchronize_session=False) + (OAuthToken.query + .filter(OAuthToken.updated_at < (db.func.now() - timedelta(days=1))) + .filter(OAuthToken.account_id == None) # noqa: E711 + .delete(synchronize_session=False)) # disable users with no tokens - unreachable = Account.query.outerjoin(Account.tokens).group_by(Account).having(db.func.count(OAuthToken.token) == 0).filter(Account.policy_enabled == True) + unreachable = ( + Account.query + .outerjoin(Account.tokens) + .group_by(Account).having(db.func.count(OAuthToken.token) == 0) + .filter(Account.policy_enabled == True)) # noqa: E712 for account in unreachable: account.policy_enabled = False # normalise mastodon instance popularity scores - biggest_instance = MastodonInstance.query.order_by(db.desc(MastodonInstance.popularity)).first() + biggest_instance = ( + MastodonInstance.query + .order_by(db.desc(MastodonInstance.popularity)).first()) if biggest_instance.popularity > 40: - MastodonInstance.query.update({MastodonInstance.popularity: MastodonInstance.popularity * 40 / biggest_instance.popularity}) + MastodonInstance.query.update({ + MastodonInstance.popularity: + MastodonInstance.popularity * 40 / biggest_instance.popularity + }) db.session.commit() + @app.task def queue_deletes(): - eligible_accounts = Account.query.filter(Account.policy_enabled == True).\ - filter(Account.next_delete < db.func.now()) + eligible_accounts = ( + Account.query.filter(Account.policy_enabled == True) # noqa: E712 + .filter(Account.next_delete < db.func.now())) for account in eligible_accounts: delete_from_account.s(account.id).apply_async() + @app.task(autoretry_for=(TwitterError, URLError, MastodonRatelimitError)) def delete_from_account(account_id): account = Account.query.get(account_id) - latest_n_posts = Post.query.with_parent(account).order_by(db.desc(Post.created_at)).limit(account.policy_keep_latest) - posts = Post.query.with_parent(account).\ - filter(Post.created_at + account.policy_keep_younger <= db.func.now()).\ - except_(latest_n_posts).\ - order_by(db.func.random()).limit(100).all() + latest_n_posts = (Post.query.with_parent(account) + .order_by(db.desc(Post.created_at)) + .limit(account.policy_keep_latest)) + posts = ( + Post.query.with_parent(account) + .filter( + Post.created_at + account.policy_keep_younger <= db.func.now()) + .except_(latest_n_posts) + .order_by(db.func.random()) + .limit(100).all()) eligible = None - action = lambda post: None + action = noop if account.service == 'twitter': action = lib.twitter.delete posts = refresh_posts(posts) - eligible = list((post for post in posts if - (not account.policy_keep_favourites or not post.favourite) - and (not account.policy_keep_media or not post.has_media) - )) + eligible = list( + (post for post in posts if + (not account.policy_keep_favourites or not post.favourite) + and (not account.policy_keep_media or not post.has_media) + )) elif account.service == 'mastodon': action = lib.mastodon.delete for post in posts: refreshed = refresh_posts((post,)) if refreshed and \ - (not account.policy_keep_favourites or not post.favourite) \ - and (not account.policy_keep_media or not post.has_media)\ - and (not account.policy_keep_direct or not post.direct): + (not account.policy_keep_favourites or not post.favourite) \ + and (not account.policy_keep_media or not post.has_media)\ + and (not account.policy_keep_direct or not post.direct): eligible = refreshed break if eligible: if account.policy_delete_every == timedelta(0) and len(eligible) > 1: - print("deleting all {} eligible posts for {}".format(len(eligible), account)) + print("deleting all {} eligible posts for {}" + .format(len(eligible), account)) for post in eligible: account.touch_delete() action(post) else: - post = random.choice(eligible) # nosec + post = random.choice(eligible) # nosec print("deleting {}".format(post)) account.touch_delete() action(post) db.session.commit() + def refresh_posts(posts): posts = list(posts) if len(posts) == 0: @@ -196,27 +228,36 @@ def refresh_posts(posts): elif posts[0].service == 'mastodon': return lib.mastodon.refresh_posts(posts) -@app.task(autoretry_for=(TwitterError, URLError), throws=(MastodonRatelimitError)) + +@app.task(autoretry_for=(TwitterError, URLError), + throws=(MastodonRatelimitError)) def refresh_account(account_id): account = Account.query.get(account_id) limit = 100 if account.service == 'mastodon': limit = 5 - posts = Post.query.with_parent(account).order_by(db.asc(Post.updated_at)).limit(limit).all() + posts = (Post.query.with_parent(account) + .order_by(db.asc(Post.updated_at)).limit(limit).all()) posts = refresh_posts(posts) account.touch_refresh() db.session.commit() -@app.task(autoretry_for=(TwitterError, URLError), throws=(MastodonRatelimitError)) + +@app.task(autoretry_for=(TwitterError, URLError), + throws=(MastodonRatelimitError)) def refresh_account_with_oldest_post(): - post = Post.query.outerjoin(Post.author).join(Account.tokens).group_by(Post).order_by(db.asc(Post.updated_at)).first() + post = (Post.query.outerjoin(Post.author).join(Account.tokens) + .group_by(Post).order_by(db.asc(Post.updated_at)).first()) refresh_account(post.author_id) -@app.task(autoretry_for=(TwitterError, URLError), throws=(MastodonRatelimitError)) + +@app.task(autoretry_for=(TwitterError, URLError), + throws=(MastodonRatelimitError)) def refresh_account_with_longest_time_since_refresh(): - acc = Account.query.join(Account.tokens).group_by(Account).order_by(db.asc(Account.last_refresh)).first() + acc = (Account.query.join(Account.tokens).group_by(Account) + .order_by(db.asc(Account.last_refresh)).first()) refresh_account(acc.id) @@ -228,4 +269,3 @@ app.add_periodic_task(90, refresh_account_with_longest_time_since_refresh) if __name__ == '__main__': app.worker_main() - diff --git a/tools/write-version.sh b/tools/write-version.sh index 724f454..1deb2fb 100755 --- a/tools/write-version.sh +++ b/tools/write-version.sh @@ -1,5 +1,5 @@ #!/bin/bash cd $(dirname $0)/.. -git describe --tags --long --always | python -c 'from jinja2 import Template; print(Template("version=\"{{input}}\"").render(input=input()))' > version.py +git describe --tags --long --always | python -c 'from jinja2 import Template; print(Template("version = \"{{input}}\"").render(input=input()))' > version.py diff --git a/version.py b/version.py index f3ebdf4..a827e57 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -version='v0.0.8' +version = 'v0.0.8'