Mastodon
/
forget-cancellare-vecchi-toot
mirror of https://github.com/codl/forget
2
0
Fork 0
Code Issues Releases Wiki Activity

flakes8

This commit is contained in:
codl 2017-08-29 14:46:32 +02:00
parent 2c4d6b9f63
commit 007aec7529
No known key found for this signature in database
GPG Key ID: 6CD7C8891ED1233A
17 changed files with 472 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app.py
View File

@ -6,7 +6,6 @@ from flask_migrate import Migrate
import version
from lib import cachebust
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
from lib import get_viewer
import os
import mimetypes
@ -29,7 +28,7 @@ app.config.update(default_config)
app.config.from_pyfile('config.py', True)
metadata = MetaData(naming_convention = {
metadata = MetaData(naming_convention={
"ix": 'ix_%(column_0_label)s',
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
@ -55,12 +54,14 @@ if 'SENTRY_DSN' in app.config:
url_for = cachebust(app)
@app.context_processor
def inject_static():
def static(filename, **kwargs):
return url_for('static', filename=filename, **kwargs)
return {'st': static}
def rate_limit_key():
viewer = get_viewer()
if viewer:
@ -71,16 +72,25 @@ def rate_limit_key():
return address
return request.remote_addr
limiter = Limiter(app, key_func=rate_limit_key)
@app.after_request
def install_security_headers(resp):
csp = "default-src 'none'; img-src 'self' https:; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'"
csp = ("default-src 'none';"
"img-src 'self' https:;"
"script-src 'self';"
"style-src 'self' 'unsafe-inline';"
"connect-src 'self';"
"frame-ancestors 'none';"
)
if 'CSP_REPORT_URI' in app.config:
csp += "; report-uri " + app.config.get('CSP_REPORT_URI')
if app.config.get('HTTPS'):
resp.headers.set('strict-transport-security', 'max-age={}'.format(60*60*24*365))
resp.headers.set('strict-transport-security',
'max-age={}'.format(60*60*24*365))
csp += "; upgrade-insecure-requests"
resp.headers.set('Content-Security-Policy', csp)
@ -91,4 +101,5 @@ def install_security_headers(resp):
return resp
mimetypes.add_type('image/webp', '.webp')

33
dodo.py
View File

@ -1,10 +1,12 @@
from doit import create_after
def reltouch(source_filename, dest_filename):
from os import stat, utime
stat_res = stat(source_filename)
utime(dest_filename, ns=(stat_res.st_atime_ns, stat_res.st_mtime_ns))
def resize_image(basename, width, format):
from PIL import Image
with Image.open('assets/{}.png'.format(basename)) as im:
@ -13,23 +15,25 @@ def resize_image(basename, width, format):
else:
im = im.convert('RGB')
height = im.height * width // im.width
new = im.resize((width,height), resample=Image.LANCZOS)
new = im.resize((width, height), resample=Image.LANCZOS)
if format == 'jpeg':
kwargs = dict(
optimize = True,
progressive = True,
quality = 80,
optimize=True,
progressive=True,
quality=80,
)
elif format == 'webp':
kwargs = dict(
quality = 79,
quality=79,
)
elif format == 'png':
kwargs = dict(
optimize = True,
optimize=True,
)
new.save('static/{}-{}.{}'.format(basename, width, format), **kwargs)
reltouch('assets/{}.png'.format(basename), 'static/{}-{}.{}'.format(basename, width, format))
reltouch('assets/{}.png'.format(basename),
'static/{}-{}.{}'.format(basename, width, format))
def task_logotype():
"""resize and convert logotype"""
@ -45,9 +49,10 @@ def task_logotype():
clean=True,
)
def task_service_icon():
"""resize and convert service icons"""
widths = (20,40,80)
widths = (20, 40, 80)
formats = ('webp', 'png')
for width in widths:
for format in formats:
@ -55,11 +60,13 @@ def task_service_icon():
yield dict(
name='{}-{}.{}'.format(basename, width, format),
actions=[(resize_image, (basename, width, format))],
targets=['static/{}-{}.{}'.format(basename,width,format)],
targets=[
'static/{}-{}.{}'.format(basename, width, format)],
file_dep=['assets/{}.png'.format(basename)],
clean=True,
)
def task_copy():
"copy assets verbatim"
@ -81,6 +88,7 @@ def task_copy():
clean=True,
)
def task_minify_css():
"""minify css file with csscompressor"""
@ -99,12 +107,16 @@ def task_minify_css():
clean=True,
)
@create_after('logotype')
@create_after('service_icon')
@create_after('copy')
@create_after('minify_css')
def task_compress():
"make gzip and brotli compressed versions of each static file for the server to lazily serve"
"""
make gzip and brotli compressed versions of each
static file for the server to lazily serve
"""
from glob import glob
from itertools import chain
@ -146,6 +158,7 @@ def task_compress():
clean=True,
)
if __name__ == '__main__':
import doit
doit.run(globals())

4
forget.py
View File

@ -1,2 +1,2 @@
from app import app
import routes
from app import app # noqa: F401
import routes # noqa: F401

6
lib/auth.py
View File

@ -1,6 +1,7 @@
from flask import g, redirect, jsonify, make_response, abort, request
from functools import wraps
def require_auth(fun):
@wraps(fun)
def wrapper(*args, **kwargs):
@ -9,11 +10,14 @@ def require_auth(fun):
return fun(*args, **kwargs)
return wrapper
def require_auth_api(fun):
@wraps(fun)
def wrapper(*args, **kwargs):
if not g.viewer:
return make_response((jsonify(status='error', error='not logged in'), 403))
return make_response((
jsonify(status='error', error='not logged in'),
403))
return fun(*args, **kwargs)
return wrapper

20
lib/brotli.py
View File

@ -6,6 +6,7 @@ import redis
import os.path
import mimetypes
class BrotliCache(object):
def __init__(self, redis_kwargs={}, max_wait=0.020, expire=60*60*6):
self.redis = redis.StrictRedis(**redis_kwargs)
@ -32,8 +33,13 @@ class BrotliCache(object):
response.headers.set('x-brotli-cache', 'MISS')
lock_key = 'brotlicache:lock:{}'.format(digest)
if self.redis.set(lock_key, 1, nx=True, ex=10):
mode = brotli_.MODE_TEXT if response.content_type.startswith('text/') else brotli_.MODE_GENERIC
t = Thread(target=self.compress, args=(cache_key, lock_key, body, mode))
mode = (
brotli_.MODE_TEXT
if response.content_type.startswith('text/')
else brotli_.MODE_GENERIC)
t = Thread(
target=self.compress,
args=(cache_key, lock_key, body, mode))
t.start()
if self.max_wait > 0:
t.join(self.max_wait)
@ -50,8 +56,10 @@ class BrotliCache(object):
return response
def brotli(app, static = True, dynamic = True):
def brotli(app, static=True, dynamic=True):
original_static = app.view_functions['static']
def static_maybe_gzip_brotli(filename=None):
path = os.path.join(app.static_folder, filename)
for encoding, extension in (('br', '.br'), ('gzip', '.gz')):
@ -59,10 +67,12 @@ def brotli(app, static = True, dynamic = True):
continue
encpath = path + extension
if os.path.isfile(encpath):
resp = make_response(original_static(filename=filename + extension))
resp = make_response(
original_static(filename=filename + extension))
resp.headers.set('content-encoding', encoding)
resp.headers.set('vary', 'accept-encoding')
mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
mimetype = (mimetypes.guess_type(filename)[0]
or 'application/octet-stream')
resp.headers.set('content-type', mimetype)
return resp
return original_static(filename=filename)

8
lib/cachebust.py
View File

@ -1,5 +1,7 @@
from flask import url_for, abort
import os
def cachebust(app):
@app.route('/static-cb/<int:timestamp>/<path:filename>')
def static_cachebust(timestamp, filename):
@ -12,14 +14,16 @@ def cachebust(app):
abort(404)
else:
resp = app.view_functions['static'](filename=filename)
resp.headers.set('cache-control', 'public, immutable, max-age=%s' % (60*60*24*365,))
resp.headers.set(
'cache-control',
'public, immutable, max-age={}'.format(60*60*24*365))
if 'expires' in resp.headers:
resp.headers.remove('expires')
return resp
@app.context_processor
def replace_url_for():
return dict(url_for = cachebust_url_for)
return dict(url_for=cachebust_url_for)
def cachebust_url_for(endpoint, **kwargs):
if endpoint == 'static':

29
lib/interval.py
View File

@ -1,30 +1,6 @@
from datetime import timedelta, datetime
from statistics import mean
from scales import SCALES
SCALES = [
('minutes', timedelta(minutes=1)),
('hours', timedelta(hours=1)),
('days', timedelta(days=1)),
('weeks', timedelta(days=7)),
('months', timedelta(days=
# you, a fool: a month is 30 days
# me, wise:
mean((31,
mean((29 if year % 400 == 0
or (year % 100 != 0 and year % 4 == 0)
else 28
for year in range(400)))
,31,30,31,30,31,31,30,31,30,31))
)),
('years', timedelta(days=
# you, a fool: ok. a year is 365.25 days. happy?
# me, wise: absolutely not
mean((366 if year % 400 == 0
or (year % 100 != 0 and year % 4 == 0)
else 365
for year in range(400)))
)),
]
def decompose_interval(attrname):
scales = [scale[1] for scale in SCALES]
@ -69,7 +45,6 @@ def decompose_interval(attrname):
raise ValueError("Incorrect time interval", e)
setattr(self, attrname, value * getattr(self, scl_name))
setattr(cls, scl_name, scale)
setattr(cls, sig_name, significand)
@ -77,6 +52,7 @@ def decompose_interval(attrname):
return decorator
def relative(interval):
# special cases
if interval > timedelta(seconds=-15) and interval < timedelta(0):
@ -99,5 +75,6 @@ def relative(interval):
else:
return '{} ago'.format(output)
def relnow(time):
return relative(time - datetime.now())

70
lib/mastodon.py
View File

@ -1,4 +1,3 @@
import mastodon
from mastodon import Mastodon
from mastodon.Mastodon import MastodonAPIError
from model import MastodonApp, Account, OAuthToken, Post
@ -7,6 +6,7 @@ from app import db
from math import inf
import iso8601
def get_or_create_app(instance_url, callback, website):
instance_url = instance_url
app = MastodonApp.query.get(instance_url)
@ -18,7 +18,8 @@ def get_or_create_app(instance_url, callback, website):
proto = 'http'
if not app:
client_id, client_secret = Mastodon.create_app('forget',
client_id, client_secret = Mastodon.create_app(
'forget',
scopes=('read', 'write'),
api_base_url='{}://{}'.format(proto, instance_url),
redirect_uris=callback,
@ -31,18 +32,22 @@ def get_or_create_app(instance_url, callback, website):
app.protocol = proto
return app
def anonymous_api(app):
return Mastodon(app.client_id,
client_secret = app.client_secret,
return Mastodon(
app.client_id,
client_secret=app.client_secret,
api_base_url='{}://{}'.format(app.protocol, app.instance),
)
def login_url(app, callback):
return anonymous_api(app).auth_request_url(
redirect_uris=callback,
scopes=('read', 'write',)
)
def receive_code(code, app, callback):
api = anonymous_api(app)
access_token = api.log_in(
@ -54,7 +59,7 @@ def receive_code(code, app, callback):
remote_acc = api.account_verify_credentials()
acc = account_from_api_object(remote_acc, app.instance)
acc = db.session.merge(acc)
token = OAuthToken(token = access_token)
token = OAuthToken(token=access_token)
token = db.session.merge(token)
token.account = acc
@ -64,12 +69,12 @@ def receive_code(code, app, callback):
def get_api_for_acc(account):
app = MastodonApp.query.get(account.mastodon_instance)
for token in account.tokens:
api = Mastodon(app.client_id,
client_secret = app.client_secret,
api_base_url = '{}://{}'.format(app.protocol, app.instance),
access_token = token.token,
ratelimit_method = 'throw',
#debug_requests = True,
api = Mastodon(
app.client_id,
client_secret=app.client_secret,
api_base_url='{}://{}'.format(app.protocol, app.instance),
access_token=token.token,
ratelimit_method='throw',
)
# api.verify_credentials()
@ -91,15 +96,18 @@ def fetch_acc(acc, cursor=None):
print('no access, aborting')
return None
newacc = account_from_api_object(api.account_verify_credentials(), acc.mastodon_instance)
newacc = account_from_api_object(
api.account_verify_credentials(), acc.mastodon_instance)
acc = db.session.merge(newacc)
kwargs = dict(limit = 40)
kwargs = dict(limit=40)
if cursor:
kwargs.update(cursor)
if 'max_id' not in kwargs:
most_recent_post = Post.query.with_parent(acc).order_by(db.desc(Post.created_at)).first()
most_recent_post = (
Post.query.with_parent(acc)
.order_by(db.desc(Post.created_at)).first())
if most_recent_post:
kwargs['since_id'] = most_recent_post.mastodon_id
@ -120,27 +128,31 @@ def fetch_acc(acc, cursor=None):
return kwargs
def post_from_api_object(obj, instance):
return Post(
mastodon_instance = instance,
mastodon_id = obj['id'],
favourite = obj['favourited'],
has_media = 'media_attachments' in obj and bool(obj['media_attachments']),
created_at = iso8601.parse_date(obj['created_at']),
author_id = account_from_api_object(obj['account'], instance).id,
direct = obj['visibility'] == 'direct',
mastodon_instance=instance,
mastodon_id=obj['id'],
favourite=obj['favourited'],
has_media=('media_attachments' in obj
and bool(obj['media_attachments'])),
created_at=iso8601.parse_date(obj['created_at']),
author_id=account_from_api_object(obj['account'], instance).id,
direct=obj['visibility'] == 'direct',
)
def account_from_api_object(obj, instance):
return Account(
mastodon_instance = instance,
mastodon_id = obj['id'],
screen_name = obj['username'],
display_name = obj['display_name'],
avatar_url = obj['avatar'],
reported_post_count = obj['statuses_count'],
mastodon_instance=instance,
mastodon_id=obj['id'],
screen_name=obj['username'],
display_name=obj['display_name'],
avatar_url=obj['avatar'],
reported_post_count=obj['statuses_count'],
)
def refresh_posts(posts):
acc = posts[0].author
api = get_api_for_acc(acc)
@ -151,7 +163,8 @@ def refresh_posts(posts):
for post in posts:
try:
status = api.status(post.mastodon_id)
new_post = db.session.merge(post_from_api_object(status, post.mastodon_instance))
new_post = db.session.merge(
post_from_api_object(status, post.mastodon_instance))
new_posts.append(new_post)
except MastodonAPIError as e:
if str(e) == 'Endpoint not found.':
@ -161,6 +174,7 @@ def refresh_posts(posts):
return new_posts
def delete(post):
api = get_api_for_acc(post.author)
api.status_delete(post.mastodon_id)

29
lib/scales.py Normal file
View File

@ -0,0 +1,29 @@
# flake8: noqa
from datetime import timedelta
from statistics import mean
SCALES = [
('minutes', timedelta(minutes=1)),
('hours', timedelta(hours=1)),
('days', timedelta(days=1)),
('weeks', timedelta(days=7)),
('months', timedelta(days=
# you, a fool: a month is 30 days
# me, wise:
mean((31,
mean((29 if year % 400 == 0
or (year % 100 != 0 and year % 4 == 0)
else 28
for year in range(400)))
,31,30,31,30,31,31,30,31,30,31))
)),
('years', timedelta(days=
# you, a fool: ok. a year is 365.25 days. happy?
# me, wise: absolutely not
mean((366 if year % 400 == 0
or (year % 100 != 0 and year % 4 == 0)
else 365
for year in range(400)))
)),
]

6
lib/session.py
View File

@ -1,17 +1,21 @@
from flask import request
def set_session_cookie(session, response, secure=True):
response.set_cookie('forget_sid', session.id,
response.set_cookie(
'forget_sid', session.id,
max_age=60*60*48,
httponly=True,
secure=secure)
def get_viewer_session():
from model import Session
sid = request.cookies.get('forget_sid', None)
if sid:
return Session.query.get(sid)
def get_viewer():
session = get_viewer_session()
if session:

80
lib/twitter.py
View File

@ -8,6 +8,7 @@ import locale
from zipfile import ZipFile
from io import BytesIO
def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
twitter = Twitter(
auth=OAuth('', '', consumer_key, consumer_secret),
@ -16,33 +17,42 @@ def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
oauth_token = resp['oauth_token']
oauth_token_secret = resp['oauth_token_secret']
token = OAuthToken(token = oauth_token, token_secret = oauth_token_secret)
token = OAuthToken(token=oauth_token, token_secret=oauth_token_secret)
db.session.merge(token)
db.session.commit()
return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,)
return (
"https://api.twitter.com/oauth/authenticate?oauth_token=%s"
% (oauth_token,))
def account_from_api_user_object(obj):
return Account(
twitter_id = obj['id_str'],
display_name = obj['name'],
screen_name = obj['screen_name'],
avatar_url = obj['profile_image_url_https'],
reported_post_count = obj['statuses_count'])
twitter_id=obj['id_str'],
display_name=obj['name'],
screen_name=obj['screen_name'],
avatar_url=obj['profile_image_url_https'],
reported_post_count=obj['statuses_count'])
def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None):
def receive_verifier(oauth_token, oauth_verifier,
consumer_key=None, consumer_secret=None):
temp_token = OAuthToken.query.get(oauth_token)
if not temp_token:
raise Exception("OAuth token has expired")
twitter = Twitter(
auth=OAuth(temp_token.token, temp_token.token_secret, consumer_key, consumer_secret),
auth=OAuth(temp_token.token, temp_token.token_secret,
consumer_key, consumer_secret),
format='', api_version=None)
resp = url_decode(twitter.oauth.access_token(oauth_verifier = oauth_verifier))
resp = url_decode(
twitter.oauth.access_token(oauth_verifier=oauth_verifier))
db.session.delete(temp_token)
new_token = OAuthToken(token = resp['oauth_token'], token_secret = resp['oauth_token_secret'])
new_token = OAuthToken(token=resp['oauth_token'],
token_secret=resp['oauth_token_secret'])
new_token = db.session.merge(new_token)
new_twitter = Twitter(
auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret))
auth=OAuth(new_token.token, new_token.token_secret,
consumer_key, consumer_secret))
remote_acct = new_twitter.account.verify_credentials()
acct = account_from_api_user_object(remote_acct)
acct = db.session.merge(acct)
@ -52,15 +62,17 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
return new_token
def get_twitter_for_acc(account):
def get_twitter_for_acc(account):
consumer_key = app.config['TWITTER_CONSUMER_KEY']
consumer_secret = app.config['TWITTER_CONSUMER_SECRET']
tokens = OAuthToken.query.with_parent(account).order_by(db.desc(OAuthToken.created_at)).all()
tokens = (OAuthToken.query.with_parent(account)
.order_by(db.desc(OAuthToken.created_at)).all())
for token in tokens:
t = Twitter(
auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret))
auth=OAuth(token.token, token.token_secret,
consumer_key, consumer_secret))
try:
t.account.verify_credentials()
return t
@ -79,24 +91,30 @@ def get_twitter_for_acc(account):
account.force_log_out()
return None
locale.setlocale(locale.LC_TIME, 'C')
def post_from_api_tweet_object(tweet, post=None):
if not post:
post = Post()
post.twitter_id = tweet['id_str']
try:
post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
post.created_at = datetime.strptime(
tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
except ValueError:
post.created_at = datetime.strptime(tweet['created_at'], '%Y-%m-%d %H:%M:%S %z')
#whyyy
post.created_at = datetime.strptime(
tweet['created_at'], '%Y-%m-%d %H:%M:%S %z')
# whyyy
post.author_id = 'twitter:{}'.format(tweet['user']['id_str'])
if 'favorited' in tweet:
post.favourite = tweet['favorited']
if 'entities' in tweet:
post.has_media = bool('media' in tweet['entities'] and tweet['entities']['media'])
post.has_media = bool(
'media' in tweet['entities'] and tweet['entities']['media'])
return post
def fetch_acc(account, cursor):
t = get_twitter_for_acc(account)
if not t:
@ -106,12 +124,19 @@ def fetch_acc(account, cursor):
user = t.account.verify_credentials()
db.session.merge(account_from_api_user_object(user))
kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' }
kwargs = {
'user_id': account.twitter_id,
'count': 200,
'trim_user': True,
'tweet_mode': 'extended',
}
if cursor:
kwargs.update(cursor)
if 'max_id' not in kwargs:
most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first()
most_recent_post = (
Post.query.order_by(db.desc(Post.created_at))
.filter(Post.author_id == account.id).first())
if most_recent_post:
kwargs['since_id'] = most_recent_post.twitter_id
@ -142,11 +167,14 @@ def refresh_posts(posts):
t = get_twitter_for_acc(posts[0].author)
if not t:
raise Exception('shit idk. twitter says no')
tweets = t.statuses.lookup(_id=",".join((post.twitter_id for post in posts)),
trim_user = True, tweet_mode = 'extended')
tweets = t.statuses.lookup(
_id=",".join((post.twitter_id for post in posts)),
trim_user=True, tweet_mode='extended')
refreshed_posts = list()
for post in posts:
tweet = next((tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), None)
tweet = next(
(tweet for tweet in tweets if tweet['id_str'] == post.twitter_id),
None)
if not tweet:
db.session.delete(post)
else:
@ -166,7 +194,9 @@ def chunk_twitter_archive(archive_id):
ta = TwitterArchive.query.get(archive_id)
with ZipFile(BytesIO(ta.body), 'r') as zipfile:
files = [filename for filename in zipfile.namelist() if filename.startswith('data/js/tweets/') and filename.endswith('.js')]
files = [filename for filename in zipfile.namelist()
if filename.startswith('data/js/tweets/')
and filename.endswith('.js')]
files.sort()

2
lib/version.py
View File

@ -3,9 +3,9 @@ import re
version_re = re.compile('(?P<tag>.+)-(?P<commits>[0-9]+)-g(?P<hash>[0-9a-f]+)')
def url_for_version(ver):
match = version_re.match(ver)
if not match:
return app.config['REPO_URL']
return app.config['COMMIT_URL'].format(**match.groupdict())

128
model.py
View File

@ -4,12 +4,16 @@ from app import db
import secrets
from lib import decompose_interval
class TimestampMixin(object):
created_at = db.Column(db.DateTime, server_default=db.func.now(), nullable=False)
updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now(), nullable=False)
created_at = db.Column(db.DateTime, server_default=db.func.now(),
nullable=False)
updated_at = db.Column(db.DateTime, server_default=db.func.now(),
onupdate=db.func.now(), nullable=False)
def touch(self):
self.updated_at=db.func.now()
self.updated_at = db.func.now()
class RemoteIDMixin(object):
@property
@ -23,7 +27,9 @@ class RemoteIDMixin(object):
if not self.id:
return None
if self.service != "twitter":
raise Exception("tried to get twitter id for a {} {}".format(self.service, type(self)))
raise Exception(
"tried to get twitter id for a {} {}"
.format(self.service, type(self)))
return self.id.split(":")[1]
@twitter_id.setter
@ -35,7 +41,9 @@ class RemoteIDMixin(object):
if not self.id:
return None
if self.service != "mastodon":
raise Exception("tried to get mastodon instance for a {} {}".format(self.service, type(self)))
raise Exception(
"tried to get mastodon instance for a {} {}"
.format(self.service, type(self)))
return self.id.split(":", 1)[1].split('@')[1]
@mastodon_instance.setter
@ -47,7 +55,9 @@ class RemoteIDMixin(object):
if not self.id:
return None
if self.service != "mastodon":
raise Exception("tried to get mastodon id for a {} {}".format(self.service, type(self)))
raise Exception(
"tried to get mastodon id for a {} {}"
.format(self.service, type(self)))
return self.id.split(":", 1)[1].split('@')[0]
@mastodon_id.setter
@ -61,13 +71,20 @@ class Account(TimestampMixin, RemoteIDMixin):
__tablename__ = 'accounts'
id = db.Column(db.String, primary_key=True)
policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
policy_keep_latest = db.Column(db.Integer, server_default='100', nullable=False)
policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False)
policy_keep_media = db.Column(db.Boolean, server_default='FALSE', nullable=False)
policy_delete_every = db.Column(db.Interval, server_default='30 minutes', nullable=False)
policy_keep_younger = db.Column(db.Interval, server_default='365 days', nullable=False)
policy_keep_direct = db.Column(db.Boolean, server_default='TRUE', nullable=False)
policy_enabled = db.Column(db.Boolean, server_default='FALSE',
nullable=False)
policy_keep_latest = db.Column(db.Integer, server_default='100',
nullable=False)
policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE',
nullable=False)
policy_keep_media = db.Column(db.Boolean, server_default='FALSE',
nullable=False)
policy_delete_every = db.Column(db.Interval, server_default='30 minutes',
nullable=False)
policy_keep_younger = db.Column(db.Interval, server_default='365 days',
nullable=False)
policy_keep_direct = db.Column(db.Boolean, server_default='TRUE',
nullable=False)
display_name = db.Column(db.String)
screen_name = db.Column(db.String)
@ -96,7 +113,8 @@ class Account(TimestampMixin, RemoteIDMixin):
def validate_intervals(self, key, value):
if not (value == timedelta(0) or value >= timedelta(minutes=1)):
value = timedelta(minutes=1)
if key == 'policy_delete_every' and datetime.now() + value < self.next_delete:
if key == 'policy_delete_every' and \
datetime.now() + value < self.next_delete:
# make sure that next delete is not in the far future
self.next_delete = datetime.now() + value
return value
@ -107,7 +125,6 @@ class Account(TimestampMixin, RemoteIDMixin):
return 0
return value
# backref: tokens
# backref: twitter_archives
# backref: posts
@ -121,20 +138,24 @@ class Account(TimestampMixin, RemoteIDMixin):
def estimate_eligible_for_delete(self):
"""
this is an estimation because we do not know if favourite status has changed since last time a post was refreshed
and it is unfeasible to refresh every single post every time we need to know how many posts are eligible to delete
this is an estimation because we do not know if favourite status has
changed since last time a post was refreshed and it is unfeasible to
refresh every single post every time we need to know how many posts are
eligible to delete
"""
latest_n_posts = Post.query.with_parent(self).order_by(db.desc(Post.created_at)).limit(self.policy_keep_latest)
query = Post.query.with_parent(self).\
filter(Post.created_at <= db.func.now() - self.policy_keep_younger).\
except_(latest_n_posts)
latest_n_posts = (Post.query.with_parent(self)
.order_by(db.desc(Post.created_at))
.limit(self.policy_keep_latest))
query = (Post.query.with_parent(self)
.filter(Post.created_at <=
db.func.now() - self.policy_keep_younger)
.except_(latest_n_posts))
if(self.policy_keep_favourites):
query = query.filter_by(favourite = False)
query = query.filter_by(favourite=False)
if(self.policy_keep_media):
query = query.filter_by(has_media = False)
query = query.filter_by(has_media=False)
return query.count()
def force_log_out(self):
Session.query.with_parent(self).delete()
db.session.commit()
@ -150,22 +171,36 @@ class OAuthToken(db.Model, TimestampMixin):
token = db.Column(db.String, primary_key=True)
token_secret = db.Column(db.String, nullable=True)
account_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=True, index=True)
account = db.relationship(Account, backref=db.backref('tokens', order_by=lambda: db.desc(OAuthToken.created_at)))
account_id = db.Column(db.String,
db.ForeignKey('accounts.id', ondelete='CASCADE',
onupdate='CASCADE'),
nullable=True, index=True)
account = db.relationship(
Account,
backref=db.backref('tokens',
order_by=lambda: db.desc(OAuthToken.created_at))
)
# note: account_id is nullable here because we don't know what account a token is for
# until we call /account/verify_credentials with it
# note: account_id is nullable here because we don't know what account a
# token is for until we call /account/verify_credentials with it
class Session(db.Model, TimestampMixin):
__tablename__ = 'sessions'
id = db.Column(db.String, primary_key=True, default=lambda: secrets.token_urlsafe())
id = db.Column(db.String, primary_key=True,
default=lambda: secrets.token_urlsafe())
account_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False, index=True)
account_id = db.Column(
db.String,
db.ForeignKey('accounts.id',
ondelete='CASCADE', onupdate='CASCADE'),
nullable=False, index=True)
account = db.relationship(Account, lazy='joined', backref='sessions')
csrf_token = db.Column(db.String, default=lambda: secrets.token_urlsafe(), nullable=False)
csrf_token = db.Column(db.String,
default=lambda: secrets.token_urlsafe(),
nullable=False)
class Post(db.Model, TimestampMixin, RemoteIDMixin):
@ -173,9 +208,15 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin):
id = db.Column(db.String, primary_key=True)
author_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
author = db.relationship(Account,
backref=db.backref('posts', order_by=lambda: db.desc(Post.created_at)))
author_id = db.Column(
db.String,
db.ForeignKey('accounts.id',
ondelete='CASCADE', onupdate='CASCADE'),
nullable=False)
author = db.relationship(
Account,
backref=db.backref('posts',
order_by=lambda: db.desc(Post.created_at)))
favourite = db.Column(db.Boolean, server_default='FALSE', nullable=False)
has_media = db.Column(db.Boolean, server_default='FALSE', nullable=False)
@ -184,17 +225,27 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin):
def __repr__(self):
return '<Post ({}, Author: {})>'.format(self.id, self.author_id)
db.Index('ix_posts_author_id_created_at', Post.author_id, Post.created_at)
class TwitterArchive(db.Model, TimestampMixin):
__tablename__ = 'twitter_archives'
id = db.Column(db.Integer, primary_key=True)
account_id = db.Column(db.String, db.ForeignKey('accounts.id', onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
account = db.relationship(Account, backref=db.backref('twitter_archives', order_by=lambda: db.desc(TwitterArchive.id)))
account_id = db.Column(
db.String,
db.ForeignKey('accounts.id',
onupdate='CASCADE', ondelete='CASCADE'),
nullable=False)
account = db.relationship(
Account,
backref=db.backref('twitter_archives',
order_by=lambda: db.desc(TwitterArchive.id)))
body = db.deferred(db.Column(db.LargeBinary, nullable=False))
chunks = db.Column(db.Integer)
chunks_successful = db.Column(db.Integer, server_default='0', nullable=False)
chunks_successful = db.Column(db.Integer,
server_default='0', nullable=False)
chunks_failed = db.Column(db.Integer, server_default='0', nullable=False)
def status(self):
@ -204,8 +255,10 @@ class TwitterArchive(db.Model, TimestampMixin):
return 'successful'
return 'pending'
ProtoEnum = db.Enum('http', 'https', name='enum_protocol')
class MastodonApp(db.Model, TimestampMixin):
__tablename__ = 'mastodon_apps'
@ -214,6 +267,7 @@ class MastodonApp(db.Model, TimestampMixin):
client_secret = db.Column(db.String, nullable=False)
protocol = db.Column(ProtoEnum, nullable=False)
class MastodonInstance(db.Model):
"""
this is for the autocomplete in the mastodon login form

138
routes.py
View File

@ -1,4 +1,5 @@
from flask import render_template, url_for, redirect, request, g, Response, jsonify
from flask import render_template, url_for, redirect, request, g, Response,\
jsonify
from datetime import datetime, timedelta
import lib.twitter
import lib.mastodon
@ -6,7 +7,7 @@ import lib
from lib.auth import require_auth, require_auth_api, csrf
from lib import set_session_cookie
from lib import get_viewer_session, get_viewer
from model import Account, Session, Post, TwitterArchive, MastodonApp, MastodonInstance
from model import Session, TwitterArchive, MastodonApp, MastodonInstance
from app import app, db, sentry, limiter
import tasks
from zipfile import BadZipFile
@ -15,6 +16,7 @@ from urllib.error import URLError
import version
import lib.version
@app.before_request
def load_viewer():
g.viewer = get_viewer_session()
@ -25,6 +27,7 @@ def load_viewer():
'service': g.viewer.account.service
})
@app.context_processor
def inject_version():
return dict(
@ -32,6 +35,7 @@ def inject_version():
repo_url=lib.version.url_for_version(version.version),
)
@app.context_processor
def inject_sentry():
if sentry:
@ -41,6 +45,7 @@ def inject_sentry():
return dict(sentry_dsn=client_dsn)
return dict()
@app.after_request
def touch_viewer(resp):
if 'viewer' in g and g.viewer:
@ -52,29 +57,40 @@ def touch_viewer(resp):
lib.brotli.brotli(app)
@app.route('/')
def index():
if g.viewer:
return render_template('logged_in.html', scales=lib.interval.SCALES,
tweet_archive_failed = 'tweet_archive_failed' in request.args,
settings_error = 'settings_error' in request.args
)
return render_template(
'logged_in.html',
scales=lib.interval.SCALES,
tweet_archive_failed='tweet_archive_failed' in request.args,
settings_error='settings_error' in request.args)
else:
instances = MastodonInstance.query.filter(MastodonInstance.popularity > 13).order_by(db.desc(MastodonInstance.popularity), MastodonInstance.instance).limit(5)
return render_template('index.html',
mastodon_instances = instances,
twitter_login_error = 'twitter_login_error' in request.args)
instances = (
MastodonInstance.query
.filter(MastodonInstance.popularity > 13)
.order_by(db.desc(MastodonInstance.popularity),
MastodonInstance.instance)
.limit(5))
return render_template(
'index.html',
mastodon_instances=instances,
twitter_login_error='twitter_login_error' in request.args)
@app.route('/login/twitter')
@limiter.limit('3/minute')
def twitter_login_step1():
try:
return redirect(lib.twitter.get_login_url(
callback = url_for('twitter_login_step2', _external=True),
callback=url_for('twitter_login_step2', _external=True),
**app.config.get_namespace("TWITTER_")
))
except (TwitterError, URLError):
return redirect(url_for('index', twitter_login_error='', _anchor='log_in'))
return redirect(
url_for('index', twitter_login_error='', _anchor='log_in'))
@app.route('/login/twitter/callback')
@limiter.limit('3/minute')
@ -82,9 +98,11 @@ def twitter_login_step2():
try:
oauth_token = request.args['oauth_token']
oauth_verifier = request.args['oauth_verifier']
token = lib.twitter.receive_verifier(oauth_token, oauth_verifier, **app.config.get_namespace("TWITTER_"))
token = lib.twitter.receive_verifier(
oauth_token, oauth_verifier,
**app.config.get_namespace("TWITTER_"))
session = Session(account_id = token.account_id)
session = Session(account_id=token.account_id)
db.session.add(session)
db.session.commit()
@ -94,17 +112,21 @@ def twitter_login_step2():
set_session_cookie(session, resp, app.config.get('HTTPS'))
return resp
except (TwitterError, URLError):
return redirect(url_for('index', twitter_login_error='', _anchor='log_in'))
return redirect(
url_for('index', twitter_login_error='', _anchor='log_in'))
class TweetArchiveEmptyException(Exception):
pass
@app.route('/upload_tweet_archive', methods=('POST',))
@limiter.limit('10/10 minutes')
@require_auth
def upload_tweet_archive():
ta = TwitterArchive(account = g.viewer.account,
body = request.files['file'].read())
ta = TwitterArchive(
account=g.viewer.account,
body=request.files['file'].read())
db.session.add(ta)
db.session.commit()
@ -120,10 +142,12 @@ def upload_tweet_archive():
for filename in files:
tasks.import_twitter_archive_month.s(ta.id, filename).apply_async()
return redirect(url_for('index', _anchor='recent_archives'))
except (BadZipFile, TweetArchiveEmptyException):
return redirect(url_for('index', tweet_archive_failed='', _anchor='tweet_archive_import'))
return redirect(
url_for('index', tweet_archive_failed='',
_anchor='tweet_archive_import'))
@app.route('/settings', methods=('POST',))
@csrf
@ -138,9 +162,9 @@ def settings():
except ValueError:
return 400
return redirect(url_for('index', settings_saved=''))
@app.route('/disable', methods=('POST',))
@csrf
@require_auth
@ -150,24 +174,37 @@ def disable():
return redirect(url_for('index'))
@app.route('/enable', methods=('POST',))
@csrf
@require_auth
def enable():
risky = False
if not 'confirm' in request.form and not g.viewer.account.policy_enabled:
if 'confirm' not in request.form and not g.viewer.account.policy_enabled:
if g.viewer.account.policy_delete_every == timedelta(0):
approx = g.viewer.account.estimate_eligible_for_delete()
return render_template('warn.html', message=f"""You've set the time between deleting posts to 0. Every post that matches your expiration rules will be deleted within minutes.
{ ("That's about " + str(approx) + " posts.") if approx > 0 else "" }
Go ahead?""")
return render_template(
'warn.html',
message=f"""
You've set the time between deleting posts to 0. Every post
that matches your expiration rules will be deleted within
minutes.
{ ("That's about " + str(approx) + " posts.") if approx > 0
else "" }
Go ahead?
""")
if g.viewer.account.next_delete < datetime.now() - timedelta(days=365):
return render_template('warn.html', message="""Once you enable Forget, posts that match your expiration rules will be deleted <b>permanently</b>. We can't bring them back. Make sure that you won't miss them.""")
return render_template(
'warn.html',
message="""
Once you enable Forget, posts that match your
expiration rules will be deleted <b>permanently</b>.
We can't bring them back. Make sure that you won't
miss them.
""")
if not g.viewer.account.policy_enabled:
g.viewer.account.next_delete = datetime.now() + g.viewer.account.policy_delete_every
g.viewer.account.next_delete = (
datetime.now() + g.viewer.account.policy_delete_every)
g.viewer.account.policy_enabled = True
db.session.commit()
@ -184,6 +221,7 @@ def logout():
g.viewer = None
return redirect(url_for('index'))
@app.route('/api/settings', methods=('PUT',))
@require_auth_api
def api_settings_put():
@ -197,6 +235,7 @@ def api_settings_put():
db.session.commit()
return jsonify(status='success', updated=updated)
@app.route('/api/viewer')
@require_auth_api
def api_viewer():
@ -211,6 +250,7 @@ def api_viewer():
service=viewer.service,
)
@app.route('/api/viewer/timers')
@require_auth_api
def api_viewer_timers():
@ -224,23 +264,33 @@ def api_viewer_timers():
next_delete_rel=lib.interval.relnow(viewer.next_delete),
)
@app.route('/login/mastodon', methods=('GET', 'POST'))
def mastodon_login_step1(instance=None):
instances = MastodonInstance.query.filter(MastodonInstance.popularity > 1).order_by(db.desc(MastodonInstance.popularity), MastodonInstance.instance).limit(30)
instances = (
MastodonInstance
.query.filter(MastodonInstance.popularity > 1)
.order_by(db.desc(MastodonInstance.popularity),
MastodonInstance.instance)
.limit(30))
instance_url = request.args.get('instance_url', None) or request.form.get('instance_url', None)
instance_url = (request.args.get('instance_url', None)
or request.form.get('instance_url', None))
if not instance_url:
return render_template('mastodon_login.html', instances=instances,
address_error = request.method == 'POST',
generic_error = 'error' in request.args
return render_template(
'mastodon_login.html', instances=instances,
address_error=request.method == 'POST',
generic_error='error' in request.args
)
instance_url = instance_url.split("@")[-1].lower()
callback = url_for('mastodon_login_step2', instance=instance_url, _external=True)
callback = url_for('mastodon_login_step2',
instance=instance_url, _external=True)
app = lib.mastodon.get_or_create_app(instance_url,
app = lib.mastodon.get_or_create_app(
instance_url,
callback,
url_for('index', _external=True))
db.session.merge(app)
@ -249,24 +299,26 @@ def mastodon_login_step1(instance=None):
return redirect(lib.mastodon.login_url(app, callback))
@app.route('/login/mastodon/callback/<instance>')
def mastodon_login_step2(instance):
def mastodon_login_step2(instance_url):
code = request.args.get('code', None)
app = MastodonApp.query.get(instance)
app = MastodonApp.query.get(instance_url)
if not code or not app:
return redirect('mastodon_login_step1', error=True)
callback = url_for('mastodon_login_step2', instance=instance, _external=True)
callback = url_for('mastodon_login_step2',
instance=instance_url, _external=True)
token = lib.mastodon.receive_code(code, app, callback)
account = token.account
sess = Session(account = account)
sess = Session(account=account)
db.session.add(sess)
i=MastodonInstance(instance=instance)
i=db.session.merge(i)
i.bump()
instance = MastodonInstance(instance=instance_url)
instance = db.session.merge(instance)
instance.bump()
db.session.commit()

126
tasks.py
View File

@ -1,14 +1,14 @@
from celery import Celery, Task
from app import app as flaskapp
from app import db
from model import Session, Account, TwitterArchive, Post, OAuthToken, MastodonInstance
from model import Session, Account, TwitterArchive, Post, OAuthToken,\
MastodonInstance
import lib.twitter
import lib.mastodon
from mastodon.Mastodon import MastodonRatelimitError
from twitter import TwitterError
from urllib.error import URLError
from datetime import timedelta, datetime
from datetime import timedelta
from zipfile import ZipFile
from io import BytesIO, TextIOWrapper
import json
@ -16,7 +16,9 @@ from kombu import Queue
import random
import version
app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle')
app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'],
task_serializer='pickle')
app.conf.task_queues = (
Queue('default', routing_key='celery'),
Queue('high_prio', routing_key='high'),
@ -41,14 +43,20 @@ class DBTask(Task):
finally:
db.session.close()
app.Task = DBTask
def noop(*args, **kwargs):
pass
@app.task(autoretry_for=(TwitterError, URLError, MastodonRatelimitError))
def fetch_acc(id, cursor=None):
acc = Account.query.get(id)
print(f'fetching {acc}')
try:
action = lambda acc, cursor: None
action = noop
if(acc.service == 'twitter'):
action = lib.twitter.fetch_acc
elif(acc.service == 'mastodon'):
@ -61,8 +69,10 @@ def fetch_acc(id, cursor=None):
acc.touch_fetch()
db.session.commit()
@app.task
def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limit=20):
def queue_fetch_for_most_stale_accounts(
min_staleness=timedelta(minutes=5), limit=20):
accs = Account.query\
.join(Account.tokens).group_by(Account)\
.filter(Account.last_fetch < db.func.now() - min_staleness)\
@ -70,7 +80,6 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi
.limit(limit)
for acc in accs:
fetch_acc.s(acc.id).delay()
#acc.touch_fetch()
db.session.commit()
@ -92,8 +101,8 @@ def import_twitter_archive_month(archive_id, month_path):
post = lib.twitter.post_from_api_tweet_object(tweet)
existing_post = db.session.query(Post).get(post.id)
if post.author_id != ta.account_id \
or existing_post and existing_post.author_id != ta.account_id:
if post.author_id != ta.account_id or\
existing_post and existing_post.author_id != ta.account_id:
raise Exception("Shenanigans!")
post = db.session.merge(post)
@ -111,81 +120,104 @@ def import_twitter_archive_month(archive_id, month_path):
@app.task
def periodic_cleanup():
# delete sessions after 48 hours
Session.query.filter(Session.updated_at < (db.func.now() - timedelta(hours=48))).\
delete(synchronize_session=False)
(Session.query
.filter(Session.updated_at < (db.func.now() - timedelta(hours=48)))
.delete(synchronize_session=False))
# delete twitter archives after 3 days
TwitterArchive.query.filter(TwitterArchive.updated_at < (db.func.now() - timedelta(days=3))).\
delete(synchronize_session=False)
(TwitterArchive.query
.filter(TwitterArchive.updated_at < (db.func.now() - timedelta(days=3)))
.delete(synchronize_session=False))
# delete anonymous oauth tokens after 1 day
OAuthToken.query.filter(OAuthToken.updated_at < (db.func.now() - timedelta(days=1)))\
.filter(OAuthToken.account_id == None)\
.delete(synchronize_session=False)
(OAuthToken.query
.filter(OAuthToken.updated_at < (db.func.now() - timedelta(days=1)))
.filter(OAuthToken.account_id == None) # noqa: E711
.delete(synchronize_session=False))
# disable users with no tokens
unreachable = Account.query.outerjoin(Account.tokens).group_by(Account).having(db.func.count(OAuthToken.token) == 0).filter(Account.policy_enabled == True)
unreachable = (
Account.query
.outerjoin(Account.tokens)
.group_by(Account).having(db.func.count(OAuthToken.token) == 0)
.filter(Account.policy_enabled == True)) # noqa: E712
for account in unreachable:
account.policy_enabled = False
# normalise mastodon instance popularity scores
biggest_instance = MastodonInstance.query.order_by(db.desc(MastodonInstance.popularity)).first()
biggest_instance = (
MastodonInstance.query
.order_by(db.desc(MastodonInstance.popularity)).first())
if biggest_instance.popularity > 40:
MastodonInstance.query.update({MastodonInstance.popularity: MastodonInstance.popularity * 40 / biggest_instance.popularity})
MastodonInstance.query.update({
MastodonInstance.popularity:
MastodonInstance.popularity * 40 / biggest_instance.popularity
})
db.session.commit()
@app.task
def queue_deletes():
eligible_accounts = Account.query.filter(Account.policy_enabled == True).\
filter(Account.next_delete < db.func.now())
eligible_accounts = (
Account.query.filter(Account.policy_enabled == True) # noqa: E712
.filter(Account.next_delete < db.func.now()))
for account in eligible_accounts:
delete_from_account.s(account.id).apply_async()
@app.task(autoretry_for=(TwitterError, URLError, MastodonRatelimitError))
def delete_from_account(account_id):
account = Account.query.get(account_id)
latest_n_posts = Post.query.with_parent(account).order_by(db.desc(Post.created_at)).limit(account.policy_keep_latest)
posts = Post.query.with_parent(account).\
filter(Post.created_at + account.policy_keep_younger <= db.func.now()).\
except_(latest_n_posts).\
order_by(db.func.random()).limit(100).all()
latest_n_posts = (Post.query.with_parent(account)
.order_by(db.desc(Post.created_at))
.limit(account.policy_keep_latest))
posts = (
Post.query.with_parent(account)
.filter(
Post.created_at + account.policy_keep_younger <= db.func.now())
.except_(latest_n_posts)
.order_by(db.func.random())
.limit(100).all())
eligible = None
action = lambda post: None
action = noop
if account.service == 'twitter':
action = lib.twitter.delete
posts = refresh_posts(posts)
eligible = list((post for post in posts if
(not account.policy_keep_favourites or not post.favourite)
and (not account.policy_keep_media or not post.has_media)
))
eligible = list(
(post for post in posts if
(not account.policy_keep_favourites or not post.favourite)
and (not account.policy_keep_media or not post.has_media)
))
elif account.service == 'mastodon':
action = lib.mastodon.delete
for post in posts:
refreshed = refresh_posts((post,))
if refreshed and \
(not account.policy_keep_favourites or not post.favourite) \
and (not account.policy_keep_media or not post.has_media)\
and (not account.policy_keep_direct or not post.direct):
(not account.policy_keep_favourites or not post.favourite) \
and (not account.policy_keep_media or not post.has_media)\
and (not account.policy_keep_direct or not post.direct):
eligible = refreshed
break
if eligible:
if account.policy_delete_every == timedelta(0) and len(eligible) > 1:
print("deleting all {} eligible posts for {}".format(len(eligible), account))
print("deleting all {} eligible posts for {}"
.format(len(eligible), account))
for post in eligible:
account.touch_delete()
action(post)
else:
post = random.choice(eligible) # nosec
post = random.choice(eligible) # nosec
print("deleting {}".format(post))
account.touch_delete()
action(post)
db.session.commit()
def refresh_posts(posts):
posts = list(posts)
if len(posts) == 0:
@ -196,27 +228,36 @@ def refresh_posts(posts):
elif posts[0].service == 'mastodon':
return lib.mastodon.refresh_posts(posts)
@app.task(autoretry_for=(TwitterError, URLError), throws=(MastodonRatelimitError))
@app.task(autoretry_for=(TwitterError, URLError),
throws=(MastodonRatelimitError))
def refresh_account(account_id):
account = Account.query.get(account_id)
limit = 100
if account.service == 'mastodon':
limit = 5
posts = Post.query.with_parent(account).order_by(db.asc(Post.updated_at)).limit(limit).all()
posts = (Post.query.with_parent(account)
.order_by(db.asc(Post.updated_at)).limit(limit).all())
posts = refresh_posts(posts)
account.touch_refresh()
db.session.commit()
@app.task(autoretry_for=(TwitterError, URLError), throws=(MastodonRatelimitError))
@app.task(autoretry_for=(TwitterError, URLError),
throws=(MastodonRatelimitError))
def refresh_account_with_oldest_post():
post = Post.query.outerjoin(Post.author).join(Account.tokens).group_by(Post).order_by(db.asc(Post.updated_at)).first()
post = (Post.query.outerjoin(Post.author).join(Account.tokens)
.group_by(Post).order_by(db.asc(Post.updated_at)).first())
refresh_account(post.author_id)
@app.task(autoretry_for=(TwitterError, URLError), throws=(MastodonRatelimitError))
@app.task(autoretry_for=(TwitterError, URLError),
throws=(MastodonRatelimitError))
def refresh_account_with_longest_time_since_refresh():
acc = Account.query.join(Account.tokens).group_by(Account).order_by(db.asc(Account.last_refresh)).first()
acc = (Account.query.join(Account.tokens).group_by(Account)
.order_by(db.asc(Account.last_refresh)).first())
refresh_account(acc.id)
@ -228,4 +269,3 @@ app.add_periodic_task(90, refresh_account_with_longest_time_since_refresh)
if __name__ == '__main__':
app.worker_main()

2
tools/write-version.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
cd $(dirname $0)/..
git describe --tags --long --always | python -c 'from jinja2 import Template; print(Template("version=\"{{input}}\"").render(input=input()))' > version.py
git describe --tags --long --always | python -c 'from jinja2 import Template; print(Template("version = \"{{input}}\"").render(input=input()))' > version.py

2
version.py
View File

@ -1 +1 @@
version='v0.0.8'
version = 'v0.0.8'