From e08a0eea130c8f2f707177f124154d8a41b14658 Mon Sep 17 00:00:00 2001 From: codl Date: Wed, 2 Aug 2017 01:35:09 +0200 Subject: [PATCH] ahhh im super good at computing --- lib/interval.py | 5 +-- lib/twitter.py | 55 ++++++++++++++++++---------- migrations/versions/7379bfac4379_.py | 28 ++++++++++++++ migrations/versions/995890c26959_.py | 30 +++++++++++++++ model.py | 11 +++++- routes.py | 2 +- tasks.py | 38 ++++++++++++++++++- templates/index.html | 2 +- templates/settings.html | 4 +- 9 files changed, 146 insertions(+), 29 deletions(-) create mode 100644 migrations/versions/7379bfac4379_.py create mode 100644 migrations/versions/995890c26959_.py diff --git a/lib/interval.py b/lib/interval.py index 295426b..6dabc0a 100644 --- a/lib/interval.py +++ b/lib/interval.py @@ -1,7 +1,6 @@ from datetime import timedelta SCALES = [ - ('seconds', timedelta(seconds=1)), ('minutes', timedelta(minutes=1)), ('hours', timedelta(hours=1)), ('days', timedelta(days=1)), @@ -21,13 +20,13 @@ def decompose_interval(attrname): def scale(self): if getattr(self, attrname) == timedelta(0): - return timedelta(seconds=1) + return timedelta(minutes=1) for m in scales: if getattr(self, attrname) % m == timedelta(0): return m - return timedelta(seconds=1) + return timedelta(minutes=1) @scale.setter def scale(self, value): diff --git a/lib/twitter.py b/lib/twitter.py index 36bfe09..4685083 100644 --- a/lib/twitter.py +++ b/lib/twitter.py @@ -1,7 +1,7 @@ from twitter import Twitter, OAuth from werkzeug.urls import url_decode from model import OAuthToken, Account, Post -from app import db +from app import db, app from math import inf from datetime import datetime import locale @@ -45,30 +45,22 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se return new_token -def get_twitter_for_acc(account, consumer_key=None, consumer_secret=None): - token = account.tokens[0] +def get_twitter_for_acc(account): + + consumer_key = app.config['TWITTER_CONSUMER_KEY'] + consumer_secret = app.config['TWITTER_CONSUMER_SECRET'] + + token = OAuthToken.query.with_parent(account).order_by(db.desc(OAuthToken.created_at)).first() t = Twitter( auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret)) return t locale.setlocale(locale.LC_TIME, 'C') -def csv_tweet_to_json_tweet(tweet, account): - tweet.update({ - 'id': int(tweet['tweet_id']), - 'id_str': tweet['tweet_id'], - 'created_at': datetime.strptime(tweet['timestamp'], - '%Y-%m-%d %H:%M:%S %z')\ - .strftime('%a %b %d %H:%M:%S %z %Y'), - 'user': { - 'id': int(account.twitter_id), - 'id_str': account.twitter_id - } - }) - return tweet - -def tweet_to_post(tweet): - post = Post(twitter_id=tweet['id_str']) +def tweet_to_post(tweet, post=None): + if not post: + post = Post() + post.twitter_id = tweet['id_str'] try: post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y') except ValueError: @@ -79,10 +71,12 @@ def tweet_to_post(tweet): else: post.body = tweet['text'] post.author_id = 'twitter:{}'.format(tweet['user']['id_str']) + if 'favorited' in tweet: + post.favourite = tweet['favorited'] return post def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): - t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret) + t = get_twitter_for_acc(account) user = t.account.verify_credentials() @@ -117,3 +111,24 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): return kwargs + +def refresh_posts(posts): + t = get_twitter_for_acc(posts[0].author) + tweets = t.statuses.lookup(_id=",".join((post.twitter_id for post in posts)), + trim_user = True, tweet_mode = 'extended') + refreshed_posts = list() + for post in posts: + tweet = next((tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), None) + if not tweet: + session.delete(post) + else: + post = db.session.merge(tweet_to_post(tweet)) + refreshed_posts.append(post) + + return refreshed_posts + + +def delete(post): + t = get_twitter_for_acc(post.author) + t.statuses.destroy(id=post.twitter_id) + db.session.delete(post) diff --git a/migrations/versions/7379bfac4379_.py b/migrations/versions/7379bfac4379_.py new file mode 100644 index 0000000..e69cae5 --- /dev/null +++ b/migrations/versions/7379bfac4379_.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: 7379bfac4379 +Revises: 995890c26959 +Create Date: 2017-08-01 23:35:17.924428 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7379bfac4379' +down_revision = '995890c26959' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('posts', sa.Column('favourite', sa.Boolean(), server_default='FALSE', nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('posts', 'favourite') + # ### end Alembic commands ### diff --git a/migrations/versions/995890c26959_.py b/migrations/versions/995890c26959_.py new file mode 100644 index 0000000..b9599a4 --- /dev/null +++ b/migrations/versions/995890c26959_.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: 995890c26959 +Revises: e036b007017c +Create Date: 2017-08-01 23:24:22.223674 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '995890c26959' +down_revision = 'e036b007017c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('accounts', sa.Column('policy_keep_favourites', sa.Boolean(), server_default='TRUE', nullable=True)) + op.drop_column('accounts', 'policy_ignore_favourites') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('accounts', sa.Column('policy_ignore_favourites', sa.BOOLEAN(), server_default=sa.text('true'), autoincrement=False, nullable=True)) + op.drop_column('accounts', 'policy_keep_favourites') + # ### end Alembic commands ### diff --git a/model.py b/model.py index 6165e2f..1e15483 100644 --- a/model.py +++ b/model.py @@ -5,6 +5,7 @@ from app import db from twitter import Twitter, OAuth import secrets from lib import decompose_interval +from datetime import timedelta class TimestampMixin(object): created_at = db.Column(db.DateTime, server_default=db.func.now()) @@ -42,7 +43,7 @@ class Account(TimestampMixin, RemoteIDMixin): policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False) policy_keep_latest = db.Column(db.Integer, server_default='0') - policy_ignore_favourites = db.Column(db.Boolean, server_default='TRUE') + policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE') policy_delete_every = db.Column(db.Interval, server_default='0') policy_keep_younger = db.Column(db.Interval, server_default='0') @@ -104,6 +105,14 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin): author = db.relationship(Account, backref=db.backref('posts', order_by=lambda: db.desc(Post.created_at))) + favourite = db.Column(db.Boolean, server_default='FALSE', nullable=False) + + def __repr__(self): + snippet = self.body + if len(snippet) > 20: + snippet = snippet[:19] + "…" + return ''.format(self.id, snippet, self.author_id) + class TwitterArchive(db.Model, TimestampMixin): __tablename__ = 'twitter_archives' diff --git a/routes.py b/routes.py index 12ec85b..10ef841 100644 --- a/routes.py +++ b/routes.py @@ -73,7 +73,7 @@ def upload_tweet_archive(): def settings(): if request.method == 'POST': for attr in ('policy_enabled', - 'policy_ignore_favourites', + 'policy_keep_favourites', 'policy_keep_latest', 'policy_delete_every_significand', 'policy_delete_every_scale', diff --git a/tasks.py b/tasks.py index 2a7f0ce..6ff3ee7 100644 --- a/tasks.py +++ b/tasks.py @@ -11,6 +11,7 @@ from zipfile import ZipFile from io import BytesIO, TextIOWrapper import json from kombu import Queue +import random app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle') app.conf.task_queues = ( @@ -105,8 +106,43 @@ def periodic_cleanup(): delete(synchronize_session=False) db.session.commit() + +@app.task +def queue_deletes(): + eligible_accounts = Account.query.filter(Account.policy_enabled == True).\ + filter(Account.last_delete + Account.policy_delete_every < db.func.now()) + for account in eligible_accounts: + delete_from_account.s(account.id).apply_async() + +@app.task +def delete_from_account(account_id): + account = Account.query.get(account_id) + latest_n_posts = db.session.query(Post.id).with_parent(account).order_by(db.desc(Post.created_at)).limit(account.policy_keep_latest) + posts = Post.query.with_parent(account).\ + filter(Post.created_at + account.policy_keep_younger <= db.func.now()).\ + filter(~Post.id.in_(latest_n_posts)).\ + order_by(db.func.random()).limit(100).all() + + if account.service == 'twitter': + posts = lib.twitter.refresh_posts(posts) + eligible = list((post for post in posts if not account.policy_keep_favourites or not post.favourite)) + if eligible: + if account.policy_delete_every == timedelta(0): + print("deleting all {} eligible posts for {}".format(len(eligible), account)) + for post in eligible: + lib.twitter.delete(post) + else: + post = random.choice(list((post for post in posts if not account.policy_keep_favourites or not post.favourite))) + print("deleting {}".format(post)) + lib.twitter.delete(post) + account.last_delete = db.func.now() + + db.session.commit() + app.add_periodic_task(6*60*60, periodic_cleanup) -app.add_periodic_task(60, queue_fetch_for_most_stale_accounts) +app.add_periodic_task(45, queue_fetch_for_most_stale_accounts) +app.add_periodic_task(45, queue_deletes) if __name__ == '__main__': app.worker_main() + diff --git a/templates/index.html b/templates/index.html index 987c787..20323ee 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,7 +11,7 @@

Recent posts ({{g.viewer.account.post_count()}} total posts):

{% for post in posts %} -

{{post.body}}

+

{{ "♥ " if post.favourite }}{{post.body}}

{% else %}

no posts :(

{% endfor %} diff --git a/templates/settings.html b/templates/settings.html index 36d4c19..437f2cd 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -22,8 +22,8 @@ latest posts

Keep posts that I have favourited - - + +