From e08a0eea130c8f2f707177f124154d8a41b14658 Mon Sep 17 00:00:00 2001
From: codl
Date: Wed, 2 Aug 2017 01:35:09 +0200
Subject: [PATCH] ahhh im super good at computing
---
lib/interval.py | 5 +--
lib/twitter.py | 55 ++++++++++++++++++----------
migrations/versions/7379bfac4379_.py | 28 ++++++++++++++
migrations/versions/995890c26959_.py | 30 +++++++++++++++
model.py | 11 +++++-
routes.py | 2 +-
tasks.py | 38 ++++++++++++++++++-
templates/index.html | 2 +-
templates/settings.html | 4 +-
9 files changed, 146 insertions(+), 29 deletions(-)
create mode 100644 migrations/versions/7379bfac4379_.py
create mode 100644 migrations/versions/995890c26959_.py
diff --git a/lib/interval.py b/lib/interval.py
index 295426b..6dabc0a 100644
--- a/lib/interval.py
+++ b/lib/interval.py
@@ -1,7 +1,6 @@
from datetime import timedelta
SCALES = [
- ('seconds', timedelta(seconds=1)),
('minutes', timedelta(minutes=1)),
('hours', timedelta(hours=1)),
('days', timedelta(days=1)),
@@ -21,13 +20,13 @@ def decompose_interval(attrname):
def scale(self):
if getattr(self, attrname) == timedelta(0):
- return timedelta(seconds=1)
+ return timedelta(minutes=1)
for m in scales:
if getattr(self, attrname) % m == timedelta(0):
return m
- return timedelta(seconds=1)
+ return timedelta(minutes=1)
@scale.setter
def scale(self, value):
diff --git a/lib/twitter.py b/lib/twitter.py
index 36bfe09..4685083 100644
--- a/lib/twitter.py
+++ b/lib/twitter.py
@@ -1,7 +1,7 @@
from twitter import Twitter, OAuth
from werkzeug.urls import url_decode
from model import OAuthToken, Account, Post
-from app import db
+from app import db, app
from math import inf
from datetime import datetime
import locale
@@ -45,30 +45,22 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
return new_token
-def get_twitter_for_acc(account, consumer_key=None, consumer_secret=None):
- token = account.tokens[0]
+def get_twitter_for_acc(account):
+
+ consumer_key = app.config['TWITTER_CONSUMER_KEY']
+ consumer_secret = app.config['TWITTER_CONSUMER_SECRET']
+
+ token = OAuthToken.query.with_parent(account).order_by(db.desc(OAuthToken.created_at)).first()
t = Twitter(
auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret))
return t
locale.setlocale(locale.LC_TIME, 'C')
-def csv_tweet_to_json_tweet(tweet, account):
- tweet.update({
- 'id': int(tweet['tweet_id']),
- 'id_str': tweet['tweet_id'],
- 'created_at': datetime.strptime(tweet['timestamp'],
- '%Y-%m-%d %H:%M:%S %z')\
- .strftime('%a %b %d %H:%M:%S %z %Y'),
- 'user': {
- 'id': int(account.twitter_id),
- 'id_str': account.twitter_id
- }
- })
- return tweet
-
-def tweet_to_post(tweet):
- post = Post(twitter_id=tweet['id_str'])
+def tweet_to_post(tweet, post=None):
+ if not post:
+ post = Post()
+ post.twitter_id = tweet['id_str']
try:
post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
except ValueError:
@@ -79,10 +71,12 @@ def tweet_to_post(tweet):
else:
post.body = tweet['text']
post.author_id = 'twitter:{}'.format(tweet['user']['id_str'])
+ if 'favorited' in tweet:
+ post.favourite = tweet['favorited']
return post
def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
- t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret)
+ t = get_twitter_for_acc(account)
user = t.account.verify_credentials()
@@ -117,3 +111,24 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
return kwargs
+
+def refresh_posts(posts):
+ t = get_twitter_for_acc(posts[0].author)
+ tweets = t.statuses.lookup(_id=",".join((post.twitter_id for post in posts)),
+ trim_user = True, tweet_mode = 'extended')
+ refreshed_posts = list()
+ for post in posts:
+ tweet = next((tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), None)
+ if not tweet:
+ session.delete(post)
+ else:
+ post = db.session.merge(tweet_to_post(tweet))
+ refreshed_posts.append(post)
+
+ return refreshed_posts
+
+
+def delete(post):
+ t = get_twitter_for_acc(post.author)
+ t.statuses.destroy(id=post.twitter_id)
+ db.session.delete(post)
diff --git a/migrations/versions/7379bfac4379_.py b/migrations/versions/7379bfac4379_.py
new file mode 100644
index 0000000..e69cae5
--- /dev/null
+++ b/migrations/versions/7379bfac4379_.py
@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 7379bfac4379
+Revises: 995890c26959
+Create Date: 2017-08-01 23:35:17.924428
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7379bfac4379'
+down_revision = '995890c26959'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('posts', sa.Column('favourite', sa.Boolean(), server_default='FALSE', nullable=False))
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column('posts', 'favourite')
+ # ### end Alembic commands ###
diff --git a/migrations/versions/995890c26959_.py b/migrations/versions/995890c26959_.py
new file mode 100644
index 0000000..b9599a4
--- /dev/null
+++ b/migrations/versions/995890c26959_.py
@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 995890c26959
+Revises: e036b007017c
+Create Date: 2017-08-01 23:24:22.223674
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '995890c26959'
+down_revision = 'e036b007017c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('accounts', sa.Column('policy_keep_favourites', sa.Boolean(), server_default='TRUE', nullable=True))
+ op.drop_column('accounts', 'policy_ignore_favourites')
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('accounts', sa.Column('policy_ignore_favourites', sa.BOOLEAN(), server_default=sa.text('true'), autoincrement=False, nullable=True))
+ op.drop_column('accounts', 'policy_keep_favourites')
+ # ### end Alembic commands ###
diff --git a/model.py b/model.py
index 6165e2f..1e15483 100644
--- a/model.py
+++ b/model.py
@@ -5,6 +5,7 @@ from app import db
from twitter import Twitter, OAuth
import secrets
from lib import decompose_interval
+from datetime import timedelta
class TimestampMixin(object):
created_at = db.Column(db.DateTime, server_default=db.func.now())
@@ -42,7 +43,7 @@ class Account(TimestampMixin, RemoteIDMixin):
policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
policy_keep_latest = db.Column(db.Integer, server_default='0')
- policy_ignore_favourites = db.Column(db.Boolean, server_default='TRUE')
+ policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE')
policy_delete_every = db.Column(db.Interval, server_default='0')
policy_keep_younger = db.Column(db.Interval, server_default='0')
@@ -104,6 +105,14 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin):
author = db.relationship(Account,
backref=db.backref('posts', order_by=lambda: db.desc(Post.created_at)))
+ favourite = db.Column(db.Boolean, server_default='FALSE', nullable=False)
+
+ def __repr__(self):
+ snippet = self.body
+ if len(snippet) > 20:
+ snippet = snippet[:19] + "…"
+ return ''.format(self.id, snippet, self.author_id)
+
class TwitterArchive(db.Model, TimestampMixin):
__tablename__ = 'twitter_archives'
diff --git a/routes.py b/routes.py
index 12ec85b..10ef841 100644
--- a/routes.py
+++ b/routes.py
@@ -73,7 +73,7 @@ def upload_tweet_archive():
def settings():
if request.method == 'POST':
for attr in ('policy_enabled',
- 'policy_ignore_favourites',
+ 'policy_keep_favourites',
'policy_keep_latest',
'policy_delete_every_significand',
'policy_delete_every_scale',
diff --git a/tasks.py b/tasks.py
index 2a7f0ce..6ff3ee7 100644
--- a/tasks.py
+++ b/tasks.py
@@ -11,6 +11,7 @@ from zipfile import ZipFile
from io import BytesIO, TextIOWrapper
import json
from kombu import Queue
+import random
app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle')
app.conf.task_queues = (
@@ -105,8 +106,43 @@ def periodic_cleanup():
delete(synchronize_session=False)
db.session.commit()
+
+@app.task
+def queue_deletes():
+ eligible_accounts = Account.query.filter(Account.policy_enabled == True).\
+ filter(Account.last_delete + Account.policy_delete_every < db.func.now())
+ for account in eligible_accounts:
+ delete_from_account.s(account.id).apply_async()
+
+@app.task
+def delete_from_account(account_id):
+ account = Account.query.get(account_id)
+ latest_n_posts = db.session.query(Post.id).with_parent(account).order_by(db.desc(Post.created_at)).limit(account.policy_keep_latest)
+ posts = Post.query.with_parent(account).\
+ filter(Post.created_at + account.policy_keep_younger <= db.func.now()).\
+ filter(~Post.id.in_(latest_n_posts)).\
+ order_by(db.func.random()).limit(100).all()
+
+ if account.service == 'twitter':
+ posts = lib.twitter.refresh_posts(posts)
+ eligible = list((post for post in posts if not account.policy_keep_favourites or not post.favourite))
+ if eligible:
+ if account.policy_delete_every == timedelta(0):
+ print("deleting all {} eligible posts for {}".format(len(eligible), account))
+ for post in eligible:
+ lib.twitter.delete(post)
+ else:
+ post = random.choice(list((post for post in posts if not account.policy_keep_favourites or not post.favourite)))
+ print("deleting {}".format(post))
+ lib.twitter.delete(post)
+ account.last_delete = db.func.now()
+
+ db.session.commit()
+
app.add_periodic_task(6*60*60, periodic_cleanup)
-app.add_periodic_task(60, queue_fetch_for_most_stale_accounts)
+app.add_periodic_task(45, queue_fetch_for_most_stale_accounts)
+app.add_periodic_task(45, queue_deletes)
if __name__ == '__main__':
app.worker_main()
+
diff --git a/templates/index.html b/templates/index.html
index 987c787..20323ee 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -11,7 +11,7 @@
Recent posts ({{g.viewer.account.post_count()}} total posts):
{% for post in posts %}
- {{post.body}}
+ {{ "♥ " if post.favourite }}{{post.body}}
{% else %}
no posts :(
{% endfor %}
diff --git a/templates/settings.html b/templates/settings.html
index 36d4c19..437f2cd 100644
--- a/templates/settings.html
+++ b/templates/settings.html
@@ -22,8 +22,8 @@
latest posts
Keep posts that I have favourited
-
-
+
+