ahhh im super good at computing

lib/interval.py

@@ -1,7 +1,6 @@
 from datetime import timedelta
 
 SCALES = [
-    ('seconds', timedelta(seconds=1)),
     ('minutes', timedelta(minutes=1)),
     ('hours', timedelta(hours=1)),
     ('days', timedelta(days=1)),
@@ -21,13 +20,13 @@ def decompose_interval(attrname):
         def scale(self):
 
             if getattr(self, attrname) == timedelta(0):
-                return timedelta(seconds=1)
+                return timedelta(minutes=1)
 
             for m in scales:
                 if getattr(self, attrname) % m == timedelta(0):
                     return m
 
-            return timedelta(seconds=1)
+            return timedelta(minutes=1)
 
         @scale.setter
         def scale(self, value):

lib/twitter.py

@@ -1,7 +1,7 @@
 from twitter import Twitter, OAuth
 from werkzeug.urls import url_decode
 from model import OAuthToken, Account, Post
-from app import db
+from app import db, app
 from math import inf
 from datetime import datetime
 import locale
@@ -45,30 +45,22 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
 
     return new_token
 
-def get_twitter_for_acc(account, consumer_key=None, consumer_secret=None):
-    token = account.tokens[0]
+def get_twitter_for_acc(account):
+
+    consumer_key = app.config['TWITTER_CONSUMER_KEY']
+    consumer_secret = app.config['TWITTER_CONSUMER_SECRET']
+
+    token = OAuthToken.query.with_parent(account).order_by(db.desc(OAuthToken.created_at)).first()
     t = Twitter(
             auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret))
     return t
 
 locale.setlocale(locale.LC_TIME, 'C')
 
-def csv_tweet_to_json_tweet(tweet, account):
-    tweet.update({
-        'id': int(tweet['tweet_id']),
-        'id_str': tweet['tweet_id'],
-        'created_at': datetime.strptime(tweet['timestamp'],
-            '%Y-%m-%d %H:%M:%S %z')\
-            .strftime('%a %b %d %H:%M:%S %z %Y'),
-        'user': {
-            'id': int(account.twitter_id),
-            'id_str': account.twitter_id
-            }
-    })
-    return tweet
-
-def tweet_to_post(tweet):
-    post = Post(twitter_id=tweet['id_str'])
+def tweet_to_post(tweet, post=None):
+    if not post:
+        post = Post()
+    post.twitter_id = tweet['id_str']
     try:
         post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
     except ValueError:
@@ -79,10 +71,12 @@ def tweet_to_post(tweet):
     else:
         post.body = tweet['text']
     post.author_id = 'twitter:{}'.format(tweet['user']['id_str'])
+    if 'favorited' in tweet:
+        post.favourite = tweet['favorited']
     return post
 
 def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
-    t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret)
+    t = get_twitter_for_acc(account)
 
     user = t.account.verify_credentials()
 
@@ -117,3 +111,24 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
 
     return kwargs
 
+
+def refresh_posts(posts):
+    t = get_twitter_for_acc(posts[0].author)
+    tweets = t.statuses.lookup(_id=",".join((post.twitter_id for post in posts)),
+            trim_user = True, tweet_mode = 'extended')
+    refreshed_posts = list()
+    for post in posts:
+        tweet = next((tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), None)
+        if not tweet:
+            session.delete(post)
+        else:
+            post = db.session.merge(tweet_to_post(tweet))
+            refreshed_posts.append(post)
+
+    return refreshed_posts
+
+
+def delete(post):
+    t = get_twitter_for_acc(post.author)
+    t.statuses.destroy(id=post.twitter_id)
+    db.session.delete(post)

migrations/versions/7379bfac4379_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 7379bfac4379
+Revises: 995890c26959
+Create Date: 2017-08-01 23:35:17.924428
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7379bfac4379'
+down_revision = '995890c26959'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('posts', sa.Column('favourite', sa.Boolean(), server_default='FALSE', nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('posts', 'favourite')
+    # ### end Alembic commands ###

migrations/versions/995890c26959_.py

@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 995890c26959
+Revises: e036b007017c
+Create Date: 2017-08-01 23:24:22.223674
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '995890c26959'
+down_revision = 'e036b007017c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('accounts', sa.Column('policy_keep_favourites', sa.Boolean(), server_default='TRUE', nullable=True))
+    op.drop_column('accounts', 'policy_ignore_favourites')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('accounts', sa.Column('policy_ignore_favourites', sa.BOOLEAN(), server_default=sa.text('true'), autoincrement=False, nullable=True))
+    op.drop_column('accounts', 'policy_keep_favourites')
+    # ### end Alembic commands ###

model.py

@@ -5,6 +5,7 @@ from app import db
 from twitter import Twitter, OAuth
 import secrets
 from lib import decompose_interval
+from datetime import timedelta
 
 class TimestampMixin(object):
     created_at = db.Column(db.DateTime, server_default=db.func.now())
@@ -42,7 +43,7 @@ class Account(TimestampMixin, RemoteIDMixin):
 
     policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
     policy_keep_latest = db.Column(db.Integer, server_default='0')
-    policy_ignore_favourites = db.Column(db.Boolean, server_default='TRUE')
+    policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE')
     policy_delete_every = db.Column(db.Interval, server_default='0')
     policy_keep_younger = db.Column(db.Interval, server_default='0')
 
@@ -104,6 +105,14 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin):
     author = db.relationship(Account,
             backref=db.backref('posts', order_by=lambda: db.desc(Post.created_at)))
 
+    favourite = db.Column(db.Boolean, server_default='FALSE', nullable=False)
+
+    def __repr__(self):
+        snippet = self.body
+        if len(snippet) > 20:
+            snippet = snippet[:19] + "…"
+        return '<Post ({}, "{}", Author: {})>'.format(self.id, snippet, self.author_id)
+
 class TwitterArchive(db.Model, TimestampMixin):
     __tablename__ = 'twitter_archives'
 

routes.py

@@ -73,7 +73,7 @@ def upload_tweet_archive():
 def settings():
     if request.method == 'POST':
         for attr in ('policy_enabled',
-                'policy_ignore_favourites',
+                'policy_keep_favourites',
                 'policy_keep_latest',
                 'policy_delete_every_significand',
                 'policy_delete_every_scale',

tasks.py

@@ -11,6 +11,7 @@ from zipfile import ZipFile
 from io import BytesIO, TextIOWrapper
 import json
 from kombu import Queue
+import random
 
 app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle')
 app.conf.task_queues = (
@@ -105,8 +106,43 @@ def periodic_cleanup():
             delete(synchronize_session=False)
     db.session.commit()
 
+
+@app.task
+def queue_deletes():
+    eligible_accounts = Account.query.filter(Account.policy_enabled == True).\
+            filter(Account.last_delete + Account.policy_delete_every < db.func.now())
+    for account in eligible_accounts:
+        delete_from_account.s(account.id).apply_async()
+
+@app.task
+def delete_from_account(account_id):
+    account = Account.query.get(account_id)
+    latest_n_posts = db.session.query(Post.id).with_parent(account).order_by(db.desc(Post.created_at)).limit(account.policy_keep_latest)
+    posts = Post.query.with_parent(account).\
+        filter(Post.created_at + account.policy_keep_younger <= db.func.now()).\
+        filter(~Post.id.in_(latest_n_posts)).\
+        order_by(db.func.random()).limit(100).all()
+
+    if account.service == 'twitter':
+        posts = lib.twitter.refresh_posts(posts)
+        eligible = list((post for post in posts if not account.policy_keep_favourites or not post.favourite))
+        if eligible:
+            if account.policy_delete_every == timedelta(0):
+                print("deleting all {} eligible posts for {}".format(len(eligible), account))
+                for post in eligible:
+                    lib.twitter.delete(post)
+            else:
+                post = random.choice(list((post for post in posts if not account.policy_keep_favourites or not post.favourite)))
+                print("deleting {}".format(post))
+                lib.twitter.delete(post)
+                account.last_delete = db.func.now()
+
+    db.session.commit()
+
 app.add_periodic_task(6*60*60, periodic_cleanup)
-app.add_periodic_task(60, queue_fetch_for_most_stale_accounts)
+app.add_periodic_task(45, queue_fetch_for_most_stale_accounts)
+app.add_periodic_task(45, queue_deletes)
 
 if __name__ == '__main__':
     app.worker_main()
+

templates/index.html

@@ -11,7 +11,7 @@
     <h2>Recent posts ({{g.viewer.account.post_count()}} total posts):</h2>
 
     {% for post in posts %}
-        <p>{{post.body}}</p>
+    <p>{{ "♥ " if post.favourite }}{{post.body}}</p>
     {% else %}
         <p>no posts :(</p>
     {% endfor %}

templates/settings.html

@@ -22,8 +22,8 @@
         latest posts
     </p>
     <p>Keep posts that I have favourited
-        <label><input type=radio name=policy_ignore_favourites value=true {{ "checked" if g.viewer.account.policy_ignore_favourites }}> Yes</label>
-        <label><input type=radio name=policy_ignore_favourites value=false {{ "checked" if not g.viewer.account.policy_ignore_favourites }}> No</label>
+        <label><input type=radio name=policy_keep_favourites value=true {{ "checked" if g.viewer.account.policy_keep_favourites }}> Yes</label>
+        <label><input type=radio name=policy_keep_favourites value=false {{ "checked" if not g.viewer.account.policy_keep_favourites }}> No</label>
     </p>
     <input type=submit value='Save'>
 </form>