ahhhhhhhhhhhhhh!!!!!!!!!!!!!!!!!!!!!!!!! posts

lib/twitter.py

@@ -20,6 +20,14 @@ def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
 
     return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,)
 
+def account_from_api_user_object(obj):
+    return Account(
+            twitter_id = obj['id_str'],
+            display_name = obj['name'],
+            screen_name = obj['screen_name'],
+            avatar_url = obj['profile_image_url_https'],
+            reported_post_count = obj['statuses_count'])
+
 def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None):
     temp_token = OAuthToken.query.get(oauth_token)
     if not temp_token:
@@ -34,12 +42,9 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
     new_twitter = Twitter(
             auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret))
     remote_acct = new_twitter.account.verify_credentials()
-    acct = Account(twitter_id = remote_acct['id_str'])
+    acct = account_from_api_user_object(remote_acct)
     acct = db.session.merge(acct)
 
-    acct.display_name = remote_acct['name']
-    acct.screen_name = remote_acct['screen_name']
-    acct.avatar_url = remote_acct['profile_image_url_https']
     new_token.account = acct
     db.session.commit()
 
@@ -57,7 +62,7 @@ def get_twitter_for_acc(account):
 
 locale.setlocale(locale.LC_TIME, 'C')
 
-def tweet_to_post(tweet, post=None):
+def post_from_api_tweet_object(tweet, post=None):
     if not post:
         post = Post()
     post.twitter_id = tweet['id_str']
@@ -79,13 +84,11 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
     t = get_twitter_for_acc(account)
 
     user = t.account.verify_credentials()
-
-    account.remote_display_name = user['name']
-    account.remote_screen_name = user['screen_name']
-    account.remote_avatar_url = user['profile_image_url_https']
+    db.session.merge(account_from_api_user_object(user))
 
     kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' }
-    kwargs.update(cursor or {})
+    if cursor:
+        kwargs.update(cursor)
 
     if 'max_id' not in kwargs:
         most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first()
@@ -101,7 +104,7 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
         kwargs['max_id'] = +inf
 
         for tweet in tweets:
-            db.session.merge(tweet_to_post(tweet))
+            db.session.merge(post_from_api_tweet_object(tweet))
             kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id'])
 
     else:
@@ -125,7 +128,7 @@ def refresh_posts(posts):
         if not tweet:
             session.delete(post)
         else:
-            post = db.session.merge(tweet_to_post(tweet))
+            post = db.session.merge(post_from_api_tweet_object(tweet))
             refreshed_posts.append(post)
 
     return refreshed_posts

migrations/versions/0983f1227366_add_reported_post_count_column_to_.py

@@ -0,0 +1,24 @@
+"""add reported post count column to account
+
+Revision ID: 0983f1227366
+Revises: 7afc95e24778
+Create Date: 2017-08-03 19:16:55.883575
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0983f1227366'
+down_revision = '7afc95e24778'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('reported_post_count', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('accounts', 'reported_post_count')

migrations/versions/2e3a2882e5a4_add_policy_defaults.py

@@ -0,0 +1,28 @@
+"""add policy defaults
+
+Revision ID: 2e3a2882e5a4
+Revises: 0983f1227366
+Create Date: 2017-08-03 19:54:20.643627
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2e3a2882e5a4'
+down_revision = '0983f1227366'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column('accounts', 'policy_keep_latest', server_default='100')
+    op.alter_column('accounts', 'policy_keep_younger', server_default='365 days')
+    op.alter_column('accounts', 'policy_delete_every', server_default='30 minutes')
+
+
+def downgrade():
+    op.alter_column('accounts', 'policy_keep_latest', server_default='0')
+    op.alter_column('accounts', 'policy_keep_younger', server_default='0')
+    op.alter_column('accounts', 'policy_delete_every', server_default='0')

model.py

@@ -42,14 +42,15 @@ class Account(TimestampMixin, RemoteIDMixin):
     id = db.Column(db.String, primary_key=True)
 
     policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
-    policy_keep_latest = db.Column(db.Integer, server_default='0', nullable=False)
+    policy_keep_latest = db.Column(db.Integer, server_default='100', nullable=False)
     policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False)
-    policy_delete_every = db.Column(db.Interval, server_default='0', nullable=False)
-    policy_keep_younger = db.Column(db.Interval, server_default='0', nullable=False)
+    policy_delete_every = db.Column(db.Interval, server_default='30 minutes', nullable=False)
+    policy_keep_younger = db.Column(db.Interval, server_default='365 days', nullable=False)
 
     display_name = db.Column(db.String)
     screen_name = db.Column(db.String)
     avatar_url = db.Column(db.String)
+    reported_post_count = db.Column(db.Integer)
 
     last_fetch = db.Column(db.DateTime, server_default='epoch')
     last_delete = db.Column(db.DateTime, server_default='epoch')
@@ -77,6 +78,7 @@ class Account(TimestampMixin, RemoteIDMixin):
         return Post.query.with_parent(self).count()
 
 
+
 class Account(Account, db.Model):
     pass
 

routes.py

@@ -63,9 +63,9 @@ def upload_tweet_archive():
     db.session.add(ta)
     db.session.commit()
 
-    tasks.import_twitter_archive.s(ta.id).apply_async(routing_key='high')
+    tasks.chunk_twitter_archive(ta.id)
 
-    return redirect(url_for('index'))
+    return redirect(url_for('index', _anchor='recent_archives'))
 
 @app.route('/settings', methods=('POST',))
 @require_auth

tasks.py

@@ -49,7 +49,7 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi
     db.session.commit()
 
 @app.task
-def import_twitter_archive(archive_id):
+def chunk_twitter_archive(archive_id):
     ta = TwitterArchive.query.get(archive_id)
 
     with ZipFile(BytesIO(ta.body), 'r') as zipfile:
@@ -79,7 +79,7 @@ def import_twitter_archive_month(archive_id, month_path):
                 tweets = json.load(f)
 
         for tweet in tweets:
-            post = lib.twitter.tweet_to_post(tweet)
+            post = lib.twitter.post_from_api_tweet_object(tweet)
             existing_post = db.session.query(Post).get(post.id)
 
             if post.author_id != ta.account_id \

templates/logged_in.html

@@ -20,7 +20,13 @@
     </form>
 {% endif %}
 </div>
-<p>Currently keeping track of {{g.viewer.account.post_count()}} posts</p>
+{% set post_count = g.viewer.account.post_count() %}
+<p>Currently keeping track of {{ post_count }} posts</p>
+{% if g.viewer.account.service == 'twitter'
+        and post_count < g.viewer.account.reported_post_count * 3/4
+        and g.viewer.account.reported_post_count > 3200 -%}
+<div class='banner warning'>Are we missing some of your posts? Try uploading a tweet archive below</div>
+{% endif %}
 </section>
 
 <section>
@@ -64,7 +70,7 @@
 </form>
 
 {% if g.viewer.account.twitter_archives %}
-    <h3 >Recent imports</h3>
+    <h3 name="recent_archives">Recent archives</h3>
     <ul>
     {% for archive in g.viewer.account.twitter_archives %}
         <li>{{archive.created_at.strftime('%Y-%m-%d %H:%M')}}<br>