From 9722dbde26c799273ca21cc49abe8d23aeae3ad4 Mon Sep 17 00:00:00 2001 From: codl Date: Thu, 3 Aug 2017 20:01:06 +0200 Subject: [PATCH] ahhhhhhhhhhhhhh!!!!!!!!!!!!!!!!!!!!!!!!! posts --- lib/twitter.py | 27 ++++++++++-------- ...7366_add_reported_post_count_column_to_.py | 24 ++++++++++++++++ .../2e3a2882e5a4_add_policy_defaults.py | 28 +++++++++++++++++++ model.py | 8 ++++-- routes.py | 4 +-- tasks.py | 4 +-- templates/logged_in.html | 10 +++++-- 7 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 migrations/versions/0983f1227366_add_reported_post_count_column_to_.py create mode 100644 migrations/versions/2e3a2882e5a4_add_policy_defaults.py diff --git a/lib/twitter.py b/lib/twitter.py index a03e15a..1de464d 100644 --- a/lib/twitter.py +++ b/lib/twitter.py @@ -20,6 +20,14 @@ def get_login_url(callback='oob', consumer_key=None, consumer_secret=None): return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,) +def account_from_api_user_object(obj): + return Account( + twitter_id = obj['id_str'], + display_name = obj['name'], + screen_name = obj['screen_name'], + avatar_url = obj['profile_image_url_https'], + reported_post_count = obj['statuses_count']) + def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None): temp_token = OAuthToken.query.get(oauth_token) if not temp_token: @@ -34,12 +42,9 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se new_twitter = Twitter( auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret)) remote_acct = new_twitter.account.verify_credentials() - acct = Account(twitter_id = remote_acct['id_str']) + acct = account_from_api_user_object(remote_acct) acct = db.session.merge(acct) - acct.display_name = remote_acct['name'] - acct.screen_name = remote_acct['screen_name'] - acct.avatar_url = remote_acct['profile_image_url_https'] new_token.account = acct db.session.commit() @@ -57,7 +62,7 @@ def get_twitter_for_acc(account): locale.setlocale(locale.LC_TIME, 'C') -def tweet_to_post(tweet, post=None): +def post_from_api_tweet_object(tweet, post=None): if not post: post = Post() post.twitter_id = tweet['id_str'] @@ -79,13 +84,11 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): t = get_twitter_for_acc(account) user = t.account.verify_credentials() - - account.remote_display_name = user['name'] - account.remote_screen_name = user['screen_name'] - account.remote_avatar_url = user['profile_image_url_https'] + db.session.merge(account_from_api_user_object(user)) kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' } - kwargs.update(cursor or {}) + if cursor: + kwargs.update(cursor) if 'max_id' not in kwargs: most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first() @@ -101,7 +104,7 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): kwargs['max_id'] = +inf for tweet in tweets: - db.session.merge(tweet_to_post(tweet)) + db.session.merge(post_from_api_tweet_object(tweet)) kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id']) else: @@ -125,7 +128,7 @@ def refresh_posts(posts): if not tweet: session.delete(post) else: - post = db.session.merge(tweet_to_post(tweet)) + post = db.session.merge(post_from_api_tweet_object(tweet)) refreshed_posts.append(post) return refreshed_posts diff --git a/migrations/versions/0983f1227366_add_reported_post_count_column_to_.py b/migrations/versions/0983f1227366_add_reported_post_count_column_to_.py new file mode 100644 index 0000000..389d8c6 --- /dev/null +++ b/migrations/versions/0983f1227366_add_reported_post_count_column_to_.py @@ -0,0 +1,24 @@ +"""add reported post count column to account + +Revision ID: 0983f1227366 +Revises: 7afc95e24778 +Create Date: 2017-08-03 19:16:55.883575 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0983f1227366' +down_revision = '7afc95e24778' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('accounts', sa.Column('reported_post_count', sa.Integer(), nullable=True)) + + +def downgrade(): + op.drop_column('accounts', 'reported_post_count') diff --git a/migrations/versions/2e3a2882e5a4_add_policy_defaults.py b/migrations/versions/2e3a2882e5a4_add_policy_defaults.py new file mode 100644 index 0000000..9a53a88 --- /dev/null +++ b/migrations/versions/2e3a2882e5a4_add_policy_defaults.py @@ -0,0 +1,28 @@ +"""add policy defaults + +Revision ID: 2e3a2882e5a4 +Revises: 0983f1227366 +Create Date: 2017-08-03 19:54:20.643627 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2e3a2882e5a4' +down_revision = '0983f1227366' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column('accounts', 'policy_keep_latest', server_default='100') + op.alter_column('accounts', 'policy_keep_younger', server_default='365 days') + op.alter_column('accounts', 'policy_delete_every', server_default='30 minutes') + + +def downgrade(): + op.alter_column('accounts', 'policy_keep_latest', server_default='0') + op.alter_column('accounts', 'policy_keep_younger', server_default='0') + op.alter_column('accounts', 'policy_delete_every', server_default='0') diff --git a/model.py b/model.py index f06435a..8a3c83e 100644 --- a/model.py +++ b/model.py @@ -42,14 +42,15 @@ class Account(TimestampMixin, RemoteIDMixin): id = db.Column(db.String, primary_key=True) policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False) - policy_keep_latest = db.Column(db.Integer, server_default='0', nullable=False) + policy_keep_latest = db.Column(db.Integer, server_default='100', nullable=False) policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False) - policy_delete_every = db.Column(db.Interval, server_default='0', nullable=False) - policy_keep_younger = db.Column(db.Interval, server_default='0', nullable=False) + policy_delete_every = db.Column(db.Interval, server_default='30 minutes', nullable=False) + policy_keep_younger = db.Column(db.Interval, server_default='365 days', nullable=False) display_name = db.Column(db.String) screen_name = db.Column(db.String) avatar_url = db.Column(db.String) + reported_post_count = db.Column(db.Integer) last_fetch = db.Column(db.DateTime, server_default='epoch') last_delete = db.Column(db.DateTime, server_default='epoch') @@ -77,6 +78,7 @@ class Account(TimestampMixin, RemoteIDMixin): return Post.query.with_parent(self).count() + class Account(Account, db.Model): pass diff --git a/routes.py b/routes.py index c4dba18..c4b3928 100644 --- a/routes.py +++ b/routes.py @@ -63,9 +63,9 @@ def upload_tweet_archive(): db.session.add(ta) db.session.commit() - tasks.import_twitter_archive.s(ta.id).apply_async(routing_key='high') + tasks.chunk_twitter_archive(ta.id) - return redirect(url_for('index')) + return redirect(url_for('index', _anchor='recent_archives')) @app.route('/settings', methods=('POST',)) @require_auth diff --git a/tasks.py b/tasks.py index ebb7966..72a9847 100644 --- a/tasks.py +++ b/tasks.py @@ -49,7 +49,7 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi db.session.commit() @app.task -def import_twitter_archive(archive_id): +def chunk_twitter_archive(archive_id): ta = TwitterArchive.query.get(archive_id) with ZipFile(BytesIO(ta.body), 'r') as zipfile: @@ -79,7 +79,7 @@ def import_twitter_archive_month(archive_id, month_path): tweets = json.load(f) for tweet in tweets: - post = lib.twitter.tweet_to_post(tweet) + post = lib.twitter.post_from_api_tweet_object(tweet) existing_post = db.session.query(Post).get(post.id) if post.author_id != ta.account_id \ diff --git a/templates/logged_in.html b/templates/logged_in.html index 924cc59..5a388b7 100644 --- a/templates/logged_in.html +++ b/templates/logged_in.html @@ -20,7 +20,13 @@ {% endif %} -

Currently keeping track of {{g.viewer.account.post_count()}} posts

+{% set post_count = g.viewer.account.post_count() %} +

Currently keeping track of {{ post_count }} posts

+{% if g.viewer.account.service == 'twitter' + and post_count < g.viewer.account.reported_post_count * 3/4 + and g.viewer.account.reported_post_count > 3200 -%} + +{% endif %}
@@ -64,7 +70,7 @@ {% if g.viewer.account.twitter_archives %} -

Recent imports

+

Recent archives