diff --git a/lib/twitter.py b/lib/twitter.py index 78b24ec..48921c1 100644 --- a/lib/twitter.py +++ b/lib/twitter.py @@ -37,6 +37,7 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se acct = db.session.merge(acct) acct.remote_display_name = remote_acct['name'] + acct.remote_screen_name = remote_acct['screen_name'] acct.remote_avatar_url = remote_acct['profile_image_url_https'] new_token.account = acct db.session.commit() @@ -55,6 +56,12 @@ locale.setlocale(locale.LC_TIME, 'C') # jeez i hate that i have to do this def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret) + user = t.account.verify_credentials() + + account.remote_display_name = user['name'] + account.remote_screen_name = user['screen_name'] + account.remote_avatar_url = user['profile_image_url_https'] + kwargs = { 'user_id': account.remote_id, 'count': 200, 'trim_user': True } kwargs.update(cursor or {}) @@ -67,14 +74,14 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): tweets = t.statuses.user_timeline(**kwargs) - print("processing %s tweets" % (len(tweets),)) + print("processing {} tweets for {acc}".format(len(tweets), acc=account)) if len(tweets) > 0: kwargs['max_id'] = +inf for tweet in tweets: - print(tweet['text']) + print("TWEET ", tweet['text']) post = Post(remote_id=tweet['id_str']) post = db.session.merge(post) post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y') @@ -85,8 +92,6 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): else: kwargs = None - - account.last_fetch = db.func.now() db.session.commit() return kwargs diff --git a/migrations/versions/1727266feaff_.py b/migrations/versions/1727266feaff_.py new file mode 100644 index 0000000..b2debe1 --- /dev/null +++ b/migrations/versions/1727266feaff_.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 1727266feaff +Revises: 1003f9df0ae0 +Create Date: 2017-07-29 11:09:02.743619 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '1727266feaff' +down_revision = '1003f9df0ae0' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('accounts', sa.Column('remote_screen_name', sa.String(), nullable=True)) + + +def downgrade(): + op.drop_column('accounts', 'remote_screen_name') diff --git a/model.py b/model.py index 20f5b60..c762ca9 100644 --- a/model.py +++ b/model.py @@ -24,12 +24,16 @@ class Account(db.Model, TimestampMixin): # policy_ignore_favourites = db.Column(db.Boolean, server_default='TRUE') remote_display_name = db.Column(db.String) + remote_screen_name = db.Column(db.String) remote_avatar_url = db.Column(db.String) last_fetch = db.Column(db.DateTime, server_default='epoch') # backref: tokens + def __repr__(self): + return f"" + class OAuthToken(db.Model, TimestampMixin): __tablename__ = 'oauth_tokens' diff --git a/tasks.py b/tasks.py index 079c462..349ec0f 100644 --- a/tasks.py +++ b/tasks.py @@ -18,9 +18,18 @@ def remove_old_sessions(): @app.task(autoretry_for=(TwitterError, URLError)) def fetch_acc(remote_id, cursor=None): - cursor = lib.twitter.fetch_acc(Account.query.get(remote_id), cursor, **flaskapp.config.get_namespace("TWITTER_")) - if cursor: - fetch_acc.si(remote_id, cursor).apply_async() + acc = Account.query.get(remote_id) + print(f'fetching {acc}') + try: + cursor = lib.twitter.fetch_acc(acc, cursor, **flaskapp.config.get_namespace("TWITTER_")) + if cursor: + fetch_acc.si(remote_id, cursor).apply_async() + finally: + db.session.rollback() + acc.last_fetch = db.func.now() + db.session.commit() + + @app.task def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limit=20): @@ -29,7 +38,6 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi .order_by(db.asc(Account.last_fetch))\ .limit(limit) for acc in accs: - print("queueing fetch for %s" % (acc.remote_display_name)) fetch_acc.s(acc.remote_id).delay() acc.last_fetch = db.func.now() db.session.commit()