yes i do the fetching yes i do the cleaning up sessions

This commit is contained in:
codl 2017-07-28 00:08:20 +02:00
parent f5fb843fda
commit 8164f786ec
No known key found for this signature in database
GPG Key ID: 6CD7C8891ED1233A
6 changed files with 98 additions and 13 deletions

View File

@ -1,7 +1,9 @@
from twitter import Twitter, OAuth
from werkzeug.urls import url_decode
from model import OAuthToken, Account
from model import OAuthToken, Account, Post
from app import db
from math import inf
from datetime import datetime
def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
twitter = Twitter(
@ -38,3 +40,42 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
new_token.account = acct
db.session.commit()
return new_token
def get_twitter_for_acc(account, consumer_key=None, consumer_secret=None):
token = account.tokens[0]
t = Twitter(
auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret))
return t
import locale
locale.setlocale(locale.LC_TIME, 'C') # jeez i hate that i have to do this
def fetch_posts_for_acc(account, consumer_key=None, consumer_secret=None):
t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret)
kwargs = { 'user_id': account.remote_id, 'count': 200, 'trim_user': True }
#most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.remote_id).first()
#if most_recent_post:
# kwargs['since_id'] = most_recent_post.remote_id
while True:
tweets = t.statuses.user_timeline(**kwargs)
if len(tweets) == 0:
break
kwargs['max_id'] = +inf
for tweet in tweets:
post = Post(remote_id=tweet['id_str'])
post = db.session.merge(post)
post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
post.body = tweet['text']
post.author = account
kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id'])
account.last_post_fetch = datetime.now()
db.session.commit()

View File

@ -0,0 +1,28 @@
"""empty message
Revision ID: 8c2ce3a66650
Revises: a5718ca3ead1
Create Date: 2017-07-27 23:35:48.842519
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '8c2ce3a66650'
down_revision = 'a5718ca3ead1'
branch_labels = None
depends_on = None
def upgrade():
op.drop_constraint('fk_posts_author_remote_id_accounts', 'posts', type_='foreignkey')
op.alter_column('posts', 'author_remote_id', new_column_name='author_id')
op.create_foreign_key(op.f('fk_posts_author_id_accounts'), 'posts', 'accounts', ['author_id'], ['remote_id'])
def downgrade():
op.drop_constraint(op.f('fk_posts_author_id_accounts'), 'posts', type_='foreignkey')
op.alter_column('posts', 'author_id', new_column_name='author_remote_id')
op.create_foreign_key('fk_posts_author_remote_id_accounts', 'posts', 'accounts', ['author_remote_id'], ['remote_id'])

View File

@ -28,7 +28,7 @@ class Account(db.Model, TimestampMixin):
last_post_fetch = db.Column(db.DateTime, server_default='epoch')
# backref: posts
# backref: tokens
class OAuthToken(db.Model, TimestampMixin):
__tablename__ = 'oauth_tokens'
@ -37,7 +37,7 @@ class OAuthToken(db.Model, TimestampMixin):
token_secret = db.Column(db.String, nullable=False)
remote_id = db.Column(db.String, db.ForeignKey('accounts.remote_id'))
account = db.relationship(Account)
account = db.relationship(Account, backref=db.backref('tokens', order_by=lambda: db.desc(OAuthToken.created_at)))
class Session(db.Model, TimestampMixin):
__tablename__ = 'sessions'
@ -53,5 +53,5 @@ class Post(db.Model, TimestampMixin):
remote_id = db.Column(db.String, primary_key=True)
body = db.Column(db.String)
author_remote_id = db.Column(db.String, db.ForeignKey('accounts.remote_id'))
author = db.relationship(Account, lazy='joined', backref='posts')
author_id = db.Column(db.String, db.ForeignKey('accounts.remote_id'))
author = db.relationship(Account)

View File

@ -2,7 +2,7 @@ from app import app
from flask import render_template, url_for, redirect, request, g, Response
from datetime import datetime
import lib.twitter
from model import Account, Session
from model import Account, Session, Post
from app import db
@app.before_request
@ -21,6 +21,10 @@ def touch_viewer(resp):
@app.route('/')
def index():
if g.viewer:
posts = Post.query.order_by(db.desc(Post.created_at)).limit(30)
return render_template('index.html', posts=posts)
else:
return render_template('index.html')
@app.route('/login/twitter')
@ -35,7 +39,7 @@ def twitter_login_step2():
oauth_token = request.args['oauth_token']
oauth_verifier = request.args['oauth_verifier']
token = lib.twitter.receive_verifier(oauth_token, oauth_verifier, **app.config.get_namespace("TWITTER_"))
session = Session(account_remote_id = token.remote_id)
session = Session(account_id = token.remote_id)
db.session.add(session)
db.session.commit()
resp = Response(status=301, headers={"location": url_for('index')})
@ -53,5 +57,5 @@ def logout():
@app.route('/debug')
def debug():
import tasks
tasks.remove_old_sessions.delay()
tasks.fetch_posts.s('808418').delay()
return "hi"

View File

@ -2,22 +2,34 @@ from celery import Celery
from app import app as flaskapp
from app import db
from model import Session
from model import Session, Account
from lib.twitter import fetch_posts_for_acc
from datetime import timedelta
app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle')
@app.task
def remove_old_sessions():
Session.query.filter(Session.updated_at < (db.func.now() - timedelta(minutes=30))).\
Session.query.filter(Session.updated_at < (db.func.now() - timedelta(hours=12))).\
delete(synchronize_session=False)
db.session.commit()
@app.task
def fetch_posts(remote_id):
pass
fetch_posts_for_acc(Account.query.get(remote_id), **flaskapp.config.get_namespace("TWITTER_"))
@app.task
def queue_fetch_for_most_stale_accs(num=5, min_staleness=timedelta(hours=1)):
accs = Account.query\
.filter(Account.last_post_fetch < db.func.now() - min_staleness)\
.order_by(db.asc(Account.last_post_fetch))\
.limit(num)
for acc in accs:
fetch_posts.s(acc.remote_id).delay()
app.add_periodic_task(60*60, remove_old_sessions)
app.add_periodic_task(60, queue_fetch_for_most_stale_accs)

View File

@ -3,7 +3,7 @@
<img src="{{g.viewer.account.remote_avatar_url}}"/>
{{g.viewer.account.remote_display_name}}! <a href="/logout">Log out</a></p>
<p>your posts:</p>
{% for post in g.viewer.account.posts %}
{% for post in posts %}
<p>{{post.body}}</p>
{% else %}
<p>no posts :(</p>