idk!!!!!!!!!! i forgot to commit last night

lib/twitter.py

@@ -52,24 +52,29 @@ def get_twitter_for_acc(account, consumer_key=None, consumer_secret=None):
 import locale
 locale.setlocale(locale.LC_TIME, 'C') # jeez i hate that i have to do this
 
-def fetch_acc(account, consumer_key=None, consumer_secret=None):
+def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
     t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret)
 
     kwargs = { 'user_id': account.remote_id, 'count': 200, 'trim_user': True }
+    kwargs.update(cursor or {})
 
-    most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.remote_id).first()
-    if most_recent_post:
-        kwargs['since_id'] = most_recent_post.remote_id
+    if 'max_id' not in kwargs:
+        most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.remote_id).first()
+        if most_recent_post:
+            kwargs['since_id'] = most_recent_post.remote_id
 
-    while True:
-        tweets = t.statuses.user_timeline(**kwargs)
+    print(kwargs)
 
-        if len(tweets) == 0:
-            break
+    tweets = t.statuses.user_timeline(**kwargs)
+
+    print("processing %s tweets" % (len(tweets),))
+
+    if len(tweets) > 0:
 
         kwargs['max_id'] = +inf
 
         for tweet in tweets:
+            print(tweet['text'])
             post = Post(remote_id=tweet['id_str'])
             post = db.session.merge(post)
             post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
@@ -77,7 +82,12 @@ def fetch_acc(account, consumer_key=None, consumer_secret=None):
             post.author = account
             kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id'])
 
+    else:
+        kwargs = None
 
-    account.last_fetch = datetime.now()
+
+    account.last_fetch = db.func.now()
     db.session.commit()
 
+    return kwargs
+

routes.py

@@ -21,7 +21,6 @@ def touch_viewer(resp):
 
 @app.route('/')
 def index():
-    print('hey')
     if g.viewer:
         posts = Post.query.filter_by(author_id = g.viewer.account_id).order_by(db.desc(Post.created_at)).limit(30)
         return render_template('index.html', posts=posts)

tasks.py

@@ -5,6 +5,7 @@ from app import db
 from model import Session, Account
 import lib.twitter
 from twitter import TwitterError
+from urllib.error import URLError
 from datetime import timedelta, datetime
 
 app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle')
@@ -15,17 +16,20 @@ def remove_old_sessions():
             delete(synchronize_session=False)
     db.session.commit()
 
-@app.task(autoretry_for=(TwitterError,))
-def fetch_acc(remote_id):
-    lib.twitter.fetch_acc(Account.query.get(remote_id), **flaskapp.config.get_namespace("TWITTER_"))
+@app.task(autoretry_for=(TwitterError, URLError))
+def fetch_acc(remote_id, cursor=None):
+    cursor = lib.twitter.fetch_acc(Account.query.get(remote_id), cursor, **flaskapp.config.get_namespace("TWITTER_"))
+    if cursor:
+        fetch_acc.si(remote_id, cursor).apply_async()
 
 @app.task
-def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=1), limit=20):
+def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limit=20):
     accs = Account.query\
             .filter(Account.last_fetch < db.func.now() - min_staleness)\
             .order_by(db.asc(Account.last_fetch))\
             .limit(limit)
     for acc in accs:
+        print("queueing fetch for %s" % (acc.remote_display_name))
         fetch_acc.s(acc.remote_id).delay()
         acc.last_fetch = db.func.now()
     db.session.commit()