ahhhhhh!! mastodon support

assets/settings.js

@@ -37,7 +37,7 @@
         status_display.textContent='Still saving...';
     }
 
-    function on_change(e){
+    function save(){
         hide_status();
         clearTimeout(status_timeout);
         status_timeout = setTimeout(show_saving, 70);
@@ -94,7 +94,7 @@
     }
 
     for(input of form.elements){
-        input.addEventListener('change', on_change);
+        input.addEventListener('change', save);
     }
 
     // remove submit button since we're doing live updates

lib/mastodon.py

@@ -1,8 +1,11 @@
 import mastodon
 from mastodon import Mastodon
-from model import MastodonApp, Account, OAuthToken
+from mastodon.Mastodon import MastodonAPIError
+from model import MastodonApp, Account, OAuthToken, Post
 from requests import head
 from app import db
+from math import inf
+import iso8601
 
 def get_or_create_app(instance_url, callback, website):
     instance_url = instance_url
@@ -49,17 +52,10 @@ def receive_code(code, app, callback):
             )
 
     remote_acc = api.account_verify_credentials()
-    acc = Account(
-            #id = 'mastodon:{}:{}'.format(app.instance, remote_acc['username']),
-            mastodon_instance = app.instance,
-            mastodon_id = remote_acc['username'],
-            screen_name = remote_acc['username'],
-            display_name = remote_acc['display_name'],
-            avatar_url = remote_acc['avatar'],
-            reported_post_count = remote_acc['statuses_count'],
-        )
+    acc = account_from_api_object(remote_acc, app.instance)
+    acc = db.session.merge(acc)
     token = OAuthToken(account = acc, token = access_token)
-    db.session.merge(acc, token)
+    token = db.session.merge(token)
 
     return acc
 
@@ -71,18 +67,100 @@ def get_api_for_acc(account):
                 client_secret = app.client_secret,
                 api_base_url = '{}://{}'.format(app.protocol, app.instance),
                 access_token = token.token,
+                ratelimit_method = 'throw',
+                #debug_requests = True,
             )
-        try:
-            # api.verify_credentials()
-            # doesnt error even if the token is revoked lol sooooo
-            tl = api.timeline()
-            #if 'error' in tl and tl['error'] == 'The access token was revoked':
-            #ARRRRRGH
-        except mastodon.MastodonAPIError as e:
-            raise e
+
+        # api.verify_credentials()
+        # doesnt error even if the token is revoked lol
+        # https://github.com/tootsuite/mastodon/issues/4637
+        # so we have to do this:
+        tl = api.timeline()
+        if 'error' in tl:
+            db.session.delete(token)
+            continue
         return api
 
+    account.force_log_out()
 
-def fetch_acc(account, cursor=None):
-    pass
 
+def fetch_acc(acc, cursor=None):
+    api = get_api_for_acc(acc)
+    if not api:
+        print('no access, aborting')
+        return None
+
+    newacc = account_from_api_object(api.account_verify_credentials(), acc.mastodon_instance)
+    acc = db.session.merge(newacc)
+
+    kwargs = dict(limit = 40)
+    if cursor:
+        kwargs.update(cursor)
+
+    if 'max_id' not in kwargs:
+        most_recent_post = Post.query.with_parent(acc).order_by(db.desc(Post.created_at)).first()
+        if most_recent_post:
+            kwargs['since_id'] = most_recent_post.mastodon_id
+
+    statuses = api.account_statuses(acc.mastodon_id, **kwargs)
+
+    if statuses:
+        kwargs['max_id'] = +inf
+
+        for status in statuses:
+            post = post_from_api_object(status, acc.mastodon_instance)
+            db.session.merge(post)
+            kwargs['max_id'] = min(kwargs['max_id'], status['id'])
+
+    else:
+        kwargs = None
+
+    db.session.commit()
+
+    return kwargs
+
+def post_from_api_object(obj, instance):
+    return Post(
+            mastodon_instance = instance,
+            mastodon_id = obj['id'],
+            body = obj['content'],
+            favourite = obj['favourited'],
+            has_media = 'media_attachments' in obj and bool(obj['media_attachments']),
+            created_at = iso8601.parse_date(obj['created_at']),
+            author_id = account_from_api_object(obj['account'], instance).id,
+        )
+
+def account_from_api_object(obj, instance):
+    return Account(
+            mastodon_instance = instance,
+            mastodon_id = obj['id'],
+            screen_name = obj['username'],
+            display_name = obj['display_name'],
+            avatar_url = obj['avatar'],
+            reported_post_count = obj['statuses_count'],
+        )
+
+def refresh_posts(posts):
+    acc = posts[0].author
+    api = get_api_for_acc(acc)
+    if not api:
+        raise Exception('no access')
+
+    new_posts = list()
+    for post in posts:
+        try:
+            status = api.status(post.mastodon_id)
+            new_post = db.session.merge(post_from_api_object(status, post.mastodon_instance))
+            new_posts.append(new_post)
+        except MastodonAPIError as e:
+            if str(e) == 'Endpoint not found.':
+                db.session.delete(post)
+            else:
+                raise e
+
+    return new_posts
+
+def delete(post):
+    api = get_api_for_acc(post.author)
+    api.status_delete(post.mastodon_id)
+    db.session.delete(post)

lib/twitter.py

@@ -101,7 +101,7 @@ def post_from_api_tweet_object(tweet, post=None):
         post.has_media = bool('media' in tweet['entities'] and tweet['entities']['media'])
     return post
 
-def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
+def fetch_acc(account, cursor):
     t = get_twitter_for_acc(account)
     if not t:
         print("no twitter access, aborting")

model.py

@@ -95,6 +95,9 @@ class Account(TimestampMixin, RemoteIDMixin):
     def validate_intervals(self, key, value):
         if not (value == timedelta(0) or value >= timedelta(minutes=1)):
             value = timedelta(minutes=1)
+        if key == 'policy_delete_every' and datetime.now() + value < self.next_delete:
+            # make sure that next delete is not in the far future
+            self.next_delete = datetime.now() + value
         return value
 
     @db.validates('policy_keep_latest')

requirements.txt

@@ -18,13 +18,13 @@ Flask-SQLAlchemy==2.2
 gunicorn==19.7.1
 honcho==1.0.1
 idna==2.6
+iso8601==0.1.12
 itsdangerous==0.24
 Jinja2==2.9.6
 kombu==4.1.0
 limits==1.2.1
 Mako==1.0.7
 MarkupSafe==1.0
-Mastodon.py==1.0.8
 olefile==0.44
 Pillow==4.2.1
 psycopg2==2.7.3
@@ -41,3 +41,4 @@ twitter==1.17.1
 urllib3==1.22
 vine==1.1.4
 Werkzeug==0.12.2
+git+https://github.com/codl/Mastodon.py.git@forget

routes.py

@@ -51,19 +51,6 @@ lib.brotli.brotli(app)
 @app.route('/')
 def index():
     if g.viewer:
-        if g.viewer.account.service == 'mastodon':
-            import lib.mastodon
-            api = lib.mastodon.get_api_for_acc(g.viewer.account)
-            me = api.account_verify_credentials()
-            #return str(me)
-            tl = api.timeline()
-            return str(tl)
-            if not api:
-                raise Exception('frick!!!!!')
-            else:
-                raise Exception(api)
-
-
         return render_template('logged_in.html', scales=lib.interval.SCALES,
                 tweet_archive_failed = 'tweet_archive_failed' in request.args,
                 settings_error = 'settings_error' in request.args

tasks.py

@@ -4,6 +4,7 @@ from app import app as flaskapp
 from app import db
 from model import Session, Account, TwitterArchive, Post, OAuthToken
 import lib.twitter
+import lib.mastodon
 from twitter import TwitterError
 from urllib.error import URLError
 from datetime import timedelta, datetime
@@ -46,11 +47,12 @@ def fetch_acc(id, cursor=None):
     acc = Account.query.get(id)
     print(f'fetching {acc}')
     try:
+        action = lambda acc, cursor: None
         if(acc.service == 'twitter'):
             action = lib.twitter.fetch_acc
         elif(acc.service == 'mastodon'):
             action = lib.mastodon.fetch_acc
-        cursor = action(acc, cursor, **flaskapp.config.get_namespace("TWITTER_"))
+        cursor = action(acc, cursor)
         if cursor:
             fetch_acc.si(id, cursor).apply_async()
     finally:
@@ -67,7 +69,7 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi
             .limit(limit)
     for acc in accs:
         fetch_acc.s(acc.id).delay()
-        acc.touch_fetch()
+        #acc.touch_fetch()
     db.session.commit()
 
 
@@ -143,23 +145,28 @@ def delete_from_account(account_id):
         except_(latest_n_posts).\
         order_by(db.func.random()).limit(100).all()
 
-    posts = refresh_posts(posts)
+    action = lambda post: None
     if account.service == 'twitter':
-        eligible = list((post for post in posts if
-            (not account.policy_keep_favourites or not post.favourite)
-            and (not account.policy_keep_media or not post.has_media)
-            ))
-        if eligible:
-            if account.policy_delete_every == timedelta(0):
-                print("deleting all {} eligible posts for {}".format(len(eligible), account))
-                for post in eligible:
-                    account.touch_delete()
-                    lib.twitter.delete(post)
-            else:
-                post = random.choice(eligible)
-                print("deleting {}".format(post))
+        action = lib.twitter.delete
+    elif account.service == 'mastodon':
+        action = lib.mastodon.delete
+
+    posts = refresh_posts(posts)
+    eligible = list((post for post in posts if
+        (not account.policy_keep_favourites or not post.favourite)
+        and (not account.policy_keep_media or not post.has_media)
+        ))
+    if eligible:
+        if account.policy_delete_every == timedelta(0):
+            print("deleting all {} eligible posts for {}".format(len(eligible), account))
+            for post in eligible:
                 account.touch_delete()
-                lib.twitter.delete(post)
+                action(post)
+        else:
+            post = random.choice(eligible)
+            print("deleting {}".format(post))
+            account.touch_delete()
+            action(post)
 
     db.session.commit()
 
@@ -170,6 +177,8 @@ def refresh_posts(posts):
 
     if posts[0].service == 'twitter':
         return lib.twitter.refresh_posts(posts)
+    elif posts[0].service == 'mastodon':
+        return lib.mastodon.refresh_posts(posts)
 
 @app.task(autoretry_for=(TwitterError, URLError))
 def refresh_account(account_id):