WIP: implement misskey API

libforget/misskey.py

@@ -0,0 +1,210 @@
+from app import db, sentry
+from model import MisskeyApp, MisskeyInstance, Account, OAuthToken
+from requests import get, post
+from uuid import uuid4
+from hashlib import sha256
+from libforget.exceptions import TemporaryError, PermanentError
+
+def get_or_create_app(instance_url, callback, website):
+    instance_url = instance_url
+    app = MisskeyApp.query.get(instance_url)
+    
+    if not app:
+        # check if the instance uses https while getting instance infos
+        try:
+            r = post('https://{}/api/meta'.format(instance_url))
+            r.raise_for_status()
+            proto = 'https'
+        except Exception:
+            r = post('http://{}/api/meta'.format(instance_url))
+            r.raise_for_status()
+            proto = 'http'
+        
+        # check if miauth is available or we have to use legacy auth
+        miauth = 'miauth' in r.json()['features']
+        
+        app = MisskeyApp()
+        app.instance = instance_url
+        app.protocol = proto
+        app.miauth = miauth
+        
+        if miauth:
+            # apps do not have to be registered for miauth
+            app.client_secret = None
+        else:
+            # register the app
+            r = post('{}://{}/api/app/create', json = {
+                'name': 'forget',
+                'description': website,
+                'permission': ['read:favorites', 'write:notes'],
+                'callbackUrl': callback
+            })
+            r.raise_for_status()
+            app.client_secret = r.json()['secret']
+
+    return app
+
+def login_url(app, callback):
+    if app.miauth:
+        return "{}://{}/miauth/{}?name=forget&callback={}&permission=read:favorites,write:notes".format(app.protocol, app.instance, uuid4(), callback)
+    else:
+        # will use the callback we gave the server in `get_or_create_app`
+        r = post('{}://{}/api/auth/session/generate'.format(app.protocol, app.instance), json = {
+            'appSecret': app.client_secret
+        })
+        r.raise_for_status()
+        # we already get the retrieval token here, but we get it again later so
+        # we do not have to store it
+        return r.json()['url']
+
+def receive_token(token, app):
+    if app.miauth:
+        r = get('{}://{}/api/miauth/{}/check'.format(app.protocol, app.instance, token))
+        r.raise_for_status()
+        
+        token = r.json()['token']
+
+        acc = account_from_user(r.json()['user'])
+        acc = db.session.merge(acc)
+        token = OAuthToken(token = r.json()['token'])
+        token = db.session.merge(token)
+        token.account = acc
+    else:
+        r = post('{}://{}/api/auth/session/userkey'.format(app.protocol, app.instance), json = {
+            'appSecret': app.client_secret,
+            'token': token
+        })
+        r.raise_for_status()
+        
+        token = sha256(r.json()['accessToken'].encode('utf-8') + app.client_secret.encode('utf-8')).hexdigest()
+        
+    acc = account_from_user(r.json()['user'])
+    acc = db.session.merge(acc)
+    token = OAuthToken(token = token)
+    token = db.session.merge(token)
+    token.account = acc
+
+    return token
+
+def check_auth(account, app):
+    if app.miauth:
+        r = get('{}://{}/api/miauth/{}/check'.format(app.protocol, app.instance, account.token))
+
+        if r.status_code != 200:
+            raise TemporaryError("{} {}".format(r.status_code, r.body))
+
+        if not r.json()['ok']:
+            if sentry:
+                sentry.captureMessage(
+                    'Misskey auth revoked or incorrect',
+                    extra=locals())
+            db.session.delete(token)
+            db.session.commit()
+            raise PermanentError("Misskey auth revoked")
+    else: 
+        # there is no such check for legacy auth, instead we check if we can
+        # get the user info
+        r = post('{}://{}/api/i'.format(app.protocol, app.instance), json = {'i': account.token})
+
+        if r.status_code != 200:
+            raise TemporaryError("{} {}".format(r.status_code, r.body))
+
+        if r.json()['isSuspended']:
+            # this is technically a temporary error, but like for twitter
+            # its handled as permanent to not make useless API calls
+            raise PermanentError("Misskey account suspended")
+
+def account_from_user(user):
+    return Account(
+            misskey_instance=user['host'],
+            misskey_id=user['id'],
+            screen_name='{}@{}'.format(user['username'], user['host']),
+            display_name=obj['name'],
+            avatar_url=obj['avatarUrl'],
+            reported_post_count=obj['notesCount'],
+        )
+
+def post_from_api_object(obj):
+    return Post(
+            misskey_instance=user['host'],
+            misskey_id=user['id'],
+            favourite=obj['myReaction'] is not None,
+            has_media=('fileIds' in obj
+                       and bool(obj['fileIds'])),
+            created_at=obj['createdAt'],
+            author_id=account_from_user(obj['user']).id,
+            direct=obj['visibility'] == 'specified',
+            is_reblog=obj['renoteId'] is not None,
+        )
+
+def fetch_posts(acc, max_id, since_id):
+    app = MisskeyApp.query.get(acc.misskey_instance)
+    check_auth(acc, app)
+    if not verify_credentials(acc, app):
+        raise PermanentError()
+    try:
+        kwargs = dict(limit=40)
+        if max_id:
+            kwargs['untilId'] = max_id
+        if since_id:
+            kwargs['sinceId'] = since_id
+
+        notes = post('{}://{}/api/users/notes'.format(app.protocol, app.misskey_instance), json=kwargs)
+        notes.raise_for_status()
+
+        return [post_from_api_object(status) for note in notes.json()]
+
+    except Exception as e:
+        raise TemporaryError(e)
+
+
+def refresh_posts(posts):
+    acc = posts[0].author
+    app = MisskeyApp.query.get(acc.misskey_instance)
+    check_auth(acc, app)
+
+    new_posts = list()
+    with db.session.no_autoflush:
+        for post in posts:
+            print('Refreshing {}'.format(post))
+            r = post('{}://{}/api/notes/show'.format(app.protocol, app.misskey_instance), json={
+                'noteId': post.misskey_id
+            })
+            if r.status_code != 200:
+                try:
+                    if r.json()['error']['code'] == 'NO_SUCH_NOTE':
+                        db.session.delete(post)
+                        continue
+                except Exception as e:
+                    raise TemporaryError(e)
+                raise TemporaryError('{} {}'.format(r.status_code, r.body))
+
+            new_post = db.session.merge(post_from_api_object(r.json()))
+            new_post.touch()
+            new_posts.append(new_post)
+    return new_posts
+
+def delete(post):
+    app = MisskeyApp.query.get(post.misskey_instance)
+    if not app:
+        # how? if this happens, it doesnt make sense to repeat it,
+        # so use a permanent error
+        raise PermanentError("instance not registered for delete")
+
+    r = post('{}://{}/api/notes/delete'.format(app.protocol, app.misskey_instance), json = {
+        'noteId': post.misskey_id
+    })
+    
+    if r.status_code != 204:
+        raise TemporaryError("{} {}".format(r.status_code, r.body))
+    
+    db.session.delete(post)
+
+def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
+    return tuple((ins.instance for ins in (
+            MisskeyInstance.query
+            .filter(MisskeyInstance.popularity > min_popularity)
+            .filter(~MisskeyInstance.instance.in_(blocklist))
+            .order_by(db.desc(MisskeyInstance.popularity),
+                      MisskeyInstance.instance)
+            .limit(limit).all())))

model.py

@@ -67,6 +67,34 @@ class RemoteIDMixin(object):
     @mastodon_id.setter
     def mastodon_id(self, id_):
         self.id = "mastodon:{}@{}".format(id_, self.mastodon_instance)
+        
+    @property
+    def misskey_instance(self):
+        if not self.id:
+            return None
+        if self.service != "misskey":
+            raise Exception(
+                "tried to get misskey instance for a {} {}"
+                .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[1]
+        
+    @misskey_instance.setter
+    def misskey_instance(self, instance):
+        self.id = "misskey:{}@{}".format(self.misskey_id, instance)
+        
+    @property
+    def misskey_id(self):
+        if not self.id:
+            return None
+        if self.service != "misskey":
+            raise Exception(
+                    "tried to get misskey id for a {} {}"
+                    .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[0]
+
+    @misskey_id.setter
+    def misskey_id(self, id_):
+        self.id = "misskey:{}@{}".format(id_, self.misskey_instance)
 
     @property
     def remote_id(self):
@@ -74,6 +102,8 @@ class RemoteIDMixin(object):
             return self.twitter_id
         elif self.service == 'mastodon':
             return self.mastodon_id
+        elif self.service == 'misskey':
+            return self.misskey_id
 
 
 ThreeWayPolicyEnum = db.Enum('keeponly', 'deleteonly', 'none',
@@ -364,3 +394,26 @@ class MastodonInstance(db.Model):
 
     def bump(self, value=1):
         self.popularity = (self.popularity or 10) + value
+
+class MisskeyApp(db.Model, TimestampMixin):
+    __tablename__ = 'misskey_apps'
+
+    instance = db.Column(db.String, primary_key=True)
+    protocol = db.Column(db.String, nullable=False)
+    miauth = db.Column(db.Boolean, nullable=False)
+    # only legacy auth uses client_secret
+    client_secret = db.Column(db.String, nullable=True)
+
+class MisskeyInstance(db.Model):
+    """
+    this is for the autocomplete in the misskey login form
+    it isn't coupled with anything else so that we can seed it with
+    some popular instances ahead of time
+    """
+    __tablename__ = 'misskey_instances'
+
+    instance = db.Column(db.String, primary_key=True)
+    popularity = db.Column(db.Float, server_default='10', nullable=False)
+
+    def bump(self, value=1):
+        self.popularity = (self.popularity or 10) + value

routes/__init__.py

@@ -3,6 +3,7 @@ from flask import render_template, url_for, redirect, request, g,\
 from datetime import datetime, timedelta, timezone
 import libforget.twitter
 import libforget.mastodon
+import libforget.misskey
 from libforget.auth import require_auth, csrf,\
                      get_viewer
 from model import Session, TwitterArchive, MastodonApp
@@ -240,6 +241,72 @@ def mastodon_login_step2(instance_url):
     return resp
 
 
+@app.route('/login/misskey', methods=('GET', 'POST'))
+def misskey_login(instance=None):
+    instance_url = (request.args.get('instance_url', None)
+                    or request.form.get('instance_url', None))
+    
+    if not instance_url:
+        instances = libforget.misskey.suggested_instances(
+            limit = 30,
+            min_popularity = 1
+        )
+        return render_template(
+                'mastodon_login.html', instances=instances,
+                address_error=request.method == 'POST',
+                generic_error='error' in request.args
+                )
+
+    instance_url = instance_url.lower()
+    # strip protocol
+    instance_url = re.sub('^https?://', '', instance_url,
+                          count=1, flags=re.IGNORECASE)
+    # strip username
+    instance_url = instance_url.split("@")[-1]
+    # strip trailing path
+    instance_url = instance_url.split('/')[0]
+    
+    callback = url_for('misskey_callback',
+                       instance_url=instance_url, _external=True)
+
+    try:
+        app = libforget.misskey.get_or_create_app(
+                instance_url,
+                callback_legacy,
+                url_for('index', _external=True))
+        db.session.merge(app)
+
+        db.session.commit()
+
+        return redirect(libforget.misskey.login_url(app, callback))
+
+    except Exception:
+        if sentry:
+            sentry.captureException()
+        return redirect(url_for('misskey_login', error=True))
+
+
+@app.route('/login/misskey/callback/<instance_url>')
+def misskey_callback(instance_url):
+    # legacy auth and miauth use different parameter names
+    token = request.args.get('token', None) or request.args.get('session', None)
+    app = MisskeyApp.query.get(instance_url)
+    if not token or not app:
+        return redirect(url_for('misskey_login', error=True))
+
+    token = libforget.misskey.receive_token(token, app)
+    account = token.account
+
+    session = login(account.id)
+
+    db.session.commit()
+
+    g.viewer = session
+
+    resp = redirect(url_for('index', _anchor='bump_instance'))
+    return resp
+
+
 @app.route('/sentry/setup.js')
 def sentry_setup():
     client_dsn = app.config.get('SENTRY_DSN').split('@')

tasks.py

@@ -5,6 +5,7 @@ from model import Session, Account, TwitterArchive, Post, OAuthToken,\
                   MastodonInstance
 import libforget.twitter
 import libforget.mastodon
+import libforget.misskey
 from datetime import timedelta, datetime, timezone
 from time import time
 from zipfile import ZipFile
@@ -151,6 +152,8 @@ def fetch_acc(id_):
             fetch_posts = libforget.twitter.fetch_posts
         elif (account.service == 'mastodon'):
             fetch_posts = libforget.mastodon.fetch_posts
+        elif (account.service == 'misskey'):
+            fetch_posts = libforget.misskey.fetch_posts
         posts = fetch_posts(account, max_id, since_id)
 
         if posts is None:
@@ -291,6 +294,10 @@ def delete_from_account(account_id):
                 if refreshed and is_eligible(refreshed[0]):
                     to_delete = refreshed[0]
                     break
+        elif account.service == 'misskey':
+            action = libforget.misskey.delete
+            posts = refresh_posts(posts)
+            to_delete = next(filter(is_eligible, posts), None)
 
         if to_delete:
             print("Deleting {}".format(to_delete))
@@ -317,6 +324,8 @@ def refresh_posts(posts):
         return libforget.twitter.refresh_posts(posts)
     elif posts[0].service == 'mastodon':
         return libforget.mastodon.refresh_posts(posts)
+    elif posts[0].service == 'misskey':
+        return libforget.misskey.refresh_posts(posts)
 
 
 @app.task()
@@ -474,6 +483,33 @@ def update_mastodon_instances_popularity():
     })
     db.session.commit()
 
+@app.task
+def update_misskey_instances_popularity():
+    # bump score for each active account
+    for acct in (Account.query.options(db.joinedload(Account.sessions))
+                 .filter(~Account.dormant).filter(
+                     Account.id.like('misskey:%'))):
+        instance = MisskeyInstance.query.get(acct.misskey_instance)
+        if not instance:
+            instance = MisskeyInstance(
+                instance=acct.Misskey_instance, popularity=10)
+            db.session.add(instance)
+        amount = 0.01
+        if acct.policy_enabled:
+            amount = 0.5
+        for _ in acct.sessions:
+            amount += 0.1
+        instance.bump(amount / max(1, instance.popularity))
+
+    # normalise scores so the top is 20
+    top_pop = (db.session.query(db.func.max(MisskeyInstance.popularity))
+               .scalar())
+    MisskeyInstance.query.update({
+        MisskeyInstance.popularity:
+        MisskeyInstance.popularity * 20 / top_pop
+    })
+    db.session.commit()
+
 
 app.add_periodic_task(40, queue_fetch_for_most_stale_accounts)
 app.add_periodic_task(9, queue_deletes)
@@ -481,6 +517,7 @@ app.add_periodic_task(6, refresh_account_with_oldest_post)
 app.add_periodic_task(50, refresh_account_with_longest_time_since_refresh)
 app.add_periodic_task(300, periodic_cleanup)
 app.add_periodic_task(300, update_mastodon_instances_popularity)
+app.add_periodic_task(300, update_misskey_instances_popularity)
 
 if __name__ == '__main__':
     app.worker_main()