diff --git a/libforget/misskey.py b/libforget/misskey.py new file mode 100644 index 0000000..d5d4c0a --- /dev/null +++ b/libforget/misskey.py @@ -0,0 +1,210 @@ +from app import db, sentry +from model import MisskeyApp, MisskeyInstance, Account, OAuthToken +from requests import get, post +from uuid import uuid4 +from hashlib import sha256 +from libforget.exceptions import TemporaryError, PermanentError + +def get_or_create_app(instance_url, callback, website): + instance_url = instance_url + app = MisskeyApp.query.get(instance_url) + + if not app: + # check if the instance uses https while getting instance infos + try: + r = post('https://{}/api/meta'.format(instance_url)) + r.raise_for_status() + proto = 'https' + except Exception: + r = post('http://{}/api/meta'.format(instance_url)) + r.raise_for_status() + proto = 'http' + + # check if miauth is available or we have to use legacy auth + miauth = 'miauth' in r.json()['features'] + + app = MisskeyApp() + app.instance = instance_url + app.protocol = proto + app.miauth = miauth + + if miauth: + # apps do not have to be registered for miauth + app.client_secret = None + else: + # register the app + r = post('{}://{}/api/app/create', json = { + 'name': 'forget', + 'description': website, + 'permission': ['read:favorites', 'write:notes'], + 'callbackUrl': callback + }) + r.raise_for_status() + app.client_secret = r.json()['secret'] + + return app + +def login_url(app, callback): + if app.miauth: + return "{}://{}/miauth/{}?name=forget&callback={}&permission=read:favorites,write:notes".format(app.protocol, app.instance, uuid4(), callback) + else: + # will use the callback we gave the server in `get_or_create_app` + r = post('{}://{}/api/auth/session/generate'.format(app.protocol, app.instance), json = { + 'appSecret': app.client_secret + }) + r.raise_for_status() + # we already get the retrieval token here, but we get it again later so + # we do not have to store it + return r.json()['url'] + +def receive_token(token, app): + if app.miauth: + r = get('{}://{}/api/miauth/{}/check'.format(app.protocol, app.instance, token)) + r.raise_for_status() + + token = r.json()['token'] + + acc = account_from_user(r.json()['user']) + acc = db.session.merge(acc) + token = OAuthToken(token = r.json()['token']) + token = db.session.merge(token) + token.account = acc + else: + r = post('{}://{}/api/auth/session/userkey'.format(app.protocol, app.instance), json = { + 'appSecret': app.client_secret, + 'token': token + }) + r.raise_for_status() + + token = sha256(r.json()['accessToken'].encode('utf-8') + app.client_secret.encode('utf-8')).hexdigest() + + acc = account_from_user(r.json()['user']) + acc = db.session.merge(acc) + token = OAuthToken(token = token) + token = db.session.merge(token) + token.account = acc + + return token + +def check_auth(account, app): + if app.miauth: + r = get('{}://{}/api/miauth/{}/check'.format(app.protocol, app.instance, account.token)) + + if r.status_code != 200: + raise TemporaryError("{} {}".format(r.status_code, r.body)) + + if not r.json()['ok']: + if sentry: + sentry.captureMessage( + 'Misskey auth revoked or incorrect', + extra=locals()) + db.session.delete(token) + db.session.commit() + raise PermanentError("Misskey auth revoked") + else: + # there is no such check for legacy auth, instead we check if we can + # get the user info + r = post('{}://{}/api/i'.format(app.protocol, app.instance), json = {'i': account.token}) + + if r.status_code != 200: + raise TemporaryError("{} {}".format(r.status_code, r.body)) + + if r.json()['isSuspended']: + # this is technically a temporary error, but like for twitter + # its handled as permanent to not make useless API calls + raise PermanentError("Misskey account suspended") + +def account_from_user(user): + return Account( + misskey_instance=user['host'], + misskey_id=user['id'], + screen_name='{}@{}'.format(user['username'], user['host']), + display_name=obj['name'], + avatar_url=obj['avatarUrl'], + reported_post_count=obj['notesCount'], + ) + +def post_from_api_object(obj): + return Post( + misskey_instance=user['host'], + misskey_id=user['id'], + favourite=obj['myReaction'] is not None, + has_media=('fileIds' in obj + and bool(obj['fileIds'])), + created_at=obj['createdAt'], + author_id=account_from_user(obj['user']).id, + direct=obj['visibility'] == 'specified', + is_reblog=obj['renoteId'] is not None, + ) + +def fetch_posts(acc, max_id, since_id): + app = MisskeyApp.query.get(acc.misskey_instance) + check_auth(acc, app) + if not verify_credentials(acc, app): + raise PermanentError() + try: + kwargs = dict(limit=40) + if max_id: + kwargs['untilId'] = max_id + if since_id: + kwargs['sinceId'] = since_id + + notes = post('{}://{}/api/users/notes'.format(app.protocol, app.misskey_instance), json=kwargs) + notes.raise_for_status() + + return [post_from_api_object(status) for note in notes.json()] + + except Exception as e: + raise TemporaryError(e) + + +def refresh_posts(posts): + acc = posts[0].author + app = MisskeyApp.query.get(acc.misskey_instance) + check_auth(acc, app) + + new_posts = list() + with db.session.no_autoflush: + for post in posts: + print('Refreshing {}'.format(post)) + r = post('{}://{}/api/notes/show'.format(app.protocol, app.misskey_instance), json={ + 'noteId': post.misskey_id + }) + if r.status_code != 200: + try: + if r.json()['error']['code'] == 'NO_SUCH_NOTE': + db.session.delete(post) + continue + except Exception as e: + raise TemporaryError(e) + raise TemporaryError('{} {}'.format(r.status_code, r.body)) + + new_post = db.session.merge(post_from_api_object(r.json())) + new_post.touch() + new_posts.append(new_post) + return new_posts + +def delete(post): + app = MisskeyApp.query.get(post.misskey_instance) + if not app: + # how? if this happens, it doesnt make sense to repeat it, + # so use a permanent error + raise PermanentError("instance not registered for delete") + + r = post('{}://{}/api/notes/delete'.format(app.protocol, app.misskey_instance), json = { + 'noteId': post.misskey_id + }) + + if r.status_code != 204: + raise TemporaryError("{} {}".format(r.status_code, r.body)) + + db.session.delete(post) + +def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()): + return tuple((ins.instance for ins in ( + MisskeyInstance.query + .filter(MisskeyInstance.popularity > min_popularity) + .filter(~MisskeyInstance.instance.in_(blocklist)) + .order_by(db.desc(MisskeyInstance.popularity), + MisskeyInstance.instance) + .limit(limit).all()))) diff --git a/model.py b/model.py index 01cfd29..5951d6a 100644 --- a/model.py +++ b/model.py @@ -67,6 +67,34 @@ class RemoteIDMixin(object): @mastodon_id.setter def mastodon_id(self, id_): self.id = "mastodon:{}@{}".format(id_, self.mastodon_instance) + + @property + def misskey_instance(self): + if not self.id: + return None + if self.service != "misskey": + raise Exception( + "tried to get misskey instance for a {} {}" + .format(self.service, type(self))) + return self.id.split(":", 1)[1].split('@')[1] + + @misskey_instance.setter + def misskey_instance(self, instance): + self.id = "misskey:{}@{}".format(self.misskey_id, instance) + + @property + def misskey_id(self): + if not self.id: + return None + if self.service != "misskey": + raise Exception( + "tried to get misskey id for a {} {}" + .format(self.service, type(self))) + return self.id.split(":", 1)[1].split('@')[0] + + @misskey_id.setter + def misskey_id(self, id_): + self.id = "misskey:{}@{}".format(id_, self.misskey_instance) @property def remote_id(self): @@ -74,6 +102,8 @@ class RemoteIDMixin(object): return self.twitter_id elif self.service == 'mastodon': return self.mastodon_id + elif self.service == 'misskey': + return self.misskey_id ThreeWayPolicyEnum = db.Enum('keeponly', 'deleteonly', 'none', @@ -364,3 +394,26 @@ class MastodonInstance(db.Model): def bump(self, value=1): self.popularity = (self.popularity or 10) + value + +class MisskeyApp(db.Model, TimestampMixin): + __tablename__ = 'misskey_apps' + + instance = db.Column(db.String, primary_key=True) + protocol = db.Column(db.String, nullable=False) + miauth = db.Column(db.Boolean, nullable=False) + # only legacy auth uses client_secret + client_secret = db.Column(db.String, nullable=True) + +class MisskeyInstance(db.Model): + """ + this is for the autocomplete in the misskey login form + it isn't coupled with anything else so that we can seed it with + some popular instances ahead of time + """ + __tablename__ = 'misskey_instances' + + instance = db.Column(db.String, primary_key=True) + popularity = db.Column(db.Float, server_default='10', nullable=False) + + def bump(self, value=1): + self.popularity = (self.popularity or 10) + value diff --git a/routes/__init__.py b/routes/__init__.py index bfa2976..1e9fd5d 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -3,6 +3,7 @@ from flask import render_template, url_for, redirect, request, g,\ from datetime import datetime, timedelta, timezone import libforget.twitter import libforget.mastodon +import libforget.misskey from libforget.auth import require_auth, csrf,\ get_viewer from model import Session, TwitterArchive, MastodonApp @@ -240,6 +241,72 @@ def mastodon_login_step2(instance_url): return resp +@app.route('/login/misskey', methods=('GET', 'POST')) +def misskey_login(instance=None): + instance_url = (request.args.get('instance_url', None) + or request.form.get('instance_url', None)) + + if not instance_url: + instances = libforget.misskey.suggested_instances( + limit = 30, + min_popularity = 1 + ) + return render_template( + 'mastodon_login.html', instances=instances, + address_error=request.method == 'POST', + generic_error='error' in request.args + ) + + instance_url = instance_url.lower() + # strip protocol + instance_url = re.sub('^https?://', '', instance_url, + count=1, flags=re.IGNORECASE) + # strip username + instance_url = instance_url.split("@")[-1] + # strip trailing path + instance_url = instance_url.split('/')[0] + + callback = url_for('misskey_callback', + instance_url=instance_url, _external=True) + + try: + app = libforget.misskey.get_or_create_app( + instance_url, + callback_legacy, + url_for('index', _external=True)) + db.session.merge(app) + + db.session.commit() + + return redirect(libforget.misskey.login_url(app, callback)) + + except Exception: + if sentry: + sentry.captureException() + return redirect(url_for('misskey_login', error=True)) + + +@app.route('/login/misskey/callback/') +def misskey_callback(instance_url): + # legacy auth and miauth use different parameter names + token = request.args.get('token', None) or request.args.get('session', None) + app = MisskeyApp.query.get(instance_url) + if not token or not app: + return redirect(url_for('misskey_login', error=True)) + + token = libforget.misskey.receive_token(token, app) + account = token.account + + session = login(account.id) + + db.session.commit() + + g.viewer = session + + resp = redirect(url_for('index', _anchor='bump_instance')) + return resp + + @app.route('/sentry/setup.js') def sentry_setup(): client_dsn = app.config.get('SENTRY_DSN').split('@') diff --git a/tasks.py b/tasks.py index 07c7bf7..d26a757 100644 --- a/tasks.py +++ b/tasks.py @@ -5,6 +5,7 @@ from model import Session, Account, TwitterArchive, Post, OAuthToken,\ MastodonInstance import libforget.twitter import libforget.mastodon +import libforget.misskey from datetime import timedelta, datetime, timezone from time import time from zipfile import ZipFile @@ -151,6 +152,8 @@ def fetch_acc(id_): fetch_posts = libforget.twitter.fetch_posts elif (account.service == 'mastodon'): fetch_posts = libforget.mastodon.fetch_posts + elif (account.service == 'misskey'): + fetch_posts = libforget.misskey.fetch_posts posts = fetch_posts(account, max_id, since_id) if posts is None: @@ -291,6 +294,10 @@ def delete_from_account(account_id): if refreshed and is_eligible(refreshed[0]): to_delete = refreshed[0] break + elif account.service == 'misskey': + action = libforget.misskey.delete + posts = refresh_posts(posts) + to_delete = next(filter(is_eligible, posts), None) if to_delete: print("Deleting {}".format(to_delete)) @@ -317,6 +324,8 @@ def refresh_posts(posts): return libforget.twitter.refresh_posts(posts) elif posts[0].service == 'mastodon': return libforget.mastodon.refresh_posts(posts) + elif posts[0].service == 'misskey': + return libforget.misskey.refresh_posts(posts) @app.task() @@ -474,6 +483,33 @@ def update_mastodon_instances_popularity(): }) db.session.commit() +@app.task +def update_misskey_instances_popularity(): + # bump score for each active account + for acct in (Account.query.options(db.joinedload(Account.sessions)) + .filter(~Account.dormant).filter( + Account.id.like('misskey:%'))): + instance = MisskeyInstance.query.get(acct.misskey_instance) + if not instance: + instance = MisskeyInstance( + instance=acct.Misskey_instance, popularity=10) + db.session.add(instance) + amount = 0.01 + if acct.policy_enabled: + amount = 0.5 + for _ in acct.sessions: + amount += 0.1 + instance.bump(amount / max(1, instance.popularity)) + + # normalise scores so the top is 20 + top_pop = (db.session.query(db.func.max(MisskeyInstance.popularity)) + .scalar()) + MisskeyInstance.query.update({ + MisskeyInstance.popularity: + MisskeyInstance.popularity * 20 / top_pop + }) + db.session.commit() + app.add_periodic_task(40, queue_fetch_for_most_stale_accounts) app.add_periodic_task(9, queue_deletes) @@ -481,6 +517,7 @@ app.add_periodic_task(6, refresh_account_with_oldest_post) app.add_periodic_task(50, refresh_account_with_longest_time_since_refresh) app.add_periodic_task(300, periodic_cleanup) app.add_periodic_task(300, update_mastodon_instances_popularity) +app.add_periodic_task(300, update_misskey_instances_popularity) if __name__ == '__main__': app.worker_main()