WIP: implement misskey API

This commit is contained in:
Johann150 2021-11-09 10:07:56 +01:00
parent ce35aa939b
commit ed1c42d30d
No known key found for this signature in database
GPG Key ID: 9EE6577A2A06F8F1
4 changed files with 367 additions and 0 deletions

210
libforget/misskey.py Normal file
View File

@ -0,0 +1,210 @@
from app import db, sentry
from model import MisskeyApp, MisskeyInstance, Account, OAuthToken
from requests import get, post
from uuid import uuid4
from hashlib import sha256
from libforget.exceptions import TemporaryError, PermanentError
def get_or_create_app(instance_url, callback, website):
instance_url = instance_url
app = MisskeyApp.query.get(instance_url)
if not app:
# check if the instance uses https while getting instance infos
try:
r = post('https://{}/api/meta'.format(instance_url))
r.raise_for_status()
proto = 'https'
except Exception:
r = post('http://{}/api/meta'.format(instance_url))
r.raise_for_status()
proto = 'http'
# check if miauth is available or we have to use legacy auth
miauth = 'miauth' in r.json()['features']
app = MisskeyApp()
app.instance = instance_url
app.protocol = proto
app.miauth = miauth
if miauth:
# apps do not have to be registered for miauth
app.client_secret = None
else:
# register the app
r = post('{}://{}/api/app/create', json = {
'name': 'forget',
'description': website,
'permission': ['read:favorites', 'write:notes'],
'callbackUrl': callback
})
r.raise_for_status()
app.client_secret = r.json()['secret']
return app
def login_url(app, callback):
if app.miauth:
return "{}://{}/miauth/{}?name=forget&callback={}&permission=read:favorites,write:notes".format(app.protocol, app.instance, uuid4(), callback)
else:
# will use the callback we gave the server in `get_or_create_app`
r = post('{}://{}/api/auth/session/generate'.format(app.protocol, app.instance), json = {
'appSecret': app.client_secret
})
r.raise_for_status()
# we already get the retrieval token here, but we get it again later so
# we do not have to store it
return r.json()['url']
def receive_token(token, app):
if app.miauth:
r = get('{}://{}/api/miauth/{}/check'.format(app.protocol, app.instance, token))
r.raise_for_status()
token = r.json()['token']
acc = account_from_user(r.json()['user'])
acc = db.session.merge(acc)
token = OAuthToken(token = r.json()['token'])
token = db.session.merge(token)
token.account = acc
else:
r = post('{}://{}/api/auth/session/userkey'.format(app.protocol, app.instance), json = {
'appSecret': app.client_secret,
'token': token
})
r.raise_for_status()
token = sha256(r.json()['accessToken'].encode('utf-8') + app.client_secret.encode('utf-8')).hexdigest()
acc = account_from_user(r.json()['user'])
acc = db.session.merge(acc)
token = OAuthToken(token = token)
token = db.session.merge(token)
token.account = acc
return token
def check_auth(account, app):
if app.miauth:
r = get('{}://{}/api/miauth/{}/check'.format(app.protocol, app.instance, account.token))
if r.status_code != 200:
raise TemporaryError("{} {}".format(r.status_code, r.body))
if not r.json()['ok']:
if sentry:
sentry.captureMessage(
'Misskey auth revoked or incorrect',
extra=locals())
db.session.delete(token)
db.session.commit()
raise PermanentError("Misskey auth revoked")
else:
# there is no such check for legacy auth, instead we check if we can
# get the user info
r = post('{}://{}/api/i'.format(app.protocol, app.instance), json = {'i': account.token})
if r.status_code != 200:
raise TemporaryError("{} {}".format(r.status_code, r.body))
if r.json()['isSuspended']:
# this is technically a temporary error, but like for twitter
# its handled as permanent to not make useless API calls
raise PermanentError("Misskey account suspended")
def account_from_user(user):
return Account(
misskey_instance=user['host'],
misskey_id=user['id'],
screen_name='{}@{}'.format(user['username'], user['host']),
display_name=obj['name'],
avatar_url=obj['avatarUrl'],
reported_post_count=obj['notesCount'],
)
def post_from_api_object(obj):
return Post(
misskey_instance=user['host'],
misskey_id=user['id'],
favourite=obj['myReaction'] is not None,
has_media=('fileIds' in obj
and bool(obj['fileIds'])),
created_at=obj['createdAt'],
author_id=account_from_user(obj['user']).id,
direct=obj['visibility'] == 'specified',
is_reblog=obj['renoteId'] is not None,
)
def fetch_posts(acc, max_id, since_id):
app = MisskeyApp.query.get(acc.misskey_instance)
check_auth(acc, app)
if not verify_credentials(acc, app):
raise PermanentError()
try:
kwargs = dict(limit=40)
if max_id:
kwargs['untilId'] = max_id
if since_id:
kwargs['sinceId'] = since_id
notes = post('{}://{}/api/users/notes'.format(app.protocol, app.misskey_instance), json=kwargs)
notes.raise_for_status()
return [post_from_api_object(status) for note in notes.json()]
except Exception as e:
raise TemporaryError(e)
def refresh_posts(posts):
acc = posts[0].author
app = MisskeyApp.query.get(acc.misskey_instance)
check_auth(acc, app)
new_posts = list()
with db.session.no_autoflush:
for post in posts:
print('Refreshing {}'.format(post))
r = post('{}://{}/api/notes/show'.format(app.protocol, app.misskey_instance), json={
'noteId': post.misskey_id
})
if r.status_code != 200:
try:
if r.json()['error']['code'] == 'NO_SUCH_NOTE':
db.session.delete(post)
continue
except Exception as e:
raise TemporaryError(e)
raise TemporaryError('{} {}'.format(r.status_code, r.body))
new_post = db.session.merge(post_from_api_object(r.json()))
new_post.touch()
new_posts.append(new_post)
return new_posts
def delete(post):
app = MisskeyApp.query.get(post.misskey_instance)
if not app:
# how? if this happens, it doesnt make sense to repeat it,
# so use a permanent error
raise PermanentError("instance not registered for delete")
r = post('{}://{}/api/notes/delete'.format(app.protocol, app.misskey_instance), json = {
'noteId': post.misskey_id
})
if r.status_code != 204:
raise TemporaryError("{} {}".format(r.status_code, r.body))
db.session.delete(post)
def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
return tuple((ins.instance for ins in (
MisskeyInstance.query
.filter(MisskeyInstance.popularity > min_popularity)
.filter(~MisskeyInstance.instance.in_(blocklist))
.order_by(db.desc(MisskeyInstance.popularity),
MisskeyInstance.instance)
.limit(limit).all())))

View File

@ -68,12 +68,42 @@ class RemoteIDMixin(object):
def mastodon_id(self, id_):
self.id = "mastodon:{}@{}".format(id_, self.mastodon_instance)
@property
def misskey_instance(self):
if not self.id:
return None
if self.service != "misskey":
raise Exception(
"tried to get misskey instance for a {} {}"
.format(self.service, type(self)))
return self.id.split(":", 1)[1].split('@')[1]
@misskey_instance.setter
def misskey_instance(self, instance):
self.id = "misskey:{}@{}".format(self.misskey_id, instance)
@property
def misskey_id(self):
if not self.id:
return None
if self.service != "misskey":
raise Exception(
"tried to get misskey id for a {} {}"
.format(self.service, type(self)))
return self.id.split(":", 1)[1].split('@')[0]
@misskey_id.setter
def misskey_id(self, id_):
self.id = "misskey:{}@{}".format(id_, self.misskey_instance)
@property
def remote_id(self):
if self.service == 'twitter':
return self.twitter_id
elif self.service == 'mastodon':
return self.mastodon_id
elif self.service == 'misskey':
return self.misskey_id
ThreeWayPolicyEnum = db.Enum('keeponly', 'deleteonly', 'none',
@ -364,3 +394,26 @@ class MastodonInstance(db.Model):
def bump(self, value=1):
self.popularity = (self.popularity or 10) + value
class MisskeyApp(db.Model, TimestampMixin):
__tablename__ = 'misskey_apps'
instance = db.Column(db.String, primary_key=True)
protocol = db.Column(db.String, nullable=False)
miauth = db.Column(db.Boolean, nullable=False)
# only legacy auth uses client_secret
client_secret = db.Column(db.String, nullable=True)
class MisskeyInstance(db.Model):
"""
this is for the autocomplete in the misskey login form
it isn't coupled with anything else so that we can seed it with
some popular instances ahead of time
"""
__tablename__ = 'misskey_instances'
instance = db.Column(db.String, primary_key=True)
popularity = db.Column(db.Float, server_default='10', nullable=False)
def bump(self, value=1):
self.popularity = (self.popularity or 10) + value

View File

@ -3,6 +3,7 @@ from flask import render_template, url_for, redirect, request, g,\
from datetime import datetime, timedelta, timezone
import libforget.twitter
import libforget.mastodon
import libforget.misskey
from libforget.auth import require_auth, csrf,\
get_viewer
from model import Session, TwitterArchive, MastodonApp
@ -240,6 +241,72 @@ def mastodon_login_step2(instance_url):
return resp
@app.route('/login/misskey', methods=('GET', 'POST'))
def misskey_login(instance=None):
instance_url = (request.args.get('instance_url', None)
or request.form.get('instance_url', None))
if not instance_url:
instances = libforget.misskey.suggested_instances(
limit = 30,
min_popularity = 1
)
return render_template(
'mastodon_login.html', instances=instances,
address_error=request.method == 'POST',
generic_error='error' in request.args
)
instance_url = instance_url.lower()
# strip protocol
instance_url = re.sub('^https?://', '', instance_url,
count=1, flags=re.IGNORECASE)
# strip username
instance_url = instance_url.split("@")[-1]
# strip trailing path
instance_url = instance_url.split('/')[0]
callback = url_for('misskey_callback',
instance_url=instance_url, _external=True)
try:
app = libforget.misskey.get_or_create_app(
instance_url,
callback_legacy,
url_for('index', _external=True))
db.session.merge(app)
db.session.commit()
return redirect(libforget.misskey.login_url(app, callback))
except Exception:
if sentry:
sentry.captureException()
return redirect(url_for('misskey_login', error=True))
@app.route('/login/misskey/callback/<instance_url>')
def misskey_callback(instance_url):
# legacy auth and miauth use different parameter names
token = request.args.get('token', None) or request.args.get('session', None)
app = MisskeyApp.query.get(instance_url)
if not token or not app:
return redirect(url_for('misskey_login', error=True))
token = libforget.misskey.receive_token(token, app)
account = token.account
session = login(account.id)
db.session.commit()
g.viewer = session
resp = redirect(url_for('index', _anchor='bump_instance'))
return resp
@app.route('/sentry/setup.js')
def sentry_setup():
client_dsn = app.config.get('SENTRY_DSN').split('@')

View File

@ -5,6 +5,7 @@ from model import Session, Account, TwitterArchive, Post, OAuthToken,\
MastodonInstance
import libforget.twitter
import libforget.mastodon
import libforget.misskey
from datetime import timedelta, datetime, timezone
from time import time
from zipfile import ZipFile
@ -151,6 +152,8 @@ def fetch_acc(id_):
fetch_posts = libforget.twitter.fetch_posts
elif (account.service == 'mastodon'):
fetch_posts = libforget.mastodon.fetch_posts
elif (account.service == 'misskey'):
fetch_posts = libforget.misskey.fetch_posts
posts = fetch_posts(account, max_id, since_id)
if posts is None:
@ -291,6 +294,10 @@ def delete_from_account(account_id):
if refreshed and is_eligible(refreshed[0]):
to_delete = refreshed[0]
break
elif account.service == 'misskey':
action = libforget.misskey.delete
posts = refresh_posts(posts)
to_delete = next(filter(is_eligible, posts), None)
if to_delete:
print("Deleting {}".format(to_delete))
@ -317,6 +324,8 @@ def refresh_posts(posts):
return libforget.twitter.refresh_posts(posts)
elif posts[0].service == 'mastodon':
return libforget.mastodon.refresh_posts(posts)
elif posts[0].service == 'misskey':
return libforget.misskey.refresh_posts(posts)
@app.task()
@ -474,6 +483,33 @@ def update_mastodon_instances_popularity():
})
db.session.commit()
@app.task
def update_misskey_instances_popularity():
# bump score for each active account
for acct in (Account.query.options(db.joinedload(Account.sessions))
.filter(~Account.dormant).filter(
Account.id.like('misskey:%'))):
instance = MisskeyInstance.query.get(acct.misskey_instance)
if not instance:
instance = MisskeyInstance(
instance=acct.Misskey_instance, popularity=10)
db.session.add(instance)
amount = 0.01
if acct.policy_enabled:
amount = 0.5
for _ in acct.sessions:
amount += 0.1
instance.bump(amount / max(1, instance.popularity))
# normalise scores so the top is 20
top_pop = (db.session.query(db.func.max(MisskeyInstance.popularity))
.scalar())
MisskeyInstance.query.update({
MisskeyInstance.popularity:
MisskeyInstance.popularity * 20 / top_pop
})
db.session.commit()
app.add_periodic_task(40, queue_fetch_for_most_stale_accounts)
app.add_periodic_task(9, queue_deletes)
@ -481,6 +517,7 @@ app.add_periodic_task(6, refresh_account_with_oldest_post)
app.add_periodic_task(50, refresh_account_with_longest_time_since_refresh)
app.add_periodic_task(300, periodic_cleanup)
app.add_periodic_task(300, update_mastodon_instances_popularity)
app.add_periodic_task(300, update_misskey_instances_popularity)
if __name__ == '__main__':
app.worker_main()