diff --git a/assets/settings.js b/assets/settings.js index aec323e..1fbba79 100644 --- a/assets/settings.js +++ b/assets/settings.js @@ -37,7 +37,7 @@ status_display.textContent='Still saving...'; } - function on_change(e){ + function save(){ hide_status(); clearTimeout(status_timeout); status_timeout = setTimeout(show_saving, 70); @@ -94,7 +94,7 @@ } for(input of form.elements){ - input.addEventListener('change', on_change); + input.addEventListener('change', save); } // remove submit button since we're doing live updates diff --git a/lib/mastodon.py b/lib/mastodon.py index 539fefb..70f45ed 100644 --- a/lib/mastodon.py +++ b/lib/mastodon.py @@ -1,8 +1,11 @@ import mastodon from mastodon import Mastodon -from model import MastodonApp, Account, OAuthToken +from mastodon.Mastodon import MastodonAPIError +from model import MastodonApp, Account, OAuthToken, Post from requests import head from app import db +from math import inf +import iso8601 def get_or_create_app(instance_url, callback, website): instance_url = instance_url @@ -49,17 +52,10 @@ def receive_code(code, app, callback): ) remote_acc = api.account_verify_credentials() - acc = Account( - #id = 'mastodon:{}:{}'.format(app.instance, remote_acc['username']), - mastodon_instance = app.instance, - mastodon_id = remote_acc['username'], - screen_name = remote_acc['username'], - display_name = remote_acc['display_name'], - avatar_url = remote_acc['avatar'], - reported_post_count = remote_acc['statuses_count'], - ) + acc = account_from_api_object(remote_acc, app.instance) + acc = db.session.merge(acc) token = OAuthToken(account = acc, token = access_token) - db.session.merge(acc, token) + token = db.session.merge(token) return acc @@ -71,18 +67,100 @@ def get_api_for_acc(account): client_secret = app.client_secret, api_base_url = '{}://{}'.format(app.protocol, app.instance), access_token = token.token, + ratelimit_method = 'throw', + #debug_requests = True, ) - try: - # api.verify_credentials() - # doesnt error even if the token is revoked lol sooooo - tl = api.timeline() - #if 'error' in tl and tl['error'] == 'The access token was revoked': - #ARRRRRGH - except mastodon.MastodonAPIError as e: - raise e + + # api.verify_credentials() + # doesnt error even if the token is revoked lol + # https://github.com/tootsuite/mastodon/issues/4637 + # so we have to do this: + tl = api.timeline() + if 'error' in tl: + db.session.delete(token) + continue return api + account.force_log_out() -def fetch_acc(account, cursor=None): - pass +def fetch_acc(acc, cursor=None): + api = get_api_for_acc(acc) + if not api: + print('no access, aborting') + return None + + newacc = account_from_api_object(api.account_verify_credentials(), acc.mastodon_instance) + acc = db.session.merge(newacc) + + kwargs = dict(limit = 40) + if cursor: + kwargs.update(cursor) + + if 'max_id' not in kwargs: + most_recent_post = Post.query.with_parent(acc).order_by(db.desc(Post.created_at)).first() + if most_recent_post: + kwargs['since_id'] = most_recent_post.mastodon_id + + statuses = api.account_statuses(acc.mastodon_id, **kwargs) + + if statuses: + kwargs['max_id'] = +inf + + for status in statuses: + post = post_from_api_object(status, acc.mastodon_instance) + db.session.merge(post) + kwargs['max_id'] = min(kwargs['max_id'], status['id']) + + else: + kwargs = None + + db.session.commit() + + return kwargs + +def post_from_api_object(obj, instance): + return Post( + mastodon_instance = instance, + mastodon_id = obj['id'], + body = obj['content'], + favourite = obj['favourited'], + has_media = 'media_attachments' in obj and bool(obj['media_attachments']), + created_at = iso8601.parse_date(obj['created_at']), + author_id = account_from_api_object(obj['account'], instance).id, + ) + +def account_from_api_object(obj, instance): + return Account( + mastodon_instance = instance, + mastodon_id = obj['id'], + screen_name = obj['username'], + display_name = obj['display_name'], + avatar_url = obj['avatar'], + reported_post_count = obj['statuses_count'], + ) + +def refresh_posts(posts): + acc = posts[0].author + api = get_api_for_acc(acc) + if not api: + raise Exception('no access') + + new_posts = list() + for post in posts: + try: + status = api.status(post.mastodon_id) + new_post = db.session.merge(post_from_api_object(status, post.mastodon_instance)) + new_posts.append(new_post) + except MastodonAPIError as e: + if str(e) == 'Endpoint not found.': + db.session.delete(post) + else: + raise e + + return new_posts + +def delete(post): + api = get_api_for_acc(post.author) + api.status_delete(post.mastodon_id) + db.session.delete(post) diff --git a/lib/twitter.py b/lib/twitter.py index ba43150..49a371c 100644 --- a/lib/twitter.py +++ b/lib/twitter.py @@ -101,7 +101,7 @@ def post_from_api_tweet_object(tweet, post=None): post.has_media = bool('media' in tweet['entities'] and tweet['entities']['media']) return post -def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): +def fetch_acc(account, cursor): t = get_twitter_for_acc(account) if not t: print("no twitter access, aborting") diff --git a/model.py b/model.py index 8d82e80..72fd7bd 100644 --- a/model.py +++ b/model.py @@ -95,6 +95,9 @@ class Account(TimestampMixin, RemoteIDMixin): def validate_intervals(self, key, value): if not (value == timedelta(0) or value >= timedelta(minutes=1)): value = timedelta(minutes=1) + if key == 'policy_delete_every' and datetime.now() + value < self.next_delete: + # make sure that next delete is not in the far future + self.next_delete = datetime.now() + value return value @db.validates('policy_keep_latest') diff --git a/requirements.txt b/requirements.txt index c2fe50b..de0d2e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,13 +18,13 @@ Flask-SQLAlchemy==2.2 gunicorn==19.7.1 honcho==1.0.1 idna==2.6 +iso8601==0.1.12 itsdangerous==0.24 Jinja2==2.9.6 kombu==4.1.0 limits==1.2.1 Mako==1.0.7 MarkupSafe==1.0 -Mastodon.py==1.0.8 olefile==0.44 Pillow==4.2.1 psycopg2==2.7.3 @@ -41,3 +41,4 @@ twitter==1.17.1 urllib3==1.22 vine==1.1.4 Werkzeug==0.12.2 +git+https://github.com/codl/Mastodon.py.git@forget diff --git a/routes.py b/routes.py index 588dd5a..221a978 100644 --- a/routes.py +++ b/routes.py @@ -51,19 +51,6 @@ lib.brotli.brotli(app) @app.route('/') def index(): if g.viewer: - if g.viewer.account.service == 'mastodon': - import lib.mastodon - api = lib.mastodon.get_api_for_acc(g.viewer.account) - me = api.account_verify_credentials() - #return str(me) - tl = api.timeline() - return str(tl) - if not api: - raise Exception('frick!!!!!') - else: - raise Exception(api) - - return render_template('logged_in.html', scales=lib.interval.SCALES, tweet_archive_failed = 'tweet_archive_failed' in request.args, settings_error = 'settings_error' in request.args diff --git a/tasks.py b/tasks.py index 8d7993b..0c28371 100644 --- a/tasks.py +++ b/tasks.py @@ -4,6 +4,7 @@ from app import app as flaskapp from app import db from model import Session, Account, TwitterArchive, Post, OAuthToken import lib.twitter +import lib.mastodon from twitter import TwitterError from urllib.error import URLError from datetime import timedelta, datetime @@ -46,11 +47,12 @@ def fetch_acc(id, cursor=None): acc = Account.query.get(id) print(f'fetching {acc}') try: + action = lambda acc, cursor: None if(acc.service == 'twitter'): action = lib.twitter.fetch_acc elif(acc.service == 'mastodon'): action = lib.mastodon.fetch_acc - cursor = action(acc, cursor, **flaskapp.config.get_namespace("TWITTER_")) + cursor = action(acc, cursor) if cursor: fetch_acc.si(id, cursor).apply_async() finally: @@ -67,7 +69,7 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi .limit(limit) for acc in accs: fetch_acc.s(acc.id).delay() - acc.touch_fetch() + #acc.touch_fetch() db.session.commit() @@ -143,23 +145,28 @@ def delete_from_account(account_id): except_(latest_n_posts).\ order_by(db.func.random()).limit(100).all() - posts = refresh_posts(posts) + action = lambda post: None if account.service == 'twitter': - eligible = list((post for post in posts if - (not account.policy_keep_favourites or not post.favourite) - and (not account.policy_keep_media or not post.has_media) - )) - if eligible: - if account.policy_delete_every == timedelta(0): - print("deleting all {} eligible posts for {}".format(len(eligible), account)) - for post in eligible: - account.touch_delete() - lib.twitter.delete(post) - else: - post = random.choice(eligible) - print("deleting {}".format(post)) + action = lib.twitter.delete + elif account.service == 'mastodon': + action = lib.mastodon.delete + + posts = refresh_posts(posts) + eligible = list((post for post in posts if + (not account.policy_keep_favourites or not post.favourite) + and (not account.policy_keep_media or not post.has_media) + )) + if eligible: + if account.policy_delete_every == timedelta(0): + print("deleting all {} eligible posts for {}".format(len(eligible), account)) + for post in eligible: account.touch_delete() - lib.twitter.delete(post) + action(post) + else: + post = random.choice(eligible) + print("deleting {}".format(post)) + account.touch_delete() + action(post) db.session.commit() @@ -170,6 +177,8 @@ def refresh_posts(posts): if posts[0].service == 'twitter': return lib.twitter.refresh_posts(posts) + elif posts[0].service == 'mastodon': + return lib.mastodon.refresh_posts(posts) @app.task(autoretry_for=(TwitterError, URLError)) def refresh_account(account_id):