From b57f71ae5849e3d9a642fe1be75bf1c9f5c9067a Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 17:15:58 +0100 Subject: [PATCH 01/12] remove known instances --- libforget/known_instances.py | 86 ------------------------------------ routes/__init__.py | 14 +----- test/test_known_instances.py | 59 ------------------------- 3 files changed, 1 insertion(+), 158 deletions(-) delete mode 100644 libforget/known_instances.py delete mode 100644 test/test_known_instances.py diff --git a/libforget/known_instances.py b/libforget/known_instances.py deleted file mode 100644 index 239e9a2..0000000 --- a/libforget/known_instances.py +++ /dev/null @@ -1,86 +0,0 @@ -import json - -def find(predicate, iterator, default=None): - """ - returns the first element of iterator that matches predicate - or default if none is found - """ - try: - return next((el for el in iterator if predicate(el))) - except StopIteration: - return default - -class KnownInstances(object): - def __init__(self, serialised=None, top_slots=5): - self.instances = list() - self.top_slots = top_slots - try: - unserialised = json.loads(serialised) - if not isinstance(unserialised, list): - self.__default() - return - for instance in unserialised: - if 'instance' in instance and 'hits' in instance: - self.instances.append(dict( - instance=instance['instance'], - hits=instance['hits'] - )) - except (json.JSONDecodeError, TypeError): - self.__default() - return - - def __default(self): - self.instances = [{ - "instance": "mastodon.social", - "hits": 0 - }] - - def clear(self): - self.instances = [] - - def bump(self, instance_name, bump_by=1): - instance = find( - lambda i: i['instance'] == instance_name, - self.instances) - if not instance: - instance = dict(instance=instance_name, hits=0) - self.instances.append(instance) - instance['hits'] += bump_by - - def normalize(self): - """ - raises the top `top_slots` instances to the top, - making sure not to move instances that were already at - the top - """ - top_slots = self.top_slots - head = self.instances[:top_slots] - tail = self.instances[top_slots:] - if len(tail) == 0: - return - - def key(instance): - return instance['hits'] - - for _ in range(top_slots): - head_min = min(head, key=key) - tail_max = max(tail, key=key) - if tail_max['hits'] > head_min['hits']: - # swap them - i = head.index(head_min) - j = tail.index(tail_max) - buf = head[i] - head[i] = tail[j] - tail[j] = buf - - else: - break - - self.instances = head + tail - - def top(self): - head = self.instances[:self.top_slots] - return tuple((i['instance'] for i in head)) - - def serialize(self): - return json.dumps(self.instances) diff --git a/routes/__init__.py b/routes/__init__.py index a3eecf7..0304bc6 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -6,7 +6,6 @@ import libforget.mastodon from libforget.auth import require_auth, csrf,\ get_viewer from model import Session, TwitterArchive, MastodonApp -from libforget.known_instances import KnownInstances from app import app, db, sentry, imgproxy import tasks from zipfile import BadZipFile @@ -35,10 +34,7 @@ def index(): @app.route('/about/') def about(): - ki = KnownInstances(request.cookies.get('forget_known_instances', '')) - instances = ki.top() - instances += libforget.mastodon.suggested_instances(blacklist=instances) - instances = instances[:5] + instances = libforget.mastodon.suggested_instances() return render_template( 'about.html', mastodon_instances=instances, @@ -269,15 +265,7 @@ def mastodon_login_step2(instance_url): g.viewer = session - ki = KnownInstances(request.cookies.get('forget_known_instances', '')) - ki.bump(instance_url) - resp = redirect(url_for('index')) - resp.set_cookie( - 'forget_known_instances', ki.serialize(), - max_age=60*60*24*365, - httponly=True - ) return resp diff --git a/test/test_known_instances.py b/test/test_known_instances.py deleted file mode 100644 index 0f2e111..0000000 --- a/test/test_known_instances.py +++ /dev/null @@ -1,59 +0,0 @@ -from libforget.known_instances import KnownInstances - - -def test_known_instances_defaults(): - ki = KnownInstances() - assert len(ki.instances) == 1 - assert ki.instances[0]['instance'] == 'mastodon.social' - assert isinstance(ki.instances[0]['hits'], int) - - -def test_known_instances_clear(): - ki = KnownInstances() - ki.clear() - assert len(ki.instances) == 0 - - -def test_known_instances_deserialize(): - ki = KnownInstances(""" [ - {"instance": "chitter.xyz", "hits": 666, "foo": "bar"}, - {"instance": "invalid"} - ] """) - assert len(ki.instances) == 1 - assert ki.instances[0]['instance'] == "chitter.xyz" - assert ki.instances[0]['hits'] == 666 - - -def test_known_instances_bump(): - ki = KnownInstances() - ki.bump('chitter.xyz') - assert len(ki.instances) == 2 - assert ki.instances[1]['instance'] == "chitter.xyz" - assert ki.instances[1]['hits'] == 1 - - ki.bump('chitter.xyz') - assert len(ki.instances) == 2 - assert ki.instances[1]['instance'] == "chitter.xyz" - assert ki.instances[1]['hits'] == 2 - - -def test_known_instances_normalize_top(): - ki = KnownInstances(None, top_slots=3) - ki.clear() - ki.normalize() - assert len(ki.instances) == 0 - - ki.bump("a", 1) - ki.bump("b", 2) - ki.bump("c", 3) - ki.normalize() - assert ki.instances[0]['instance'] == "a" - assert ki.instances[1]['instance'] == "b" - assert ki.instances[2]['instance'] == "c" - - ki.bump("d", 4) - ki.normalize() - assert ki.instances[0]['instance'] == "d" - assert ki.instances[3]['instance'] == "a" - - assert ki.top() == ("d", "b", "c") From 8cf12f31c80cfa9298ed664c6f3cd79b0baf87a6 Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 17:18:43 +0100 Subject: [PATCH 02/12] add endpoint to access and clear existing known instances cookie --- routes/api.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/routes/api.py b/routes/api.py index 856f3b0..95a9673 100644 --- a/routes/api.py +++ b/routes/api.py @@ -1,6 +1,6 @@ from app import app, db from libforget.auth import require_auth_api, get_viewer -from flask import jsonify, redirect, make_response, request +from flask import jsonify, redirect, make_response, request, Response from model import Account import libforget.settings import libforget.json @@ -59,3 +59,18 @@ def users_badge(): return redirect( "https://img.shields.io/badge/active%20users-{}-blue.svg" .format(count)) + + +@app.route('/api/known_instances', methods=('GET', 'DELETE')) +def known_instances(): + if request.method == 'GET': + known = request.cookies.get('forget_known_instances', '') + if not known: + return Response('[]', 404, mimetype='application/json') + + return Response(known, mimetype='application/json') + + elif request.method == 'DELETE': + resp = Response('', 204) + resp.set_cookie('forget_known_instances', '', max_age=0) + return resp From 8cca6c2fe3774fef3b1bdbf83f171a5bbccb12d0 Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 17:59:44 +0100 Subject: [PATCH 03/12] add templates for instance buttons --- templates/about.html | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/templates/about.html b/templates/about.html index ca97eaf..af0b4df 100644 --- a/templates/about.html +++ b/templates/about.html @@ -34,7 +34,7 @@

-

+

{% for instance in mastodon_instances %} @@ -59,5 +59,35 @@

+ + + + + + + + {% endif %} {% endblock %} From ec10d15217dcb3c682f289284756df1f36fc912e Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 18:29:55 +0100 Subject: [PATCH 04/12] pad to avoid oracle attacks on /api/known_instances --- routes/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/routes/api.py b/routes/api.py index 95a9673..4f35f17 100644 --- a/routes/api.py +++ b/routes/api.py @@ -4,6 +4,7 @@ from flask import jsonify, redirect, make_response, request, Response from model import Account import libforget.settings import libforget.json +import random @app.route('/api/health_check') def health_check(): @@ -68,6 +69,10 @@ def known_instances(): if not known: return Response('[]', 404, mimetype='application/json') + # pad to avoid oracle attacks + for _ in range(random.randint(0, 1000)): + known += random.choice((' ', '\t', '\n')) + return Response(known, mimetype='application/json') elif request.method == 'DELETE': From 17f59a018fa1625dd6f1569fc69436f59b808079 Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 19:46:18 +0100 Subject: [PATCH 05/12] script to show known instances from cookie, mmoving it to localstorage --- app.py | 2 +- assets/instance_buttons.js | 51 ++++++++++++++++++++++++++++++++++++++ dodo.py | 2 +- routes/api.py | 2 +- templates/about.html | 10 ++++++-- 5 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 assets/instance_buttons.js diff --git a/app.py b/app.py index 339840f..c306f73 100644 --- a/app.py +++ b/app.py @@ -68,7 +68,7 @@ def install_security_headers(resp): csp += "script-src 'self' https://cdn.ravenjs.com/;" csp += "connect-src 'self' https://sentry.io/;" else: - csp += "script-src 'self';" + csp += "script-src 'self' 'unsafe-eval';" csp += "connect-src 'self';" if 'CSP_REPORT_URI' in app.config: diff --git a/assets/instance_buttons.js b/assets/instance_buttons.js new file mode 100644 index 0000000..71cf4c6 --- /dev/null +++ b/assets/instance_buttons.js @@ -0,0 +1,51 @@ +(function instance_buttons(){ + + const STORAGE_KEY = 'forget_known_instances'; + + const container = document.querySelector('#mastodon_instance_buttons'); + const button_template = Function('first', 'instance', + 'return `' + document.querySelector('#instance_button_template').innerHTML + '`;'); + const another_button_template = Function( + 'return `' + + document.querySelector('#another_instance_button_template').innerHTML + '`;'); + const top_instances = + Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')(); + + async function get_known(){ + let known = JSON.parse(localStorage.getItem(STORAGE_KEY)); + let has_been_fetched = false; + if(!known){ + let resp = await fetch('/api/known_instances'); + if(resp.ok && resp.headers.get('content-type') == 'application/json'){ + known = await resp.json(); + } + else { + known = []; + } + localStorage.setItem(STORAGE_KEY, JSON.stringify(known)); + fetch('/api/known_instances', {method: 'DELETE'}) + } + + return known; + } + + async function replace_buttons(){ + let known = await get_known(); + + let instances = known.concat(top_instances).slice(0, 5); + + let html = ''; + + let first = true; + for(let instance of instances){ + html += button_template(first, instance['instance']) + first = false; + } + + html += another_button_template(); + + container.innerHTML = html; + } + + replace_buttons(); +})(); diff --git a/dodo.py b/dodo.py index 8910198..7aa734f 100644 --- a/dodo.py +++ b/dodo.py @@ -116,7 +116,7 @@ def task_minify_css(): def task_rollup(): """rollup javascript bundle""" - filenames = ['settings.js'] + filenames = ['settings.js', 'instance_buttons.js'] for filename in filenames: src = 'assets/{}'.format(filename) dst = 'static/{}'.format(filename) diff --git a/routes/api.py b/routes/api.py index 4f35f17..07460c8 100644 --- a/routes/api.py +++ b/routes/api.py @@ -67,7 +67,7 @@ def known_instances(): if request.method == 'GET': known = request.cookies.get('forget_known_instances', '') if not known: - return Response('[]', 404, mimetype='application/json') + return Response('[]', 200, mimetype='application/json') # pad to avoid oracle attacks for _ in range(random.randint(0, 1000)): diff --git a/templates/about.html b/templates/about.html index af0b4df..3766b43 100644 --- a/templates/about.html +++ b/templates/about.html @@ -64,7 +64,7 @@ + {% endif %} {% endblock %} + + +{% block scripts %} + +{% endblock %} From 915a6029d70716518f80a209b4980f5bfe38b0ae Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 20:25:45 +0100 Subject: [PATCH 06/12] port algorithm for normalizing known instances to five visible slots --- assets/instance_buttons.js | 62 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/assets/instance_buttons.js b/assets/instance_buttons.js index 71cf4c6..1cd4f1f 100644 --- a/assets/instance_buttons.js +++ b/assets/instance_buttons.js @@ -1,5 +1,7 @@ (function instance_buttons(){ + const SLOTS = 5; + const STORAGE_KEY = 'forget_known_instances'; const container = document.querySelector('#mastodon_instance_buttons'); @@ -20,19 +22,73 @@ known = await resp.json(); } else { - known = []; + known = [{ + "instance": "mastodon.social", + "hits": 0 + }]; } - localStorage.setItem(STORAGE_KEY, JSON.stringify(known)); + save(known) fetch('/api/known_instances', {method: 'DELETE'}) } return known; } + function normalize(known){ + /* + + move instances with the most hits to the top SLOTS slots, + making sure not to reorder anything that is already there + + */ + let head = known.slice(0, SLOTS); + let tail = known.slice(SLOTS); + + if(tail.length == 0){ + return known; + } + + for(let i = 0; i < SLOTS; i++){ + let head_min = head.reduce((acc, cur) => acc.hits < cur.hits ? acc : cur); + let tail_max = tail.reduce((acc, cur) => acc.hits > cur.hits ? acc : cur); + if(head_min.hits < tail_max.hits){ + // swappy + let i = head.indexOf(head_min); + let j = tail.indexOf(tail_max); + let buf = head[i]; + head[i] = tail[j]; + tail[j] = buf; + } + } + + return head.concat(tail) + } + + function save(known){ + localStorage.setItem(STORAGE_KEY, JSON.stringify(known)); + } + async function replace_buttons(){ let known = await get_known(); - let instances = known.concat(top_instances).slice(0, 5); + known = normalize(known); + save(known); + + let filtered_top_instances = [] + for(let instance of top_instances){ + let found = false; + for(let k of known){ + if(k['instance'] == instance){ + found = true; + break; + } + } + if(!found){ + filtered_top_instances.push(instance) + } + } + + let instances = known.concat(filtered_top_instances).slice(0, SLOTS); let html = ''; From 2bacbaa8b1c2e614aa82d2d60fcd46d25dc54c2e Mon Sep 17 00:00:00 2001 From: codl Date: Fri, 15 Mar 2019 21:09:22 +0100 Subject: [PATCH 07/12] known instances: bump instance counter when logging in --- assets/instance_buttons.js | 18 +++++++----------- assets/known_instances.js | 13 +++++++++++++ assets/settings.js | 38 +++++++++++++++++++++++++++++++++++--- dodo.py | 3 ++- routes/__init__.py | 2 +- templates/about.html | 2 +- 6 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 assets/known_instances.js diff --git a/assets/instance_buttons.js b/assets/instance_buttons.js index 1cd4f1f..1aa517e 100644 --- a/assets/instance_buttons.js +++ b/assets/instance_buttons.js @@ -1,9 +1,9 @@ +const SLOTS = 5; + +import {known_load, known_save} from './known_instances.js'; + (function instance_buttons(){ - const SLOTS = 5; - - const STORAGE_KEY = 'forget_known_instances'; - const container = document.querySelector('#mastodon_instance_buttons'); const button_template = Function('first', 'instance', 'return `' + document.querySelector('#instance_button_template').innerHTML + '`;'); @@ -14,8 +14,7 @@ Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')(); async function get_known(){ - let known = JSON.parse(localStorage.getItem(STORAGE_KEY)); - let has_been_fetched = false; + let known = known_load(); if(!known){ let resp = await fetch('/api/known_instances'); if(resp.ok && resp.headers.get('content-type') == 'application/json'){ @@ -27,7 +26,7 @@ "hits": 0 }]; } - save(known) + known_save(known) fetch('/api/known_instances', {method: 'DELETE'}) } @@ -64,15 +63,12 @@ return head.concat(tail) } - function save(known){ - localStorage.setItem(STORAGE_KEY, JSON.stringify(known)); - } async function replace_buttons(){ let known = await get_known(); known = normalize(known); - save(known); + known_save(known); let filtered_top_instances = [] for(let instance of top_instances){ diff --git a/assets/known_instances.js b/assets/known_instances.js new file mode 100644 index 0000000..f75509f --- /dev/null +++ b/assets/known_instances.js @@ -0,0 +1,13 @@ +const STORAGE_KEY = 'forget_known_instances'; + +export function known_save(known){ + localStorage.setItem(STORAGE_KEY, JSON.stringify(known)); +} + +export function known_load(){ + let known = localStorage.getItem(STORAGE_KEY); + if(known){ + known = JSON.parse(known); + } + return known; +} diff --git a/assets/settings.js b/assets/settings.js index cc404c0..ab5df50 100644 --- a/assets/settings.js +++ b/assets/settings.js @@ -1,5 +1,6 @@ import Banner from '../components/Banner.html'; import ArchiveForm from '../components/ArchiveForm.html'; +import {known_load, known_save} from './known_instances.js' (function settings_init(){ if(!('fetch' in window)){ @@ -142,10 +143,12 @@ import ArchiveForm from '../components/ArchiveForm.html'; let last_viewer = {}; function update_viewer(viewer){ - if(last_viewer == JSON.stringify(viewer)){ + let dumped = JSON.stringify(viewer); + if(last_viewer == dumped){ + console.log('viewers is the same'); return; } - last_viewer = JSON.stringify(viewer); + last_viewer = dumped; document.querySelector('#post-count').textContent = viewer.post_count; document.querySelector('#eligible-estimate').textContent = viewer.eligible_for_delete_estimate; @@ -163,7 +166,9 @@ import ArchiveForm from '../components/ArchiveForm.html'; banner.set(viewer); } - update_viewer(JSON.parse(document.querySelector('script[data-viewer]').textContent)) + let viewer_from_dom = JSON.parse(document.querySelector('script[data-viewer]').textContent) + + update_viewer(viewer_from_dom) function set_viewer_timeout(){ setTimeout(() => fetch_viewer().then(update_viewer).then(set_viewer_timeout, set_viewer_timeout), @@ -202,4 +207,31 @@ import ArchiveForm from '../components/ArchiveForm.html'; }, }) } + + function bump_instance(instance_name){ + let known_instances = known_load(); + let found = false; + for(let instance of known_instances){ + if(instance['instance'] == instance_name){ + instance.hits ++; + found = true; + break; + } + } + if(!found){ + let instance = {"instance": instance_name, "hits": 1}; + known_instances.push(instance); + } + + known_save(known_instances); + + } + + if(viewer_from_dom['service'] == 'mastodon' && location.hash == '#bump_instance'){ + console.log('bumpin') + bump_instance(viewer_from_dom['id'].split('@')[1]) + let url = new URL(location.href) + url.hash = ''; + history.replaceState('', '', url); + } })(); diff --git a/dodo.py b/dodo.py index 7aa734f..75ceb08 100644 --- a/dodo.py +++ b/dodo.py @@ -120,6 +120,7 @@ def task_rollup(): for filename in filenames: src = 'assets/{}'.format(filename) dst = 'static/{}'.format(filename) + name = filename.split('.')[0] yield dict( name=filename, file_dep=list(chain( @@ -130,7 +131,7 @@ def task_rollup(): clean=True, actions=[ ['node_modules/.bin/rollup', '-c', - '-i', src, '-o', dst, '-f', 'iife'], + '-i', src, '-o', dst, '-n', name, '-f', 'iife'], ], ) diff --git a/routes/__init__.py b/routes/__init__.py index 0304bc6..6323712 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -265,7 +265,7 @@ def mastodon_login_step2(instance_url): g.viewer = session - resp = redirect(url_for('index')) + resp = redirect(url_for('index', _anchor='bump_instance')) return resp diff --git a/templates/about.html b/templates/about.html index 3766b43..eeab2db 100644 --- a/templates/about.html +++ b/templates/about.html @@ -74,7 +74,7 @@