Merge pull request #176 from codl/175

move known instance buttons entirely client-side

CHANGELOG.markdown

@@ -1,6 +1,8 @@
 ## next
 
 * back off before hitting rate limit on mastodon instances
+* tracking of used instances for the login buttons on the front page is now entirely client-side,
+  avoiding a potential information disclosure vulnerability ([GH-175](https://github.com/codl/forget/issues/175))
 * fix: fetch\_acc running multiple copies fetching the same posts
 * internals: increased frequency of refresh jobs, decreased frequency of bookkeeping jobs
 

app.py

@@ -68,7 +68,7 @@ def install_security_headers(resp):
         csp += "script-src 'self' https://cdn.ravenjs.com/;"
         csp += "connect-src 'self' https://sentry.io/;"
     else:
-        csp += "script-src 'self';"
+        csp += "script-src 'self' 'unsafe-eval';"
         csp += "connect-src 'self';"
 
     if 'CSP_REPORT_URI' in app.config:

assets/instance_buttons.js

@@ -0,0 +1,71 @@
+import {SLOTS, normalize_known, known_load, known_save} from './known_instances.js';
+
+(function instance_buttons(){
+
+    const container = document.querySelector('#mastodon_instance_buttons');
+    const button_template = Function('first', 'instance',
+        'return `' + document.querySelector('#instance_button_template').innerHTML + '`;');
+    const another_button_template = Function(
+        'return `' +
+            document.querySelector('#another_instance_button_template').innerHTML + '`;');
+    const top_instances =
+        Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')();
+
+    async function get_known(){
+        let known = known_load();
+        if(!known){
+            let resp = await fetch('/api/known_instances');
+            if(resp.ok && resp.headers.get('content-type') == 'application/json'){
+                known = await resp.json();
+            }
+            else {
+                known = [{
+                    "instance": "mastodon.social",
+                    "hits": 0
+                }];
+            }
+            known_save(known)
+            fetch('/api/known_instances', {method: 'DELETE'})
+        }
+
+        return known;
+    }
+
+
+    async function replace_buttons(){
+        let known = await get_known();
+
+        known = normalize_known(known);
+        known_save(known);
+
+        let filtered_top_instances = []
+        for(let instance of top_instances){
+            let found = false;
+            for(let k of known){
+                if(k['instance'] == instance['instance']){
+                    found = true;
+                    break;
+                }
+            }
+            if(!found){
+                filtered_top_instances.push(instance)
+            }
+        }
+
+        let instances = known.concat(filtered_top_instances).slice(0, SLOTS);
+
+        let html = '';
+
+        let first = true;
+        for(let instance of instances){
+            html += button_template(first, instance['instance'])
+            first = false;
+        }
+
+        html += another_button_template();
+
+        container.innerHTML = html;
+    }
+
+    replace_buttons();
+})();

assets/known_instances.js

@@ -0,0 +1,44 @@
+const STORAGE_KEY = 'forget_known_instances';
+export const SLOTS = 5;
+
+export function known_save(known){
+    localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
+}
+
+export function known_load(){
+    let known = localStorage.getItem(STORAGE_KEY);
+    if(known){
+        known = JSON.parse(known);
+    }
+    return known;
+}
+
+export function normalize_known(known){
+    /*
+
+    move instances with the most hits to the top SLOTS slots,
+    making sure not to reorder anything that is already there
+
+    */
+    let head = known.slice(0, SLOTS);
+    let tail = known.slice(SLOTS);
+
+    if(tail.length == 0){
+        return known;
+    }
+
+    for(let i = 0; i < SLOTS; i++){
+        let head_min = head.reduce((acc, cur) => acc.hits < cur.hits ? acc : cur);
+        let tail_max = tail.reduce((acc, cur) => acc.hits > cur.hits ? acc : cur);
+        if(head_min.hits < tail_max.hits){
+            // swappy
+            let i = head.indexOf(head_min);
+            let j = tail.indexOf(tail_max);
+            let buf = head[i];
+            head[i] = tail[j];
+            tail[j] = buf;
+        }
+    }
+
+    return head.concat(tail)
+}

assets/settings.js

@@ -1,5 +1,6 @@
 import Banner from '../components/Banner.html';
 import ArchiveForm from '../components/ArchiveForm.html';
+import {known_load, known_save} from './known_instances.js'
 
 (function settings_init(){
     if(!('fetch' in window)){
@@ -142,10 +143,11 @@ import ArchiveForm from '../components/ArchiveForm.html';
 
     let last_viewer = {};
     function update_viewer(viewer){
-        if(last_viewer == JSON.stringify(viewer)){
+        let dumped = JSON.stringify(viewer);
+        if(last_viewer == dumped){
             return;
         }
-        last_viewer = JSON.stringify(viewer);
+        last_viewer = dumped;
 
         document.querySelector('#post-count').textContent = viewer.post_count;
         document.querySelector('#eligible-estimate').textContent = viewer.eligible_for_delete_estimate;
@@ -163,7 +165,9 @@ import ArchiveForm from '../components/ArchiveForm.html';
         banner.set(viewer);
     }
 
-    update_viewer(JSON.parse(document.querySelector('script[data-viewer]').textContent))
+    let viewer_from_dom = JSON.parse(document.querySelector('script[data-viewer]').textContent)
+
+    update_viewer(viewer_from_dom)
 
     function set_viewer_timeout(){
         setTimeout(() => fetch_viewer().then(update_viewer).then(set_viewer_timeout, set_viewer_timeout),
@@ -202,4 +206,30 @@ import ArchiveForm from '../components/ArchiveForm.html';
             },
         })
     }
+
+    function bump_instance(instance_name){
+        let known_instances = known_load();
+        let found = false;
+        for(let instance of known_instances){
+            if(instance['instance'] == instance_name){
+                instance.hits ++;
+                found = true;
+                break;
+            }
+        }
+        if(!found){
+            let instance = {"instance": instance_name, "hits": 1};
+            known_instances.push(instance);
+        }
+
+        known_save(known_instances);
+
+    }
+
+    if(viewer_from_dom['service'] == 'mastodon' && location.hash == '#bump_instance'){
+        bump_instance(viewer_from_dom['id'].split('@')[1])
+        let url = new URL(location.href)
+        url.hash = '';
+        history.replaceState('', '', url);
+    }
 })();

dodo.py

@@ -116,10 +116,11 @@ def task_minify_css():
 def task_rollup():
     """rollup javascript bundle"""
 
-    filenames = ['settings.js']
+    filenames = ['settings.js', 'instance_buttons.js']
     for filename in filenames:
         src = 'assets/{}'.format(filename)
         dst = 'static/{}'.format(filename)
+        name = filename.split('.')[0]
         yield dict(
                 name=filename,
                 file_dep=list(chain(
@@ -130,7 +131,7 @@ def task_rollup():
                 clean=True,
                 actions=[
                     ['node_modules/.bin/rollup', '-c',
-                     '-i', src, '-o', dst, '-f', 'iife'],
+                     '-i', src, '-o', dst, '-n', name, '-f', 'iife'],
                 ],
             )
 

libforget/known_instances.py

@@ -1,86 +0,0 @@
-import json
-
-def find(predicate, iterator, default=None):
-    """
-    returns the first element of iterator that matches predicate
-    or default if none is found
-    """
-    try:
-        return next((el for el in iterator if predicate(el)))
-    except StopIteration:
-        return default
-
-class KnownInstances(object):
-    def __init__(self, serialised=None, top_slots=5):
-        self.instances = list()
-        self.top_slots = top_slots
-        try:
-            unserialised = json.loads(serialised)
-            if not isinstance(unserialised, list):
-                self.__default()
-                return
-            for instance in unserialised:
-                if 'instance' in instance and 'hits' in instance:
-                    self.instances.append(dict(
-                        instance=instance['instance'],
-                        hits=instance['hits']
-                        ))
-        except (json.JSONDecodeError, TypeError):
-            self.__default()
-            return
-
-    def __default(self):
-        self.instances = [{
-                "instance": "mastodon.social",
-                "hits": 0
-            }]
-
-    def clear(self):
-        self.instances = []
-
-    def bump(self, instance_name, bump_by=1):
-        instance = find(
-                lambda i: i['instance'] == instance_name,
-                self.instances)
-        if not instance:
-            instance = dict(instance=instance_name, hits=0)
-            self.instances.append(instance)
-        instance['hits'] += bump_by
-
-    def normalize(self):
-        """
-        raises the top `top_slots` instances to the top,
-        making sure not to move instances that were already at
-        the top
-        """
-        top_slots = self.top_slots
-        head = self.instances[:top_slots]
-        tail = self.instances[top_slots:]
-        if len(tail) == 0:
-            return
-
-        def key(instance):
-            return instance['hits']
-
-        for _ in range(top_slots):
-            head_min = min(head, key=key)
-            tail_max = max(tail, key=key)
-            if tail_max['hits'] > head_min['hits']:
-                # swap them
-                i = head.index(head_min)
-                j = tail.index(tail_max)
-                buf = head[i]
-                head[i] = tail[j]
-                tail[j] = buf
-
-            else:
-                break
-
-        self.instances = head + tail
-
-    def top(self):
-        head = self.instances[:self.top_slots]
-        return tuple((i['instance'] for i in head))
-
-    def serialize(self):
-        return json.dumps(self.instances)

routes/__init__.py

@@ -6,7 +6,6 @@ import libforget.mastodon
 from libforget.auth import require_auth, csrf,\
                      get_viewer
 from model import Session, TwitterArchive, MastodonApp
-from libforget.known_instances import KnownInstances
 from app import app, db, sentry, imgproxy
 import tasks
 from zipfile import BadZipFile
@@ -35,10 +34,7 @@ def index():
 
 @app.route('/about/')
 def about():
-    ki = KnownInstances(request.cookies.get('forget_known_instances', ''))
+    instances = libforget.mastodon.suggested_instances()
-    instances = ki.top()
-    instances += libforget.mastodon.suggested_instances(blacklist=instances)
-    instances = instances[:5]
     return render_template(
             'about.html',
             mastodon_instances=instances,
@@ -269,15 +265,7 @@ def mastodon_login_step2(instance_url):
 
     g.viewer = session
 
-    ki = KnownInstances(request.cookies.get('forget_known_instances', ''))
+    resp = redirect(url_for('index', _anchor='bump_instance'))
-    ki.bump(instance_url)
-
-    resp = redirect(url_for('index'))
-    resp.set_cookie(
-            'forget_known_instances', ki.serialize(),
-            max_age=60*60*24*365,
-            httponly=True
-            )
     return resp
 
 

routes/api.py

@@ -1,9 +1,10 @@
 from app import app, db
 from libforget.auth import require_auth_api, get_viewer
-from flask import jsonify, redirect, make_response, request
+from flask import jsonify, redirect, make_response, request, Response
 from model import Account
 import libforget.settings
 import libforget.json
+import random
 
 @app.route('/api/health_check')
 def health_check():
@@ -59,3 +60,22 @@ def users_badge():
     return redirect(
             "https://img.shields.io/badge/active%20users-{}-blue.svg"
             .format(count))
+
+
+@app.route('/api/known_instances', methods=('GET', 'DELETE'))
+def known_instances():
+    if request.method == 'GET':
+        known = request.cookies.get('forget_known_instances', '')
+        if not known:
+            return Response('[]', 404, mimetype='application/json')
+
+        # pad to avoid oracle attacks
+        for _ in range(random.randint(0, 1000)):
+                known += random.choice((' ', '\t', '\n'))
+
+        return Response(known, mimetype='application/json')
+
+    elif request.method == 'DELETE':
+        resp = Response('', 204)
+        resp.set_cookie('forget_known_instances', '', max_age=0)
+        return resp

templates/about.html

@@ -34,7 +34,7 @@
 </a>
 </p>
 
-<p>
+<p id='mastodon_instance_buttons'>
 
 {% for instance in mastodon_instances %}
 <a style='background-color:#282c37' class='btn primary' href="{{ url_for('mastodon_login_step1', instance_url=instance) }}">
@@ -59,5 +59,41 @@
 
 </p>
 </section>
+
+
+<script type="application/json" id="top_instances">
+[
+{% for instance in mastodon_instances %}
+    {"instance": "{{instance}}"}
+    {%- if not loop.last -%}
+    ,
+    {%- endif %}
+{% endfor %}
+]
+</script>
+
+<script type="text/html+template" id="instance_button_template">
+    <a style='background-color:#282c37' class='btn primary'
+        href="{{ url_for('mastodon_login_step1') }}?instance_url=${encodeURIComponent(instance)}">
+        ${ !first? '' : `
+            {{picture(st, 'mastodon', (20,40,80), ('webp', 'png'))}}
+            Log in with
+            `}
+        ${ instance }
+    </a>
+</script>
+
+<script type="text/html+template" id="another_instance_button_template">
+    <a class='btn secondary' href="{{ url_for('mastodon_login_step1') }}">
+        Another Mastodon instance
+    </a>
+</script>
+
+
 {% endif %}
 {% endblock %}
+
+
+{% block scripts %}
+<script defer src="{{st('instance_buttons.js')}}"></script>
+{% endblock %}

test/test_known_instances.py

@@ -1,59 +0,0 @@
-from libforget.known_instances import KnownInstances
-
-
-def test_known_instances_defaults():
-    ki = KnownInstances()
-    assert len(ki.instances) == 1
-    assert ki.instances[0]['instance'] == 'mastodon.social'
-    assert isinstance(ki.instances[0]['hits'], int)
-
-
-def test_known_instances_clear():
-    ki = KnownInstances()
-    ki.clear()
-    assert len(ki.instances) == 0
-
-
-def test_known_instances_deserialize():
-    ki = KnownInstances(""" [
-        {"instance": "chitter.xyz", "hits": 666, "foo": "bar"},
-        {"instance": "invalid"}
-    ] """)
-    assert len(ki.instances) == 1
-    assert ki.instances[0]['instance'] == "chitter.xyz"
-    assert ki.instances[0]['hits'] == 666
-
-
-def test_known_instances_bump():
-    ki = KnownInstances()
-    ki.bump('chitter.xyz')
-    assert len(ki.instances) == 2
-    assert ki.instances[1]['instance'] == "chitter.xyz"
-    assert ki.instances[1]['hits'] == 1
-
-    ki.bump('chitter.xyz')
-    assert len(ki.instances) == 2
-    assert ki.instances[1]['instance'] == "chitter.xyz"
-    assert ki.instances[1]['hits'] == 2
-
-
-def test_known_instances_normalize_top():
-    ki = KnownInstances(None, top_slots=3)
-    ki.clear()
-    ki.normalize()
-    assert len(ki.instances) == 0
-
-    ki.bump("a", 1)
-    ki.bump("b", 2)
-    ki.bump("c", 3)
-    ki.normalize()
-    assert ki.instances[0]['instance'] == "a"
-    assert ki.instances[1]['instance'] == "b"
-    assert ki.instances[2]['instance'] == "c"
-
-    ki.bump("d", 4)
-    ki.normalize()
-    assert ki.instances[0]['instance'] == "d"
-    assert ki.instances[3]['instance'] == "a"
-
-    assert ki.top() == ("d", "b", "c")