From b57f71ae5849e3d9a642fe1be75bf1c9f5c9067a Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 17:15:58 +0100
Subject: [PATCH 01/12] remove known instances
---
libforget/known_instances.py | 86 ------------------------------------
routes/__init__.py | 14 +-----
test/test_known_instances.py | 59 -------------------------
3 files changed, 1 insertion(+), 158 deletions(-)
delete mode 100644 libforget/known_instances.py
delete mode 100644 test/test_known_instances.py
diff --git a/libforget/known_instances.py b/libforget/known_instances.py
deleted file mode 100644
index 239e9a2..0000000
--- a/libforget/known_instances.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import json
-
-def find(predicate, iterator, default=None):
- """
- returns the first element of iterator that matches predicate
- or default if none is found
- """
- try:
- return next((el for el in iterator if predicate(el)))
- except StopIteration:
- return default
-
-class KnownInstances(object):
- def __init__(self, serialised=None, top_slots=5):
- self.instances = list()
- self.top_slots = top_slots
- try:
- unserialised = json.loads(serialised)
- if not isinstance(unserialised, list):
- self.__default()
- return
- for instance in unserialised:
- if 'instance' in instance and 'hits' in instance:
- self.instances.append(dict(
- instance=instance['instance'],
- hits=instance['hits']
- ))
- except (json.JSONDecodeError, TypeError):
- self.__default()
- return
-
- def __default(self):
- self.instances = [{
- "instance": "mastodon.social",
- "hits": 0
- }]
-
- def clear(self):
- self.instances = []
-
- def bump(self, instance_name, bump_by=1):
- instance = find(
- lambda i: i['instance'] == instance_name,
- self.instances)
- if not instance:
- instance = dict(instance=instance_name, hits=0)
- self.instances.append(instance)
- instance['hits'] += bump_by
-
- def normalize(self):
- """
- raises the top `top_slots` instances to the top,
- making sure not to move instances that were already at
- the top
- """
- top_slots = self.top_slots
- head = self.instances[:top_slots]
- tail = self.instances[top_slots:]
- if len(tail) == 0:
- return
-
- def key(instance):
- return instance['hits']
-
- for _ in range(top_slots):
- head_min = min(head, key=key)
- tail_max = max(tail, key=key)
- if tail_max['hits'] > head_min['hits']:
- # swap them
- i = head.index(head_min)
- j = tail.index(tail_max)
- buf = head[i]
- head[i] = tail[j]
- tail[j] = buf
-
- else:
- break
-
- self.instances = head + tail
-
- def top(self):
- head = self.instances[:self.top_slots]
- return tuple((i['instance'] for i in head))
-
- def serialize(self):
- return json.dumps(self.instances)
diff --git a/routes/__init__.py b/routes/__init__.py
index a3eecf7..0304bc6 100644
--- a/routes/__init__.py
+++ b/routes/__init__.py
@@ -6,7 +6,6 @@ import libforget.mastodon
from libforget.auth import require_auth, csrf,\
get_viewer
from model import Session, TwitterArchive, MastodonApp
-from libforget.known_instances import KnownInstances
from app import app, db, sentry, imgproxy
import tasks
from zipfile import BadZipFile
@@ -35,10 +34,7 @@ def index():
@app.route('/about/')
def about():
- ki = KnownInstances(request.cookies.get('forget_known_instances', ''))
- instances = ki.top()
- instances += libforget.mastodon.suggested_instances(blacklist=instances)
- instances = instances[:5]
+ instances = libforget.mastodon.suggested_instances()
return render_template(
'about.html',
mastodon_instances=instances,
@@ -269,15 +265,7 @@ def mastodon_login_step2(instance_url):
g.viewer = session
- ki = KnownInstances(request.cookies.get('forget_known_instances', ''))
- ki.bump(instance_url)
-
resp = redirect(url_for('index'))
- resp.set_cookie(
- 'forget_known_instances', ki.serialize(),
- max_age=60*60*24*365,
- httponly=True
- )
return resp
diff --git a/test/test_known_instances.py b/test/test_known_instances.py
deleted file mode 100644
index 0f2e111..0000000
--- a/test/test_known_instances.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from libforget.known_instances import KnownInstances
-
-
-def test_known_instances_defaults():
- ki = KnownInstances()
- assert len(ki.instances) == 1
- assert ki.instances[0]['instance'] == 'mastodon.social'
- assert isinstance(ki.instances[0]['hits'], int)
-
-
-def test_known_instances_clear():
- ki = KnownInstances()
- ki.clear()
- assert len(ki.instances) == 0
-
-
-def test_known_instances_deserialize():
- ki = KnownInstances(""" [
- {"instance": "chitter.xyz", "hits": 666, "foo": "bar"},
- {"instance": "invalid"}
- ] """)
- assert len(ki.instances) == 1
- assert ki.instances[0]['instance'] == "chitter.xyz"
- assert ki.instances[0]['hits'] == 666
-
-
-def test_known_instances_bump():
- ki = KnownInstances()
- ki.bump('chitter.xyz')
- assert len(ki.instances) == 2
- assert ki.instances[1]['instance'] == "chitter.xyz"
- assert ki.instances[1]['hits'] == 1
-
- ki.bump('chitter.xyz')
- assert len(ki.instances) == 2
- assert ki.instances[1]['instance'] == "chitter.xyz"
- assert ki.instances[1]['hits'] == 2
-
-
-def test_known_instances_normalize_top():
- ki = KnownInstances(None, top_slots=3)
- ki.clear()
- ki.normalize()
- assert len(ki.instances) == 0
-
- ki.bump("a", 1)
- ki.bump("b", 2)
- ki.bump("c", 3)
- ki.normalize()
- assert ki.instances[0]['instance'] == "a"
- assert ki.instances[1]['instance'] == "b"
- assert ki.instances[2]['instance'] == "c"
-
- ki.bump("d", 4)
- ki.normalize()
- assert ki.instances[0]['instance'] == "d"
- assert ki.instances[3]['instance'] == "a"
-
- assert ki.top() == ("d", "b", "c")
From 8cf12f31c80cfa9298ed664c6f3cd79b0baf87a6 Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 17:18:43 +0100
Subject: [PATCH 02/12] add endpoint to access and clear existing known
instances cookie
---
routes/api.py | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/routes/api.py b/routes/api.py
index 856f3b0..95a9673 100644
--- a/routes/api.py
+++ b/routes/api.py
@@ -1,6 +1,6 @@
from app import app, db
from libforget.auth import require_auth_api, get_viewer
-from flask import jsonify, redirect, make_response, request
+from flask import jsonify, redirect, make_response, request, Response
from model import Account
import libforget.settings
import libforget.json
@@ -59,3 +59,18 @@ def users_badge():
return redirect(
"https://img.shields.io/badge/active%20users-{}-blue.svg"
.format(count))
+
+
+@app.route('/api/known_instances', methods=('GET', 'DELETE'))
+def known_instances():
+ if request.method == 'GET':
+ known = request.cookies.get('forget_known_instances', '')
+ if not known:
+ return Response('[]', 404, mimetype='application/json')
+
+ return Response(known, mimetype='application/json')
+
+ elif request.method == 'DELETE':
+ resp = Response('', 204)
+ resp.set_cookie('forget_known_instances', '', max_age=0)
+ return resp
From 8cca6c2fe3774fef3b1bdbf83f171a5bbccb12d0 Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 17:59:44 +0100
Subject: [PATCH 03/12] add templates for instance buttons
---
templates/about.html | 32 +++++++++++++++++++++++++++++++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/templates/about.html b/templates/about.html
index ca97eaf..af0b4df 100644
--- a/templates/about.html
+++ b/templates/about.html
@@ -34,7 +34,7 @@
-
+
{% for instance in mastodon_instances %}
@@ -59,5 +59,35 @@
+
+
+
+
+
+
+
+
{% endif %}
{% endblock %}
From ec10d15217dcb3c682f289284756df1f36fc912e Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 18:29:55 +0100
Subject: [PATCH 04/12] pad to avoid oracle attacks on /api/known_instances
---
routes/api.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/routes/api.py b/routes/api.py
index 95a9673..4f35f17 100644
--- a/routes/api.py
+++ b/routes/api.py
@@ -4,6 +4,7 @@ from flask import jsonify, redirect, make_response, request, Response
from model import Account
import libforget.settings
import libforget.json
+import random
@app.route('/api/health_check')
def health_check():
@@ -68,6 +69,10 @@ def known_instances():
if not known:
return Response('[]', 404, mimetype='application/json')
+ # pad to avoid oracle attacks
+ for _ in range(random.randint(0, 1000)):
+ known += random.choice((' ', '\t', '\n'))
+
return Response(known, mimetype='application/json')
elif request.method == 'DELETE':
From 17f59a018fa1625dd6f1569fc69436f59b808079 Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 19:46:18 +0100
Subject: [PATCH 05/12] script to show known instances from cookie, mmoving it
to localstorage
---
app.py | 2 +-
assets/instance_buttons.js | 51 ++++++++++++++++++++++++++++++++++++++
dodo.py | 2 +-
routes/api.py | 2 +-
templates/about.html | 10 ++++++--
5 files changed, 62 insertions(+), 5 deletions(-)
create mode 100644 assets/instance_buttons.js
diff --git a/app.py b/app.py
index 339840f..c306f73 100644
--- a/app.py
+++ b/app.py
@@ -68,7 +68,7 @@ def install_security_headers(resp):
csp += "script-src 'self' https://cdn.ravenjs.com/;"
csp += "connect-src 'self' https://sentry.io/;"
else:
- csp += "script-src 'self';"
+ csp += "script-src 'self' 'unsafe-eval';"
csp += "connect-src 'self';"
if 'CSP_REPORT_URI' in app.config:
diff --git a/assets/instance_buttons.js b/assets/instance_buttons.js
new file mode 100644
index 0000000..71cf4c6
--- /dev/null
+++ b/assets/instance_buttons.js
@@ -0,0 +1,51 @@
+(function instance_buttons(){
+
+ const STORAGE_KEY = 'forget_known_instances';
+
+ const container = document.querySelector('#mastodon_instance_buttons');
+ const button_template = Function('first', 'instance',
+ 'return `' + document.querySelector('#instance_button_template').innerHTML + '`;');
+ const another_button_template = Function(
+ 'return `' +
+ document.querySelector('#another_instance_button_template').innerHTML + '`;');
+ const top_instances =
+ Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')();
+
+ async function get_known(){
+ let known = JSON.parse(localStorage.getItem(STORAGE_KEY));
+ let has_been_fetched = false;
+ if(!known){
+ let resp = await fetch('/api/known_instances');
+ if(resp.ok && resp.headers.get('content-type') == 'application/json'){
+ known = await resp.json();
+ }
+ else {
+ known = [];
+ }
+ localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
+ fetch('/api/known_instances', {method: 'DELETE'})
+ }
+
+ return known;
+ }
+
+ async function replace_buttons(){
+ let known = await get_known();
+
+ let instances = known.concat(top_instances).slice(0, 5);
+
+ let html = '';
+
+ let first = true;
+ for(let instance of instances){
+ html += button_template(first, instance['instance'])
+ first = false;
+ }
+
+ html += another_button_template();
+
+ container.innerHTML = html;
+ }
+
+ replace_buttons();
+})();
diff --git a/dodo.py b/dodo.py
index 8910198..7aa734f 100644
--- a/dodo.py
+++ b/dodo.py
@@ -116,7 +116,7 @@ def task_minify_css():
def task_rollup():
"""rollup javascript bundle"""
- filenames = ['settings.js']
+ filenames = ['settings.js', 'instance_buttons.js']
for filename in filenames:
src = 'assets/{}'.format(filename)
dst = 'static/{}'.format(filename)
diff --git a/routes/api.py b/routes/api.py
index 4f35f17..07460c8 100644
--- a/routes/api.py
+++ b/routes/api.py
@@ -67,7 +67,7 @@ def known_instances():
if request.method == 'GET':
known = request.cookies.get('forget_known_instances', '')
if not known:
- return Response('[]', 404, mimetype='application/json')
+ return Response('[]', 200, mimetype='application/json')
# pad to avoid oracle attacks
for _ in range(random.randint(0, 1000)):
diff --git a/templates/about.html b/templates/about.html
index af0b4df..3766b43 100644
--- a/templates/about.html
+++ b/templates/about.html
@@ -64,7 +64,7 @@
+
{% endif %}
{% endblock %}
+
+
+{% block scripts %}
+
+{% endblock %}
From 915a6029d70716518f80a209b4980f5bfe38b0ae Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 20:25:45 +0100
Subject: [PATCH 06/12] port algorithm for normalizing known instances to five
visible slots
---
assets/instance_buttons.js | 62 ++++++++++++++++++++++++++++++++++++--
1 file changed, 59 insertions(+), 3 deletions(-)
diff --git a/assets/instance_buttons.js b/assets/instance_buttons.js
index 71cf4c6..1cd4f1f 100644
--- a/assets/instance_buttons.js
+++ b/assets/instance_buttons.js
@@ -1,5 +1,7 @@
(function instance_buttons(){
+ const SLOTS = 5;
+
const STORAGE_KEY = 'forget_known_instances';
const container = document.querySelector('#mastodon_instance_buttons');
@@ -20,19 +22,73 @@
known = await resp.json();
}
else {
- known = [];
+ known = [{
+ "instance": "mastodon.social",
+ "hits": 0
+ }];
}
- localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
+ save(known)
fetch('/api/known_instances', {method: 'DELETE'})
}
return known;
}
+ function normalize(known){
+ /*
+
+ move instances with the most hits to the top SLOTS slots,
+ making sure not to reorder anything that is already there
+
+ */
+ let head = known.slice(0, SLOTS);
+ let tail = known.slice(SLOTS);
+
+ if(tail.length == 0){
+ return known;
+ }
+
+ for(let i = 0; i < SLOTS; i++){
+ let head_min = head.reduce((acc, cur) => acc.hits < cur.hits ? acc : cur);
+ let tail_max = tail.reduce((acc, cur) => acc.hits > cur.hits ? acc : cur);
+ if(head_min.hits < tail_max.hits){
+ // swappy
+ let i = head.indexOf(head_min);
+ let j = tail.indexOf(tail_max);
+ let buf = head[i];
+ head[i] = tail[j];
+ tail[j] = buf;
+ }
+ }
+
+ return head.concat(tail)
+ }
+
+ function save(known){
+ localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
+ }
+
async function replace_buttons(){
let known = await get_known();
- let instances = known.concat(top_instances).slice(0, 5);
+ known = normalize(known);
+ save(known);
+
+ let filtered_top_instances = []
+ for(let instance of top_instances){
+ let found = false;
+ for(let k of known){
+ if(k['instance'] == instance){
+ found = true;
+ break;
+ }
+ }
+ if(!found){
+ filtered_top_instances.push(instance)
+ }
+ }
+
+ let instances = known.concat(filtered_top_instances).slice(0, SLOTS);
let html = '';
From 2bacbaa8b1c2e614aa82d2d60fcd46d25dc54c2e Mon Sep 17 00:00:00 2001
From: codl
Date: Fri, 15 Mar 2019 21:09:22 +0100
Subject: [PATCH 07/12] known instances: bump instance counter when logging in
---
assets/instance_buttons.js | 18 +++++++-----------
assets/known_instances.js | 13 +++++++++++++
assets/settings.js | 38 +++++++++++++++++++++++++++++++++++---
dodo.py | 3 ++-
routes/__init__.py | 2 +-
templates/about.html | 2 +-
6 files changed, 59 insertions(+), 17 deletions(-)
create mode 100644 assets/known_instances.js
diff --git a/assets/instance_buttons.js b/assets/instance_buttons.js
index 1cd4f1f..1aa517e 100644
--- a/assets/instance_buttons.js
+++ b/assets/instance_buttons.js
@@ -1,9 +1,9 @@
+const SLOTS = 5;
+
+import {known_load, known_save} from './known_instances.js';
+
(function instance_buttons(){
- const SLOTS = 5;
-
- const STORAGE_KEY = 'forget_known_instances';
-
const container = document.querySelector('#mastodon_instance_buttons');
const button_template = Function('first', 'instance',
'return `' + document.querySelector('#instance_button_template').innerHTML + '`;');
@@ -14,8 +14,7 @@
Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')();
async function get_known(){
- let known = JSON.parse(localStorage.getItem(STORAGE_KEY));
- let has_been_fetched = false;
+ let known = known_load();
if(!known){
let resp = await fetch('/api/known_instances');
if(resp.ok && resp.headers.get('content-type') == 'application/json'){
@@ -27,7 +26,7 @@
"hits": 0
}];
}
- save(known)
+ known_save(known)
fetch('/api/known_instances', {method: 'DELETE'})
}
@@ -64,15 +63,12 @@
return head.concat(tail)
}
- function save(known){
- localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
- }
async function replace_buttons(){
let known = await get_known();
known = normalize(known);
- save(known);
+ known_save(known);
let filtered_top_instances = []
for(let instance of top_instances){
diff --git a/assets/known_instances.js b/assets/known_instances.js
new file mode 100644
index 0000000..f75509f
--- /dev/null
+++ b/assets/known_instances.js
@@ -0,0 +1,13 @@
+const STORAGE_KEY = 'forget_known_instances';
+
+export function known_save(known){
+ localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
+}
+
+export function known_load(){
+ let known = localStorage.getItem(STORAGE_KEY);
+ if(known){
+ known = JSON.parse(known);
+ }
+ return known;
+}
diff --git a/assets/settings.js b/assets/settings.js
index cc404c0..ab5df50 100644
--- a/assets/settings.js
+++ b/assets/settings.js
@@ -1,5 +1,6 @@
import Banner from '../components/Banner.html';
import ArchiveForm from '../components/ArchiveForm.html';
+import {known_load, known_save} from './known_instances.js'
(function settings_init(){
if(!('fetch' in window)){
@@ -142,10 +143,12 @@ import ArchiveForm from '../components/ArchiveForm.html';
let last_viewer = {};
function update_viewer(viewer){
- if(last_viewer == JSON.stringify(viewer)){
+ let dumped = JSON.stringify(viewer);
+ if(last_viewer == dumped){
+ console.log('viewers is the same');
return;
}
- last_viewer = JSON.stringify(viewer);
+ last_viewer = dumped;
document.querySelector('#post-count').textContent = viewer.post_count;
document.querySelector('#eligible-estimate').textContent = viewer.eligible_for_delete_estimate;
@@ -163,7 +166,9 @@ import ArchiveForm from '../components/ArchiveForm.html';
banner.set(viewer);
}
- update_viewer(JSON.parse(document.querySelector('script[data-viewer]').textContent))
+ let viewer_from_dom = JSON.parse(document.querySelector('script[data-viewer]').textContent)
+
+ update_viewer(viewer_from_dom)
function set_viewer_timeout(){
setTimeout(() => fetch_viewer().then(update_viewer).then(set_viewer_timeout, set_viewer_timeout),
@@ -202,4 +207,31 @@ import ArchiveForm from '../components/ArchiveForm.html';
},
})
}
+
+ function bump_instance(instance_name){
+ let known_instances = known_load();
+ let found = false;
+ for(let instance of known_instances){
+ if(instance['instance'] == instance_name){
+ instance.hits ++;
+ found = true;
+ break;
+ }
+ }
+ if(!found){
+ let instance = {"instance": instance_name, "hits": 1};
+ known_instances.push(instance);
+ }
+
+ known_save(known_instances);
+
+ }
+
+ if(viewer_from_dom['service'] == 'mastodon' && location.hash == '#bump_instance'){
+ console.log('bumpin')
+ bump_instance(viewer_from_dom['id'].split('@')[1])
+ let url = new URL(location.href)
+ url.hash = '';
+ history.replaceState('', '', url);
+ }
})();
diff --git a/dodo.py b/dodo.py
index 7aa734f..75ceb08 100644
--- a/dodo.py
+++ b/dodo.py
@@ -120,6 +120,7 @@ def task_rollup():
for filename in filenames:
src = 'assets/{}'.format(filename)
dst = 'static/{}'.format(filename)
+ name = filename.split('.')[0]
yield dict(
name=filename,
file_dep=list(chain(
@@ -130,7 +131,7 @@ def task_rollup():
clean=True,
actions=[
['node_modules/.bin/rollup', '-c',
- '-i', src, '-o', dst, '-f', 'iife'],
+ '-i', src, '-o', dst, '-n', name, '-f', 'iife'],
],
)
diff --git a/routes/__init__.py b/routes/__init__.py
index 0304bc6..6323712 100644
--- a/routes/__init__.py
+++ b/routes/__init__.py
@@ -265,7 +265,7 @@ def mastodon_login_step2(instance_url):
g.viewer = session
- resp = redirect(url_for('index'))
+ resp = redirect(url_for('index', _anchor='bump_instance'))
return resp
diff --git a/templates/about.html b/templates/about.html
index 3766b43..eeab2db 100644
--- a/templates/about.html
+++ b/templates/about.html
@@ -74,7 +74,7 @@