proxy avatars

this fixes some potential issues when connecting to a non-https mastodon
server

you shouldn't connect to a non-https mastodon server in general but you
know, whatever

app.py

@@ -10,6 +10,7 @@ from lib.auth import get_viewer
 import os
 import mimetypes
 import lib.brotli
+import lib.img_proxy
 
 app = Flask(__name__)
 
@@ -82,7 +83,7 @@ limiter = Limiter(app, key_func=rate_limit_key)
 @app.after_request
 def install_security_headers(resp):
     csp = ("default-src 'none';"
-           "img-src 'self' https:;"
+           "img-src 'self';"
            "style-src 'self' 'unsafe-inline';"
            "frame-ancestors 'none';"
            )
@@ -113,3 +114,5 @@ def install_security_headers(resp):
 mimetypes.add_type('image/webp', '.webp')
 
 lib.brotli.brotli(app)
+
+imgproxy = lib.img_proxy.ImgProxyCache()

lib/brotli.py

@@ -14,7 +14,7 @@ class BrotliCache(object):
         self.expire = expire
         self.redis.client_setname('brotlicache')
 
-    def compress(self, cache_key, lock_key, body, mode=brotli_.MODE_GENERIC):
+    def compress_and_cache(self, cache_key, lock_key, body, mode=brotli_.MODE_GENERIC):
         encbody = brotli_.compress(body, mode=mode)
         self.redis.set(cache_key, encbody, px=int(self.expire*1000))
         self.redis.delete(lock_key)
@@ -38,7 +38,7 @@ class BrotliCache(object):
                     if response.content_type.startswith('text/')
                     else brotli_.MODE_GENERIC)
                 t = Thread(
-                    target=self.compress,
+                    target=self.compress_and_cache,
                     args=(cache_key, lock_key, body, mode))
                 t.start()
                 if self.timeout > 0:

lib/img_proxy.py

@@ -0,0 +1,49 @@
+import requests
+import threading
+import redis
+from flask import make_response, abort
+
+
+class ImgProxyCache(object):
+    def __init__(self, redis_uri='redis://', timeout=1, expire=60*60*6,
+                 prefix='img_proxy'):
+        self.redis = redis.StrictRedis.from_url(redis_uri)
+        self.timeout = timeout
+        self.expire = expire
+        self.prefix = prefix
+        self.redis.client_setname('img_proxy')
+
+    def key(self, keytype, url):
+        return '{prefix}:{keytype}:{url}'.format(
+                prefix=self.prefix,
+                keytype=keytype,
+                url=url)
+
+    def fetch_and_cache(self, url):
+        resp = requests.get(url)
+        if(resp.status_code != 200):
+            return
+        mime = resp.headers.get('content-type', 'application/octet-stream')
+        self.redis.set(self.key('mime', url), mime, px=self.expire)
+        self.redis.set(self.key('body', url), resp.content, px=self.expire)
+
+    def respond(self, url):
+        x_imgproxy_cache = 'HIT'
+        mime = self.redis.get(self.key('mime', url))
+        body = self.redis.get(self.key('body', url))
+        if not body or not mime:
+            x_imgproxy_cache = 'MISS'
+            if self.redis.set(
+                    self.key('lock', url), 1, nx=True, ex=10*self.timeout):
+                t = threading.Thread(target=self.fetch_and_cache, args=(url,))
+                t.start()
+                t.join(self.timeout)
+                mime = self.redis.get(self.key('mime', url))
+                body = self.redis.get(self.key('body', url))
+        if not body or not mime:
+            return abort(404)
+
+        resp = make_response(body, 200)
+        resp.headers.set('content-type', mime)
+        resp.headers.set('x-imgproxy-cache', x_imgproxy_cache)
+        return resp

lib/json.py

@@ -1,4 +1,5 @@
 from json import dumps
+from flask import url_for
 
 
 def account(acc):
@@ -13,7 +14,8 @@ def account(acc):
             eligible_for_delete_estimate=acc.estimate_eligible_for_delete(),
             display_name=acc.display_name,
             screen_name=acc.screen_name,
-            avatar_url=acc.avatar_url,
+            avatar_url=url_for('avatar', urlhash=acc.avatar_url_hash()),
+            avatar_url_orig=acc.avatar_url,
             id=acc.id,
             service=acc.service,
             policy_enabled=acc.policy_enabled,

model.py

@@ -3,6 +3,7 @@ from datetime import timedelta, datetime, timezone
 from app import db
 import secrets
 from lib.interval import decompose_interval
+import hashlib
 
 
 class TimestampMixin(object):
@@ -119,6 +120,11 @@ class Account(TimestampMixin, RemoteIDMixin):
     def touch_refresh(self):
         self.last_refresh = db.func.now()
 
+    def avatar_url_hash(self):
+        if not self.avatar_url:
+            return None
+        return hashlib.sha1(self.avatar_url.encode('utf-8')).hexdigest()
+
     @db.validates('policy_keep_younger', 'policy_delete_every')
     def validate_intervals(self, key, value):
         if not (value == timedelta(0) or value >= timedelta(minutes=1)):

routes/__init__.py

@@ -1,12 +1,12 @@
 from flask import render_template, url_for, redirect, request, g,\
-                  make_response
+                  make_response, abort
 from datetime import datetime, timedelta, timezone
 import lib.twitter
 import lib.mastodon
 from lib.auth import require_auth, csrf,\
                      get_viewer
 from model import Session, TwitterArchive, MastodonApp, MastodonInstance
-from app import app, db, sentry, limiter
+from app import app, db, sentry, limiter, imgproxy
 import tasks
 from zipfile import BadZipFile
 from twitter import TwitterError
@@ -288,3 +288,13 @@ def dismiss():
     get_viewer().reason = None
     db.session.commit()
     return redirect(url_for('index'))
+
+
+@app.route('/avatar/<urlhash>')
+@require_auth
+def avatar(urlhash):
+    viewer = get_viewer()
+    if (not viewer.avatar_url or not urlhash == viewer.avatar_url_hash()):
+        return abort(404)
+
+    return imgproxy.respond(viewer.avatar_url)

templates/logged_in.html

@@ -8,7 +8,7 @@
 
 <section class=viewer>
 <p>Hello,
-    <img class=avatar id=avatar src="{{g.viewer.account.avatar_url}}"/>
+    <img class=avatar id=avatar src="{{url_for('avatar', urlhash=g.viewer.account.avatar_url_hash())}}"/>
     <span id='display-name' title='@{{g.viewer.account.screen_name}}'>{{g.viewer.account.display_name or g.viewer.account.screen_name}}</span>!
     <a href="{{url_for('logout')}}">Log out</a>
 </p>