Delete dependabot.yml

it's dirty but it's needed for versioneer

I will figure out a better way to do this later

.codacy.yml

@@ -1 +0,0 @@
-exclude_paths: ['version.py', 'versioneer.py']

.dockerignore

@@ -0,0 +1,22 @@
+.envrc.example
+.gitignore
+.tool-versions
+*.md
+Dockerfile
+docker-compose.yml
+#.git
+.github
+.codecov.yml
+.coveragerc
+.env
+.eslintrc.yml
+.gitattributes
+LICENSE
+CHANGELOG.markdown
+README.markdown
+config.example.py
+config.docker.py
+forget.example.service
+requirements-dev.txt
+data
+config.py

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: codl
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/docker-publish.yml

@@ -0,0 +1,102 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  schedule:
+    - cron: '18 10 * * *'
+  push:
+    branches: [ "master" ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ "master" ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@7e0881f8fe90b25e305bbf0309761e9314607e25
+        with:
+          cosign-release: 'v1.9.0'
+
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          COSIGN_EXPERIMENTAL: "true"
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}

.github/workflows/run-tests.yml

@@ -0,0 +1,39 @@
+name: Run tests on pushes and on PRs
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  run_tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.6"
+          cache: 'pipenv'
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "10"
+          cache: 'npm'
+
+      - name: Install pipenv and python dependencies
+        run: |
+          pip install -U pip pipenv
+          pipenv sync -d
+      - name: Install node dependencies
+        run: npm install
+
+      - name: Start Redis
+        run: docker run --name redis --publish 6379:6379 --detach redis
+
+
+      - name: Run tests with pytest
+        run: pipenv run pytest --cov=.
+
+      - uses: codecov/codecov-action@v2

.gitignore

@@ -8,3 +8,8 @@ static/*
 .cache/
 .coverage
 .pytest_cache
+
+data/*
+!data/.keep
+
+docker-compose.override.yml

.travis.yml

@@ -1,19 +0,0 @@
-dist: xenial
-sudo: true
-language: python
-python:
-    - 3.6
-    - 3.7
-    - 3.7-dev
-install:
-    - pip install -r requirements.txt -r requirements-dev.txt
-    - npm install
-script:
-    - pytest --cov=.
-after_success:
-    - codecov
-cache:
-    pip: true
-    directories:
-        - node_modules
-

CHANGELOG.markdown

@@ -1,3 +1,27 @@
+## v2.2.0
+
+Released 2022-08-10
+
+* add: instance hidelist
+  <https://github.com/codl/forget/pull/590>
+* add: docker deployment support (Thanks @shibaobun !)
+  <https://github.com/codl/forget/pull/612>
+* removed: migration path for known instances list from cookie to localstorage
+  <https://github.com/codl/forget/pull/545>
+
+## v2.1.0
+
+Released 2022-03-04
+
+* add: Misskey support (Thanks @Johann150 !)
+  <https://github.com/codl/forget/pull/544>
+* fix: lowered database impact of a background task
+  <https://github.com/codl/forget/issue/166>
+* fix: wording on "favourited posts" is unclear
+  <https://github.com/codl/forget/pull/366>
+* fix: failing to fetch some posts in some circumstances
+  <https://github.com/codl/forget/issue/584>
+
 ## v2.0.0
 
 Released 2019-09-13

Dockerfile

@@ -0,0 +1,51 @@
+# syntax=docker/dockerfile:1.4.2
+
+FROM python:3.10.6-bullseye AS pydeps
+WORKDIR /usr/src/app
+
+RUN --mount=type=cache,target=/root/.cache/pip/http pip install --upgrade pip==22.2.2
+
+COPY requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/pip/http pip install -r requirements.txt
+
+
+FROM pydeps AS pynodedeps
+
+RUN rm -f /etc/apt/apt.conf.d/docker-clean &&\
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' \
+    > /etc/apt/apt.conf.d/keep-cache
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt-get update -qq && apt-get install -qq nodejs npm
+
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,target=/root/.npm npm clean-install
+
+
+FROM scratch AS layer-cake
+WORKDIR /
+
+COPY *.py setup.cfg rollup.config.js ./
+COPY assets assets
+COPY components components
+COPY libforget libforget
+COPY migrations migrations
+COPY routes routes
+COPY static static
+COPY templates templates
+
+
+FROM pynodedeps AS build
+
+COPY --from=layer-cake / ./
+RUN doit
+
+FROM pydeps
+
+COPY --from=build /usr/src/app ./
+
+COPY .git/ .git/
+
+ENV FLASK_APP=forget.py
+
+VOLUME ["/var/run/celery"]

Pipfile

@@ -9,10 +9,11 @@ name = "pypi"
 
 alembic = "*"
 brotli = ">=1.0.1"
+#celery = "~=4.4.2"
 celery = "*"
 csscompressor = "*"
 doit = "*"
-flask = "~=1.1"
+flask = ">=1.1"
 flask-migrate = "*"
 flask-sqlalchemy = "*"
 gunicorn = ">=19.8"
@@ -24,7 +25,7 @@ redis = "*"
 requests = "*"
 sqlalchemy = "*"
 twitter = "*"
-"mastodon.py" = "~=1.2"
+"mastodon.py" = ">=1.2"
 blinker = "*"
 
 
@@ -34,5 +35,4 @@ coverage = "*"
 codecov = "*"
 pytest = "*"
 pytest-cov = "*"
-pytest-redis = "*"
 versioneer = "*"

README.markdown

@@ -1,16 +1,26 @@
-[![Forget](assets/promo.gif)](https://forget.codl.fr)
+![Forget](assets/promo.gif)
 
-![User count](https://forget.codl.fr/api/badge/users)
-![Maintenance status](https://img.shields.io/maintenance/yes/2019.svg)
+![Maintenance status](https://img.shields.io/maintenance/no/2022.svg)
 
 [![Build status](https://img.shields.io/travis/codl/forget.svg)](https://travis-ci.org/codl/forget/)
 [![Test coverage](https://img.shields.io/codecov/c/github/codl/forget.svg)](https://codecov.io/gh/codl/forget)
-[![Code quality](https://img.shields.io/codacy/grade/1780ac6071c04cbd9ccf75de0891e798.svg)](https://www.codacy.com/app/codl/forget?utm_source=github.com&utm_medium=referral&utm_content=codl/forget&utm_campaign=badger)
 
-Forget is a post deleting service for Twitter and Mastodon. It lives at
-<https://forget.codl.fr>.
+Forget is a post deleting service for Twitter, Mastodon, and Misskey.
 
 
+## Features
+
+  * Delete your posts when they cross an age threshold.
+  * Or keep your post count in check, deleting old posts when you go over.
+  * Preserve old posts that matter by giving them a favourite or a reaction.
+  * Set it and <i>forget</i> it. Forget works continuously in the background.
+
+## Non-features
+
+Forget is not a one-time purging tool. It is designed to prune your account
+continuously, not quickly. If you need a lot of posts gone fast, you may want
+to look for another more-suited tool.
+
 ## Running your own
 
 ### Requirements
@@ -18,7 +28,7 @@ Forget is a post deleting service for Twitter and Mastodon. It lives at
 * Postgresql
 * Redis
 * Python 3.6+
-* Yarn or NPM
+* Node.js 10+
 
 
 ### Set up venv
@@ -85,7 +95,6 @@ $ honcho start
 
 The application server will listen on `http://127.0.0.1:42157`.
 You'll want to use your favourite web server to proxy traffic to it.
-<small>This author suggests Caddy.</small>
 
 ### Development
 
@@ -103,9 +112,39 @@ You can run the (currently very incomplete) test suite by running `pytest`.
 You'll need redis installed on your development machine, a temporary redis
 server will be started and shut down automatically by the test suite.
 
----
+## Docker
+
+This project is also able to be deployed through Docker.
+
+1. Copy `config.docker.py` to `config.py` and add additional configurations to
+   your liking.
+1. By default, the webapp container will be listening on `127.0.0.1:42157`,
+   which you can point a reverse proxy at.
+   * If your reverse proxy is in another docker network then you'll need a
+     `docker-compose.override.yml` file to attach the `www` service to the
+     right network and not publish any ports. An example override file is
+     provided. The web app will be listening on `http://forget-www-1:42157`.
+1. By default, the `docker-compose.yml` creates relative mounts `./redis`,
+   `./postgres`, and `./celery` relative to the `docker-compose.yml` location.
+   An example `docker-compose.override.yml` file is provided that shows how to
+   change this.
+1. Run `docker-compose build` to build the image.
+1. Run `docker-compose up` to start or `docker-compose up -d` to start in the
+   background, and use `docker-compose down` to stop.
+
+## Contact
 
 If you're having trouble with Forget, or if you're not having trouble but you
 just want to tell me you like it, you can drop me a note at
 [@codl@chitter.xyz](https://chitter.xyz/@codl) or
-[codl@codl.fr](mailto:codl@codl.fr). Thanks for reading this readme.
+[codl@codl.fr](mailto:codl@codl.fr).
+
+## Greetz
+
+Thank you bea, for making ephemeral, inspiring me to make [limiter][], then this,
+in an attempt to bring ephemeral with me everywhere. ☕
+
+[limiter]: https://github.com/codl/limiter
+
+Thank you to the kind folks who have emailed me to tell me Forget has made their
+time on social media less stressful. 🌻

app.py

@@ -7,6 +7,7 @@ from libforget.cachebust import cachebust
 import mimetypes
 import libforget.brotli
 import libforget.img_proxy
+from werkzeug.middleware.proxy_fix import ProxyFix
 
 app = Flask(__name__)
 
@@ -94,3 +95,5 @@ libforget.brotli.brotli(app)
 
 imgproxy = (
     libforget.img_proxy.ImgProxyCache(redis_uri=app.config.get('REDIS_URI')))
+
+app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)

assets/instance_buttons.js

@@ -2,38 +2,26 @@ import {SLOTS, normalize_known, known_load, known_save} from './known_instances.
 
 (function instance_buttons(){
 
-    const container = document.querySelector('#mastodon_instance_buttons');
-    const button_template = Function('first', 'instance',
-        'return `' + document.querySelector('#instance_button_template').innerHTML + '`;');
-    const another_button_template = Function(
+    const mastodon_container = document.querySelector('#mastodon_instance_buttons');
+    const mastodon_button_template = Function('first', 'instance',
+        'return `' + document.querySelector('#mastodon_instance_button_template').innerHTML + '`;');
+    const mastodon_another_button_template = Function(
         'return `' +
-            document.querySelector('#another_instance_button_template').innerHTML + '`;');
-    const top_instances =
-        Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')();
-
-    async function get_known(){
-        let known = known_load();
-        if(!known){
-            let resp = await fetch('/api/known_instances');
-            if(resp.ok && resp.headers.get('content-type') == 'application/json'){
-                known = await resp.json();
-            }
-            else {
-                known = [{
-                    "instance": "mastodon.social",
-                    "hits": 0
-                }];
-            }
-            known_save(known)
-            fetch('/api/known_instances', {method: 'DELETE'})
-        }
-
-        return known;
-    }
+            document.querySelector('#mastodon_another_instance_button_template').innerHTML + '`;');
+    const mastodon_top_instances =
+        Function('return JSON.parse(`' + document.querySelector('#mastodon_top_instances').innerHTML + '`);')();
 
+    const misskey_container = document.querySelector('#misskey_instance_buttons');
+    const misskey_button_template = Function('first', 'instance',
+        'return `' + document.querySelector('#misskey_instance_button_template').innerHTML + '`;');
+    const misskey_another_button_template = Function(
+        'return `' +
+            document.querySelector('#misskey_another_instance_button_template').innerHTML + '`;');
+    const misskey_top_instances =
+        Function('return JSON.parse(`' + document.querySelector('#misskey_top_instances').innerHTML + '`);')();
 
     async function replace_buttons(){
-        let known = await get_known();
+        let known = known_load();
 
         known = normalize_known(known);
         known_save(known);
@@ -41,7 +29,7 @@ import {SLOTS, normalize_known, known_load, known_save} from './known_instances.
         let filtered_top_instances = []
         for(let instance of top_instances){
             let found = false;
-            for(let k of known){
+            for(let k of known_instances){
                 if(k['instance'] == instance['instance']){
                     found = true;
                     break;
@@ -52,20 +40,35 @@ import {SLOTS, normalize_known, known_load, known_save} from './known_instances.
             }
         }
 
-        let instances = known.concat(filtered_top_instances).slice(0, SLOTS);
+        let instances = known_instances.concat(filtered_top_instances).slice(0, SLOTS);
 
         let html = '';
 
         let first = true;
         for(let instance of instances){
-            html += button_template(first, instance['instance'])
+            html += template(first, instance['instance'])
             first = false;
         }
 
-        html += another_button_template();
+        html += template_another_instance();
 
         container.innerHTML = html;
     }
 
-    replace_buttons();
+    async function init_buttons(){
+        let known = await get_known();
+
+        known.mastodon = normalize_known(known.mastodon);
+        known.misskey = normalize_known(known.misskey);
+        known_save(known);
+
+        replace_buttons(mastodon_top_instances, known.mastodon,
+            mastodon_container, mastodon_button_template,
+            mastodon_another_button_template);
+        replace_buttons(misskey_top_instances, known.misskey,
+            misskey_container, misskey_button_template,
+            misskey_another_button_template);
+    }
+
+    init_buttons();
 })();

assets/known_instances.js

@@ -1,16 +1,44 @@
-const STORAGE_KEY = 'forget_known_instances';
+const STORAGE_KEY = 'forget_known_instances@2021-12-09';
 export const SLOTS = 5;
 
+function load_and_migrate_old(){
+    const OLD_KEY = "forget_known_instances";
+    let olddata = localStorage.getItem(OLD_KEY);
+    if(olddata != null){
+        olddata = JSON.parse(olddata)
+        let newdata = {
+            mastodon: olddata,
+            misskey: [{
+                "instance": "misskey.io",
+                "hits": 0
+            }]
+        };
+        known_save(newdata);
+        localStorage.removeItem(OLD_KEY);
+        return newdata;
+    }
+}
+
 export function known_save(known){
     localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
 }
 
 export function known_load(){
+    const default_ = {
+        mastodon:[{ "instance": "mastodon.social", "hits": 0 }],
+        misskey:[{ "instance": "misskey.io", "hits": 0 }],
+    };
+    // this makes mastodon.social and misskey.io show up on respective first
+    // buttons by default even if they are not the most popular instance
+    // according to the server
+
     let known = localStorage.getItem(STORAGE_KEY);
     if(known){
         known = JSON.parse(known);
+    } else {
+        known = load_and_migrate_old();
     }
-    return known;
+    return known || default_;
 }
 
 export function normalize_known(known){

assets/settings.js

@@ -194,10 +194,10 @@ import {known_load, known_save} from './known_instances.js'
         })
     }
 
-    function bump_instance(instance_name){
+    function bump_instance(service, instance_name){
         let known_instances = known_load();
         let found = false;
-        for(let instance of known_instances){
+        for(let instance of known_instances[service]){
             if(instance['instance'] == instance_name){
                 instance.hits ++;
                 found = true;
@@ -206,15 +206,17 @@ import {known_load, known_save} from './known_instances.js'
         }
         if(!found){
             let instance = {"instance": instance_name, "hits": 1};
-            known_instances.push(instance);
+            known_instances[service].push(instance);
         }
 
         known_save(known_instances);
 
     }
 
-    if(viewer_from_dom['service'] == 'mastodon' && location.hash == '#bump_instance'){
-        bump_instance(viewer_from_dom['id'].split('@')[1])
+    if(location.hash == '#bump_instance' && (
+        viewer_from_dom['service'] == 'mastodon' || viewer_from_dom['service'] == 'misskey'
+    )){
+        bump_instance(viewer_from_dom['service'], viewer_from_dom['id'].split('@')[1])
         let url = new URL(location.href)
         url.hash = '';
         history.replaceState('', '', url);

assets/styles.css

@@ -9,7 +9,7 @@ body {
 }
 
 body > section, body > header, body > footer {
-    max-width: 40rem;
+    max-width: 45rem;
     margin-left: auto;
     margin-right: auto;
 }
@@ -235,6 +235,10 @@ button {
     background-color: #282c37;
 }
 
+.btn.primary.misskey-colored {
+    background-color: #66b300;
+}
+
 .btn.secondary {
     background-color: rgba(255,255,255,0.5);
     box-shadow: inset 0 0 0 1px rgba(0,0,0,0.3);

config.docker.py

@@ -0,0 +1,76 @@
+"""
+this is an example config file for Forget
+
+copy this file to config.py before editing
+
+lines starting with # demonstrate default or example values
+the # should be removed before editing
+"""
+
+"""
+DATABASE URI
+
+determines where to connect to the database
+see <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls> for syntax
+only postgresql with psycopg2 driver is officially supported
+"""
+SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://postgres:postgres@db/forget'
+
+"""
+REDIS URI
+
+see <https://redis-py.readthedocs.io/en/latest/#redis.ConnectionPool.from_url>
+for syntax reference
+"""
+REDIS_URI='redis://redis'
+
+"""
+SERVER ADDRESS
+
+This is the address at which forget will be reached.
+External services will redirect to this address when logging in.
+"""
+# SERVER_NAME="0.0.0.0:5000"
+# HTTPS=True
+
+"""
+TWITTER CREDENTIALS
+
+Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
+When prompted for it, your callback URL is {SERVER_NAME}/login/twitter/callback
+"""
+# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
+# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
+
+"""
+SENTRY
+
+If you want to send exceptions to sentry, enter your sentry DSN here
+"""
+# SENTRY_DSN=''
+
+"""
+HIDDEN INSTANCES
+
+The front page shows one-click login buttons for the mastodon and
+misskey instances that see the most heavy use. Instances configured in this
+list will be prevented from appearing in these buttons.
+
+They will still appear if a user has previously logged into them and their
+browser remembers it. A user will still be able to log into them by manually
+typing the address into the log in form.
+
+This is a space-delimited list. Example syntax:
+HIDDEN_INSTANCES='social.example.com pleroma.example.net mk.example.org'
+"""
+# HIDDEN_INSTANCES=''
+
+"""
+ADVANCED FLASK CONFIG
+
+you can also use any config variable that flask expects here
+A list of these config variables is available here:
+<http://flask.pocoo.org/docs/1.0/config/#builtin-configuration-values>
+"""
+# SESSION_COOKIE_SECURE=True
+# DEBUG=True

config.example.py

@@ -2,6 +2,9 @@
 this is an example config file for Forget
 
 copy this file to config.py before editing
+
+lines starting with # demonstrate default or example values
+the # should be removed before editing
 """
 
 """
@@ -21,15 +24,6 @@ for syntax reference
 """
 # REDIS_URI='redis://'
 
-"""
-TWITTER CREDENTIALS
-
-Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
-Twitter locked me out of it so I can't guide you more than that. Sorry.
-"""
-# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
-# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
-
 """
 SERVER ADDRESS
 
@@ -39,13 +33,37 @@ External services will redirect to this address when logging in.
 # SERVER_NAME="localhost:5000"
 # HTTPS=True
 
+"""
+TWITTER CREDENTIALS
+
+Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
+When prompted for it, your callback URL is {SERVER_NAME}/login/twitter/callback
+"""
+# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
+# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
 
 """
 SENTRY
 
 If you want to send exceptions to sentry, enter your sentry DSN here
 """
-# SENTRY_DSN=
+# SENTRY_DSN=''
+
+"""
+HIDDEN INSTANCES
+
+The front page shows one-click login buttons for the mastodon and
+misskey instances that see the most heavy use. Instances configured in this
+list will be prevented from appearing in these buttons.
+
+They will still appear if a user has previously logged into them and their
+browser remembers it. A user will still be able to log into them by manually
+typing the address into the log in form.
+
+This is a space-delimited list. Example syntax:
+HIDDEN_INSTANCES='social.example.com pleroma.example.net mk.example.org'
+"""
+# HIDDEN_INSTANCES=''
 
 """
 ADVANCED FLASK CONFIG

docker-compose.override.example.yml

@@ -0,0 +1,33 @@
+## uncomment below and change network name to use a reverse proxy on an
+## existing network
+#networks:
+#  mycoolreverseproxynetwork:
+#    external: true
+
+services:
+  www:
+## uncomment below to stop listening on 127.0.0.1
+#    ports: []
+
+    networks:
+      - forget
+## uncomment below and change network name to use a reverse proxy on an
+## existing network
+#      - mycoolreverseproxynetwork
+
+
+## if you wish to change where postgres, redis, and celery persistent data is
+## stored, uncomment below and change the first part of each volume (before the `:`)
+
+#  redis:
+#    volumes:
+#      - /my/cool/redis/path:/data
+#  db:
+#    volumes:
+#      - /my/cool/postgres/path:/var/lib/postgresql/data
+#  worker:
+#    volumes:
+#      - /my/cool/celery/path:/var/run/celery
+#  beat:
+#    volumes:
+#      - /my/cool/celery/path:/var/run/celery

docker-compose.yml

@@ -0,0 +1,103 @@
+services:
+  www:
+    build:
+      context: ./
+    image: ghcr.io/codl/forget
+    pull_policy: missing
+    restart: always
+    volumes:
+      - type: bind
+        source: ./config.py
+        target: /usr/src/app/config.py
+        read_only: true
+    depends_on:
+      - redis
+      - db
+      - worker
+      - beat
+    command: bash -c "
+        flask db upgrade &&
+        gunicorn -w 9 -t 3600 -b 0.0.0.0:42157 forget:app
+      "
+    networks:
+      - forget
+    expose:
+      - 42157
+    ports:
+      - "127.0.0.1:42157:42157"
+
+  worker:
+    build:
+      context: ./
+    image: ghcr.io/codl/forget
+    pull_policy: missing
+    restart: always
+    volumes:
+      - type: bind
+        source: ./config.py
+        target: /usr/src/app/config.py
+        read_only: true
+      - ./data/celery/run:/var/run/celery
+    depends_on:
+      - redis
+      - db
+    networks:
+      - forget
+    command: bash -c "
+        mkdir -p /var/run/celery &&
+        chown -R nobody:nogroup /var/run/celery &&
+        exec celery --app=tasks worker
+                    --loglevel=INFO
+                    --statedb=/var/run/celery/worker.state
+                    --hostname=worker
+                    --uid=nobody --gid=nogroup
+      "
+
+  beat:
+    build:
+      context: ./
+    image: ghcr.io/codl/forget
+    pull_policy: missing
+    restart: always
+    volumes:
+      - type: bind
+        source: ./config.py
+        target: /usr/src/app/config.py
+        read_only: true
+      - ./data/celery/run:/var/run/celery
+    depends_on:
+      - redis
+      - db
+    networks:
+      - forget
+    command: bash -c "
+        mkdir -p /var/run/celery &&
+        chown -R nobody:nogroup /var/run/celery &&
+        exec celery --app=tasks beat
+                    --loglevel=INFO
+                    --schedule=/var/run/celery/schedule
+                    --uid=nobody --gid=nogroup
+      "
+
+  redis:
+    image: redis:4.0-alpine
+    restart: always
+    volumes:
+      - ./data/redis:/data
+    networks:
+      - forget
+
+  db:
+    image: postgres:14-alpine
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=forget
+    volumes:
+      - ./data/postgres:/var/lib/postgresql/data
+    networks:
+      - forget
+
+networks:
+  forget:

dodo.py

@@ -60,7 +60,7 @@ def task_service_icon():
     formats = ('webp', 'png')
     for width in widths:
         for image_format in formats:
-            for basename in ('twitter', 'mastodon'):
+            for basename in ('twitter', 'mastodon', 'misskey'):
                 yield dict(
                     name='{}-{}.{}'.format(basename, width, image_format),
                     actions=[(resize_image, (basename, width, image_format))],

libforget/brotli.py

@@ -5,6 +5,7 @@ from hashlib import sha256
 import redis as libredis
 import os.path
 import mimetypes
+from redis.exceptions import RedisError
 
 
 class BrotliCache(object):
@@ -34,32 +35,35 @@ class BrotliCache(object):
         digest = sha256(body).hexdigest()
         cache_key = 'brotlicache:{}'.format(digest)
 
-        encbody = self.redis.get(cache_key)
-        response.headers.set('brotli-cache', 'HIT')
-        if not encbody:
-            response.headers.set('brotli-cache', 'MISS')
-            lock_key = 'brotlicache:lock:{}'.format(digest)
-            if self.redis.set(lock_key, 1, nx=True, ex=10):
-                mode = (
-                    brotli_.MODE_TEXT
-                    if response.content_type.startswith('text/')
-                    else brotli_.MODE_GENERIC)
-                t = Thread(
-                    target=self.compress_and_cache,
-                    args=(cache_key, lock_key, body, mode))
-                t.start()
-                if self.timeout > 0:
-                    t.join(self.timeout)
-                    encbody = self.redis.get(cache_key)
-                if not encbody:
-                    response.headers.set('brotli-cache', 'TIMEOUT')
-            else:
-                response.headers.set('brotli-cache', 'LOCKED')
-        if encbody:
-            response.headers.set('content-encoding', 'br')
-            response.headers.set('vary', 'accept-encoding')
-            response.set_data(encbody)
-            return response
+        try:
+            encbody = self.redis.get(cache_key)
+            response.headers.set('brotli-cache', 'HIT')
+            if not encbody:
+                response.headers.set('brotli-cache', 'MISS')
+                lock_key = 'brotlicache:lock:{}'.format(digest)
+                if self.redis.set(lock_key, 1, nx=True, ex=10):
+                    mode = (
+                        brotli_.MODE_TEXT
+                        if response.content_type.startswith('text/')
+                        else brotli_.MODE_GENERIC)
+                    t = Thread(
+                        target=self.compress_and_cache,
+                        args=(cache_key, lock_key, body, mode))
+                    t.start()
+                    if self.timeout > 0:
+                        t.join(self.timeout)
+                        encbody = self.redis.get(cache_key)
+                    if not encbody:
+                        response.headers.set('brotli-cache', 'TIMEOUT')
+                else:
+                    response.headers.set('brotli-cache', 'LOCKED')
+            if encbody:
+                response.headers.set('content-encoding', 'br')
+                response.headers.set('vary', 'accept-encoding')
+                response.set_data(encbody)
+                return response
+        except RedisError:
+            response.headers.set('brotli-cache', 'ERROR')
 
         return response
 

libforget/img_proxy.py

@@ -65,7 +65,7 @@ class ImgProxyCache(object):
         if(resp.status_code != 200):
             return
 
-        header_whitelist = [
+        allowed_headers = [
                 'content-type',
                 'cache-control',
                 'etag',
@@ -81,7 +81,7 @@ class ImgProxyCache(object):
                 if match:
                     expire = max(self.expire, int(match.group(1)))
 
-        for key in header_whitelist:
+        for key in allowed_headers:
             if key in resp.headers:
                 headers[key] = resp.headers[key]
         self.redis.set(self.key('headers', url), pickle.dumps(headers, -1),

libforget/mastodon.py

@@ -145,8 +145,6 @@ def post_from_api_object(obj, instance):
             created_at=obj['created_at'],
             author_id=account_from_api_object(obj['account'], instance).id,
             direct=obj['visibility'] == 'direct',
-            favourites=obj['favourites_count'],
-            reblogs=obj['reblogs_count'],
             is_reblog=obj['reblog'] is not None,
         )
 
@@ -197,11 +195,11 @@ def delete(post):
         raise TemporaryError(e)
 
 
-def suggested_instances(limit=5, min_popularity=5, blacklist=tuple()):
+def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
     return tuple((ins.instance for ins in (
             MastodonInstance.query
             .filter(MastodonInstance.popularity > min_popularity)
-            .filter(~MastodonInstance.instance.in_(blacklist))
+            .filter(~MastodonInstance.instance.in_(blocklist))
             .order_by(db.desc(MastodonInstance.popularity),
                       MastodonInstance.instance)
             .limit(limit).all())))

libforget/misskey.py

@@ -0,0 +1,191 @@
+from app import db, sentry
+from model import MisskeyApp, MisskeyInstance, Account, OAuthToken, Post
+from uuid import uuid4
+from hashlib import sha256
+from libforget.exceptions import TemporaryError, PermanentError
+from libforget.session import make_session
+
+def get_or_create_app(instance_url, callback, website, session):
+    instance_url = instance_url
+    app = MisskeyApp.query.get(instance_url)
+    
+    if not app:
+        # check if the instance uses https while getting instance infos
+        try:
+            r = session.post('https://{}/api/meta'.format(instance_url))
+            r.raise_for_status()
+            proto = 'https'
+        except Exception:
+            r = session.post('http://{}/api/meta'.format(instance_url))
+            r.raise_for_status()
+            proto = 'http'
+        
+        # This is using the legacy authentication method, because the newer
+        # Miauth method breaks the ability to log out and log back into forget.
+        
+        app = MisskeyApp()
+        app.instance = instance_url
+        app.protocol = proto
+        
+        # register the app
+        r = session.post('{}://{}/api/app/create'.format(app.protocol, app.instance), json = {
+            'name': 'forget',
+            'description': website,
+            'permission': ['write:notes', 'read:reactions'],
+            'callbackUrl': callback
+        })
+        r.raise_for_status()
+        app.client_secret = r.json()['secret']
+
+    return app
+
+def login_url(app, callback, session):
+    # will use the callback we gave the server in `get_or_create_app`
+    r = session.post('{}://{}/api/auth/session/generate'.format(app.protocol, app.instance), json = {
+        'appSecret': app.client_secret
+    })
+    r.raise_for_status()
+    # we already get the retrieval token here, but we get it again later so
+    # we do not have to store it
+    return r.json()['url']
+
+def receive_token(token, app):
+    session = make_session()
+
+    r = session.post('{}://{}/api/auth/session/userkey'.format(app.protocol, app.instance), json = {
+        'appSecret': app.client_secret,
+        'token': token
+    })
+    r.raise_for_status()
+
+    token = sha256(r.json()['accessToken'].encode('utf-8') + app.client_secret.encode('utf-8')).hexdigest()
+
+    acc = account_from_user(r.json()['user'], app.instance)
+    acc = db.session.merge(acc)
+    token = OAuthToken(token = token)
+    token = db.session.merge(token)
+    token.account = acc
+
+    return token
+
+def check_auth(account, app, session):
+    # there is no explicit check, we can only try getting user info
+    r = session.post('{}://{}/api/i'.format(app.protocol, app.instance), json = {'i': account.tokens[0].token})
+
+    if r.status_code != 200:
+        raise TemporaryError("{} {}".format(r.status_code, r.text))
+
+    if r.json()['isSuspended']:
+        # this is technically a temporary error, but like for twitter
+        # its handled as permanent to not make useless API calls
+        raise PermanentError("Misskey account suspended")
+
+def account_from_user(user, host):
+    return Account(
+            # in objects that get returned from misskey, the local host is
+            # set to None
+            misskey_instance=host,
+            misskey_id=user['id'],
+            screen_name='{}@{}'.format(user['username'], host),
+            display_name=user['name'],
+            avatar_url=user['avatarUrl'],
+            # the notes count is not always included, especially not when
+            # fetching posts. in that case assume its not needed
+            reported_post_count=user.get('notesCount', None),
+        )
+
+def post_from_api_object(obj, host):
+    return Post(
+            # in objects that get returned from misskey, the local host is
+            # set to None
+            misskey_instance=host,
+            misskey_id=obj['id'],
+            favourite=('myReaction' in obj
+                       and bool(obj['myReaction'])),
+            has_media=('fileIds' in obj
+                       and bool(obj['fileIds'])),
+            created_at=obj['createdAt'],
+            author_id=account_from_user(obj['user'], host).id,
+            direct=obj['visibility'] == 'specified',
+            is_reblog=obj['renoteId'] is not None,
+        )
+
+def fetch_posts(acc, max_id, since_id):
+    app = MisskeyApp.query.get(acc.misskey_instance)
+    session = make_session()
+    check_auth(acc, app, session)
+
+    kwargs = dict(
+        limit=100,
+        userId=acc.misskey_id,
+        # for some reason the token is needed so misskey also sends `myReaction`
+        i=acc.tokens[0].token
+    )
+    if max_id:
+        kwargs['untilId'] = max_id
+    if since_id:
+        kwargs['sinceId'] = since_id
+
+    notes = session.post('{}://{}/api/users/notes'.format(app.protocol, app.instance), json=kwargs)
+
+    if notes.status_code != 200:
+        raise TemporaryError('{} {}'.format(notes.status_code, notes.text))
+
+    return [post_from_api_object(note, app.instance) for note in notes.json()]
+
+
+def refresh_posts(posts):
+    acc = posts[0].author
+    app = MisskeyApp.query.get(acc.misskey_instance)
+    session = make_session()
+    check_auth(acc, app, session)
+
+    new_posts = list()
+    with db.session.no_autoflush:
+        for post in posts:
+            print('Refreshing {}'.format(post))
+            r = session.post('{}://{}/api/notes/show'.format(app.protocol, app.instance), json={
+                'i': acc.tokens[0].token,
+                'noteId': post.misskey_id
+            })
+            if r.status_code != 200:
+                try:
+                    if r.json()['error']['code'] == 'NO_SUCH_NOTE':
+                        db.session.delete(post)
+                        continue
+                except Exception as e:
+                    raise TemporaryError(e)
+                raise TemporaryError('{} {}'.format(r.status_code, r.text))
+
+            new_post = db.session.merge(post_from_api_object(r.json(), app.instance))
+            new_post.touch()
+            new_posts.append(new_post)
+    return new_posts
+
+def delete(post):
+    acc = post.author
+    app = MisskeyApp.query.get(post.misskey_instance)
+    session = make_session()
+    if not app:
+        # how? if this happens, it doesnt make sense to repeat it,
+        # so use a permanent error
+        raise PermanentError("instance not registered for delete")
+
+    r = session.post('{}://{}/api/notes/delete'.format(app.protocol, app.instance), json = {
+        'i': acc.tokens[0].token,
+        'noteId': post.misskey_id
+    })
+    
+    if r.status_code != 204:
+        raise TemporaryError("{} {}".format(r.status_code, r.text))
+
+    db.session.delete(post)
+
+def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
+    return tuple((ins.instance for ins in (
+            MisskeyInstance.query
+            .filter(MisskeyInstance.popularity > min_popularity)
+            .filter(~MisskeyInstance.instance.in_(blocklist))
+            .order_by(db.desc(MisskeyInstance.popularity),
+                      MisskeyInstance.instance)
+            .limit(limit).all())))

libforget/twitter.py

@@ -115,10 +115,6 @@ def post_from_api_tweet_object(tweet, post=None):
     if 'entities' in tweet:
         post.has_media = bool(
                 'media' in tweet['entities'] and tweet['entities']['media'])
-    if 'favorite_count' in tweet:
-        post.favourites = tweet['favorite_count']
-    if 'retweet_count' in tweet:
-        post.reblogs = tweet['retweet_count']
     post.is_reblog = 'retweeted_status' in tweet
     return post
 

migrations/versions/740fe24a7712_add_misskey.py

@@ -0,0 +1,48 @@
+"""add misskey
+
+Revision ID: 740fe24a7712
+Revises: af763dccc0b4
+Create Date: 2021-11-10 00:13:37.344364
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '740fe24a7712'
+down_revision = 'af763dccc0b4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('misskey_instances',
+    sa.Column('instance', sa.String(), nullable=False),
+    sa.Column('popularity', sa.Float(), server_default='10', nullable=False),
+    sa.PrimaryKeyConstraint('instance', name=op.f('pk_misskey_instances'))
+    )
+    op.execute("""
+        INSERT INTO misskey_instances (instance, popularity) VALUES
+            ('misskey.io', 100),
+            ('cliq.social', 60),
+            ('misskey.dev', 50),
+            ('quietplace.xyz', 40),
+            ('mk.nixnet.social', 30),
+            ('jigglypuff.club', 20);
+    """)
+
+    op.create_table('misskey_apps',
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('instance', sa.String(), nullable=False),
+    sa.Column('client_secret', sa.String(), nullable=False),
+    sa.Column('protocol', sa.Enum('http', 'https', name='enum_protocol_misskey'), nullable=False),
+    sa.PrimaryKeyConstraint('instance', name=op.f('pk_misskey_apps'))
+    )
+
+
+def downgrade():
+    op.drop_table('misskey_instances')
+    op.drop_table('misskey_apps')
+    op.execute('DROP TYPE enum_protocol_misskey;')

migrations/versions/7b0e9b8e0887_remove_fetch_batch_fk.py

@@ -0,0 +1,64 @@
+"""remove fetch batch fk
+
+things are real bad if the associated post is deleted and this is nulled
+keeping an opaque ID and associated date should work fine
+see GH-584
+
+Revision ID: 7b0e9b8e0887
+Revises: 740fe24a7712
+Create Date: 2022-02-27 11:48:55.107299
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7b0e9b8e0887'
+down_revision = '740fe24a7712'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', type_='foreignkey')
+    op.add_column('accounts', sa.Column('fetch_current_batch_end_date', sa.DateTime(timezone=True), nullable=True))
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_date = posts.created_at
+            FROM posts WHERE accounts.fetch_current_batch_end_id = posts.id;
+            ''')
+
+    # update ids from "mastodon:69420@chitter.xyz" format to just "69420"
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_id =
+            split_part(
+                split_part(fetch_current_batch_end_id, ':', 2),
+                    '@', 1);
+        ''')
+
+
+
+def downgrade():
+    # converts ids like "69420" back to "mastodon:69420@chitter.xyz"
+    # i sure hope there isn't a mastodon-compatible out there that can have
+    # : or @ in its post ids
+    op.execute('''
+        WITH accounts_exploded_ids AS (
+            SELECT
+                id,
+                split_part(id, ':', 1) || ':' AS service,
+                CASE WHEN position('@' IN id) != 0
+                    THEN '@' || split_part(id, @, 2)
+                    ELSE ''
+                END as instance
+            FROM accounts
+        )
+        UPDATE accounts SET fetch_current_batch_end_id = e.service || fetch_current_batch_end_id || e.instance
+        FROM accounts_exploded_ids AS e WHERE e.id = accounts.id AND fetch_current_batch_end_id IS NOT NULL;
+    ''')
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_id = NULL
+            WHERE NOT EXISTS (SELECT 1 FROM posts WHERE fetch_current_batch_end_id = posts.id);
+            ''')
+    op.create_foreign_key('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', 'posts', ['fetch_current_batch_end_id'], ['id'], ondelete='SET NULL')
+    op.drop_column('accounts', 'fetch_current_batch_end_date')

migrations/versions/af763dccc0b4_remove_favourite_reblog_count_from_posts.py

@@ -0,0 +1,26 @@
+"""remove favourite, reblog count from posts
+
+Revision ID: af763dccc0b4
+Revises: 4b56cde3ebd7
+Create Date: 2021-05-14 19:45:37.429645
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'af763dccc0b4'
+down_revision = '4b56cde3ebd7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column('posts', 'reblogs')
+    op.drop_column('posts', 'favourites')
+
+
+def downgrade():
+    op.add_column('posts', sa.Column('favourites', sa.Integer(), nullable=True))
+    op.add_column('posts', sa.Column('reblogs', sa.Integer(), nullable=True))

model.py

@@ -67,6 +67,34 @@ class RemoteIDMixin(object):
     @mastodon_id.setter
     def mastodon_id(self, id_):
         self.id = "mastodon:{}@{}".format(id_, self.mastodon_instance)
+        
+    @property
+    def misskey_instance(self):
+        if not self.id:
+            return None
+        if self.service != "misskey":
+            raise Exception(
+                "tried to get misskey instance for a {} {}"
+                .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[1]
+        
+    @misskey_instance.setter
+    def misskey_instance(self, instance):
+        self.id = "misskey:{}@{}".format(self.misskey_id, instance)
+        
+    @property
+    def misskey_id(self):
+        if not self.id:
+            return None
+        if self.service != "misskey":
+            raise Exception(
+                    "tried to get misskey id for a {} {}"
+                    .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[0]
+
+    @misskey_id.setter
+    def misskey_id(self, id_):
+        self.id = "misskey:{}@{}".format(id_, self.misskey_instance)
 
     @property
     def remote_id(self):
@@ -74,6 +102,8 @@ class RemoteIDMixin(object):
             return self.twitter_id
         elif self.service == 'mastodon':
             return self.mastodon_id
+        elif self.service == 'misskey':
+            return self.misskey_id
 
 
 ThreeWayPolicyEnum = db.Enum('keeponly', 'deleteonly', 'none',
@@ -116,15 +146,8 @@ class Account(TimestampMixin, RemoteIDMixin):
     fetch_history_complete = db.Column(db.Boolean, server_default='FALSE',
                                        nullable=False)
 
-    @declared_attr
-    def fetch_current_batch_end_id(cls):
-        return db.Column(db.String, db.ForeignKey('posts.id', ondelete='SET NULL'))
-    @declared_attr
-    def fetch_current_batch_end(cls):
-        return db.relationship("Post", foreign_keys=(cls.fetch_current_batch_end_id,))
-    # the declared_attr is necessary because of the foreign key
-    # and because this class is technically one big mixin
-    # https://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/mixins.html#mixing-in-relationships
+    fetch_current_batch_end_id = db.Column(db.String)
+    fetch_current_batch_end_date = db.Column(db.DateTime(timezone=True))
 
     reason = db.Column(db.String)
     dormant = db.Column(db.Boolean, server_default='FALSE', nullable=False)
@@ -302,9 +325,6 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin):
     has_media = db.Column(db.Boolean, server_default='FALSE', nullable=False)
     direct = db.Column(db.Boolean, server_default='FALSE', nullable=False)
 
-    favourites = db.Column(db.Integer)
-    reblogs = db.Column(db.Integer)
-
     is_reblog = db.Column(db.Boolean, server_default='FALSE', nullable=False)
 
     def __str__(self):
@@ -367,3 +387,24 @@ class MastodonInstance(db.Model):
 
     def bump(self, value=1):
         self.popularity = (self.popularity or 10) + value
+
+class MisskeyApp(db.Model, TimestampMixin):
+    __tablename__ = 'misskey_apps'
+
+    instance = db.Column(db.String, primary_key=True)
+    protocol = db.Column(db.String, nullable=False)
+    client_secret = db.Column(db.String, nullable=False)
+
+class MisskeyInstance(db.Model):
+    """
+    this is for the autocomplete in the misskey login form
+    it isn't coupled with anything else so that we can seed it with
+    some popular instances ahead of time
+    """
+    __tablename__ = 'misskey_instances'
+
+    instance = db.Column(db.String, primary_key=True)
+    popularity = db.Column(db.Float, server_default='10', nullable=False)
+
+    def bump(self, value=1):
+        self.popularity = (self.popularity or 10) + value

package-lock.json

@@ -1,13 +1,156 @@
 {
+  "name": "forget",
+  "lockfileVersion": 2,
   "requires": true,
-  "lockfileVersion": 1,
-  "dependencies": {
-    "@types/estree": {
-      "version": "0.0.39",
-      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz",
-      "integrity": "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==",
+  "packages": {
+    "": {
+      "devDependencies": {
+        "rollup": "^2.42.4",
+        "rollup-plugin-node-resolve": "^5.2.0",
+        "rollup-plugin-svelte": "^7.1.0",
+        "svelte": "^3.35.0"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "12.0.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
+      "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==",
       "dev": true
     },
+    "node_modules/@types/resolve": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-0.0.8.tgz",
+      "integrity": "sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/builtin-modules": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz",
+      "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/estree-walker": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz",
+      "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==",
+      "dev": true
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/is-module": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz",
+      "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=",
+      "dev": true
+    },
+    "node_modules/path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "dev": true
+    },
+    "node_modules/require-relative": {
+      "version": "0.8.7",
+      "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz",
+      "integrity": "sha1-eZlTn8ngR6N5KPoZb44VY9q9Nt4=",
+      "dev": true
+    },
+    "node_modules/resolve": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
+      "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==",
+      "dev": true,
+      "dependencies": {
+        "path-parse": "^1.0.6"
+      }
+    },
+    "node_modules/rollup": {
+      "version": "2.68.0",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.68.0.tgz",
+      "integrity": "sha512-XrMKOYK7oQcTio4wyTz466mucnd8LzkiZLozZ4Rz0zQD+HeX4nUK4B8GrTX/2EvN2/vBF/i2WnaXboPxo0JylA==",
+      "dev": true,
+      "bin": {
+        "rollup": "dist/bin/rollup"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
+      }
+    },
+    "node_modules/rollup-plugin-node-resolve": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-5.2.0.tgz",
+      "integrity": "sha512-jUlyaDXts7TW2CqQ4GaO5VJ4PwwaV8VUGA7+km3n6k6xtOEacf61u0VXwN80phY/evMcaS+9eIeJ9MOyDxt5Zw==",
+      "deprecated": "This package has been deprecated and is no longer maintained. Please use @rollup/plugin-node-resolve.",
+      "dev": true,
+      "dependencies": {
+        "@types/resolve": "0.0.8",
+        "builtin-modules": "^3.1.0",
+        "is-module": "^1.0.0",
+        "resolve": "^1.11.1",
+        "rollup-pluginutils": "^2.8.1"
+      },
+      "peerDependencies": {
+        "rollup": ">=1.11.0"
+      }
+    },
+    "node_modules/rollup-plugin-svelte": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-svelte/-/rollup-plugin-svelte-7.1.0.tgz",
+      "integrity": "sha512-vopCUq3G+25sKjwF5VilIbiY6KCuMNHP1PFvx2Vr3REBNMDllKHFZN2B9jwwC+MqNc3UPKkjXnceLPEjTjXGXg==",
+      "dev": true,
+      "dependencies": {
+        "require-relative": "^0.8.7",
+        "rollup-pluginutils": "^2.8.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "rollup": ">=2.0.0",
+        "svelte": ">=3.5.0"
+      }
+    },
+    "node_modules/rollup-pluginutils": {
+      "version": "2.8.2",
+      "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz",
+      "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==",
+      "dev": true,
+      "dependencies": {
+        "estree-walker": "^0.6.1"
+      }
+    },
+    "node_modules/svelte": {
+      "version": "3.46.4",
+      "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.46.4.tgz",
+      "integrity": "sha512-qKJzw6DpA33CIa+C/rGp4AUdSfii0DOTCzj/2YpSKKayw5WGSS624Et9L1nU1k2OVRS9vaENQXp2CVZNU+xvIg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    }
+  },
+  "dependencies": {
     "@types/node": {
       "version": "12.0.10",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
@@ -23,12 +166,6 @@
         "@types/node": "*"
       }
     },
-    "acorn": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.2.0.tgz",
-      "integrity": "sha512-8oe72N3WPMjA+2zVG71Ia0nXZ8DpQH+QyyHO+p06jT8eg8FGG3FbcUIi8KziHlAfheJQZeoqbvq1mQSQHXKYLw==",
-      "dev": true
-    },
     "builtin-modules": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz",
@@ -41,6 +178,13 @@
       "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==",
       "dev": true
     },
+    "fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "optional": true
+    },
     "is-module": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz",
@@ -48,9 +192,9 @@
       "dev": true
     },
     "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "require-relative": {
@@ -69,22 +213,12 @@
       }
     },
     "rollup": {
-      "version": "1.17.0",
-      "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.17.0.tgz",
-      "integrity": "sha512-k/j1m0NIsI4SYgCJR4MWPstGJOWfJyd6gycKoMhyoKPVXxm+L49XtbUwZyFsrSU2YXsOkM4u1ll9CS/ZgJBUpw==",
+      "version": "2.68.0",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.68.0.tgz",
+      "integrity": "sha512-XrMKOYK7oQcTio4wyTz466mucnd8LzkiZLozZ4Rz0zQD+HeX4nUK4B8GrTX/2EvN2/vBF/i2WnaXboPxo0JylA==",
       "dev": true,
       "requires": {
-        "@types/estree": "0.0.39",
-        "@types/node": "^12.6.2",
-        "acorn": "^6.2.0"
-      },
-      "dependencies": {
-        "@types/node": {
-          "version": "12.6.8",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-12.6.8.tgz",
-          "integrity": "sha512-aX+gFgA5GHcDi89KG5keey2zf0WfZk/HAQotEamsK2kbey+8yGKcson0hbK8E+v0NArlCJQCqMP161YhV6ZXLg==",
-          "dev": true
-        }
+        "fsevents": "~2.3.2"
       }
     },
     "rollup-plugin-node-resolve": {
@@ -98,55 +232,31 @@
         "is-module": "^1.0.0",
         "resolve": "^1.11.1",
         "rollup-pluginutils": "^2.8.1"
-      },
-      "dependencies": {
-        "estree-walker": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz",
-          "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==",
-          "dev": true
-        },
-        "rollup-pluginutils": {
-          "version": "2.8.1",
-          "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.1.tgz",
-          "integrity": "sha512-J5oAoysWar6GuZo0s+3bZ6sVZAC0pfqKz68De7ZgDi5z63jOVZn1uJL/+z1jeKHNbGII8kAyHF5q8LnxSX5lQg==",
-          "dev": true,
-          "requires": {
-            "estree-walker": "^0.6.1"
-          }
-        }
       }
     },
     "rollup-plugin-svelte": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-svelte/-/rollup-plugin-svelte-5.1.0.tgz",
-      "integrity": "sha512-4MRZG29dAWDpoxEs5uIHzDnYafQEOLaKIJAuDYUtFIzEm1F1IGSTlFyjd8/qk4wltlHdu6V7YfZY53+CKryhMg==",
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-svelte/-/rollup-plugin-svelte-7.1.0.tgz",
+      "integrity": "sha512-vopCUq3G+25sKjwF5VilIbiY6KCuMNHP1PFvx2Vr3REBNMDllKHFZN2B9jwwC+MqNc3UPKkjXnceLPEjTjXGXg==",
       "dev": true,
       "requires": {
         "require-relative": "^0.8.7",
-        "rollup-pluginutils": "^2.3.3",
-        "sourcemap-codec": "^1.4.4"
+        "rollup-pluginutils": "^2.8.2"
       }
     },
     "rollup-pluginutils": {
-      "version": "2.8.1",
-      "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.1.tgz",
-      "integrity": "sha512-J5oAoysWar6GuZo0s+3bZ6sVZAC0pfqKz68De7ZgDi5z63jOVZn1uJL/+z1jeKHNbGII8kAyHF5q8LnxSX5lQg==",
+      "version": "2.8.2",
+      "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz",
+      "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==",
       "dev": true,
       "requires": {
         "estree-walker": "^0.6.1"
       }
     },
-    "sourcemap-codec": {
-      "version": "1.4.6",
-      "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.6.tgz",
-      "integrity": "sha512-1ZooVLYFxC448piVLBbtOxFcXwnymH9oUF8nRd3CuYDVvkRBxRl6pB4Mtas5a4drtL+E8LDgFkQNcgIw6tc8Hg==",
-      "dev": true
-    },
     "svelte": {
-      "version": "3.6.10",
-      "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.6.10.tgz",
-      "integrity": "sha512-2N9kIbDal5z/aZloaRCOQ9dlCtuCE08NZITDlSdG7fOl4kFrE2qnXCq+lSFtI15ABWCXPc17cX5vJvdOgUsKqw==",
+      "version": "3.46.4",
+      "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.46.4.tgz",
+      "integrity": "sha512-qKJzw6DpA33CIa+C/rGp4AUdSfii0DOTCzj/2YpSKKayw5WGSS624Et9L1nU1k2OVRS9vaENQXp2CVZNU+xvIg==",
       "dev": true
     }
   }

package.json

@@ -1,8 +1,8 @@
 {
   "devDependencies": {
-    "rollup": "^1.17.0",
+    "rollup": "^2.42.4",
     "rollup-plugin-node-resolve": "^5.2.0",
-    "rollup-plugin-svelte": "^5.1.0",
-    "svelte": "^3.6.10"
+    "rollup-plugin-svelte": "^7.1.0",
+    "svelte": "^3.35.0"
   }
 }

pytest.ini

@@ -1,2 +0,0 @@
-[pytest]
-redis_port = 15487

requirements-dev.txt

@@ -1,27 +1,73 @@
--i https://pypi.python.org/simple/
-atomicwrites==1.3.0
-attrs==19.1.0
-certifi==2019.6.16
-chardet==3.0.4
-codecov==2.0.15
-coverage==4.5.4
-idna==2.8
-importlib-metadata==0.19
-mirakuru==2.0.1
-more-itertools==7.2.0
-packaging==19.1
-pluggy==0.12.0
-port-for==0.4
-psutil==5.6.3 ; sys_platform != 'cygwin'
-py==1.8.0
-pyparsing==2.4.2
-pytest-cov==2.7.1
-pytest-redis==1.3.2
-pytest==5.1.1
-redis==3.2.1
-requests==2.22.0
-six==1.12.0
-urllib3==1.25.3
-versioneer==0.18
-wcwidth==0.1.7
-zipp==0.6.0
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements --dev
+#
+
+# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
+# requirements. To emit only development requirements, pass "--dev-only".
+
+-i https://pypi.python.org/simple
+alembic==1.7.6
+amqp==5.0.9; python_version >= '3.6'
+attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+billiard==3.6.4.0
+blinker==1.4
+blurhash==1.1.4
+brotli==1.0.9
+celery==5.2.3
+certifi==2021.10.8
+charset-normalizer==2.0.12; python_version >= '3'
+click-didyoumean==0.3.0; python_version < '4' and python_full_version >= '3.6.2'
+click-plugins==1.1.1
+click-repl==0.2.0
+click==8.0.4; python_version >= '3.6'
+cloudpickle==2.0.0; python_version >= '3.6'
+codecov==2.1.12
+coverage==6.3.2
+csscompressor==0.9.5
+decorator==5.1.1; python_version >= '3.5'
+deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+doit==0.34.2
+flask-migrate==3.1.0
+flask-sqlalchemy==2.5.1
+flask==2.0.3
+greenlet==1.1.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+gunicorn==20.1.0
+honcho==1.1.0
+idna==3.3; python_version >= '3'
+iniconfig==1.1.1
+itsdangerous==2.1.0; python_version >= '3.7'
+jinja2==3.0.3; python_version >= '3.6'
+kombu==5.2.3; python_version >= '3.7'
+mako==1.1.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+markupsafe==2.1.0; python_version >= '3.7'
+mastodon.py==1.5.1
+packaging==21.3; python_version >= '3.6'
+pillow==9.0.1
+pluggy==1.0.0; python_version >= '3.6'
+prompt-toolkit==3.0.28; python_full_version >= '3.6.2'
+psycopg2==2.9.3
+py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pyinotify==0.9.6; sys_platform == 'linux'
+pyparsing==3.0.7; python_version >= '3.6'
+pytest-cov==3.0.0
+pytest==7.0.1
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-magic==0.4.25; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pytz==2021.3
+raven==6.10.0
+redis==4.1.4
+requests==2.27.1
+setuptools==59.6.0; python_version >= '3.6'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sqlalchemy==1.4.31
+tomli==2.0.1; python_version >= '3.7'
+twitter==1.19.3
+urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+versioneer==0.21
+vine==5.0.0; python_version >= '3.6'
+wcwidth==0.2.5
+werkzeug==2.0.3; python_version >= '3.6'
+wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

requirements.txt

@@ -1,45 +1,60 @@
--i https://pypi.python.org/simple/
-alembic==1.1.0
-amqp==2.5.1
-billiard==3.6.1.0
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+-i https://pypi.python.org/simple
+alembic==1.7.6
+amqp==5.0.9; python_version >= '3.6'
+billiard==3.6.4.0
 blinker==1.4
-brotli==1.0.7
-celery==4.3.0
-certifi==2019.6.16
-chardet==3.0.4
-click==7.0
-cloudpickle==1.2.1
+blurhash==1.1.4
+brotli==1.0.9
+celery==5.2.3
+certifi==2021.10.8
+charset-normalizer==2.0.12; python_version >= '3'
+click-didyoumean==0.3.0; python_version < '4' and python_full_version >= '3.6.2'
+click-plugins==1.1.1
+click-repl==0.2.0
+click==8.0.4; python_version >= '3.6'
+cloudpickle==2.0.0; python_version >= '3.6'
 csscompressor==0.9.5
-decorator==4.4.0
-doit==0.31.1
-flask-migrate==2.5.2
-flask-sqlalchemy==2.4.0
-flask==1.1.1
-gunicorn==19.9.0
-honcho==1.0.1
-idna==2.8
-importlib-metadata==0.19
-itsdangerous==1.1.0
-jinja2==2.10.1
-kombu==4.6.4
-mako==1.1.0
-markupsafe==1.1.1
-mastodon.py==1.4.6
-more-itertools==7.2.0
-pillow==6.1.0
-psycopg2==2.8.3
-pyinotify==0.9.6 ; sys_platform == 'linux'
-python-dateutil==2.8.0
-python-editor==1.0.4
-python-magic==0.4.15
-pytz==2019.2
+decorator==5.1.1; python_version >= '3.5'
+deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+doit==0.34.2
+flask-migrate==3.1.0
+flask-sqlalchemy==2.5.1
+flask==2.0.3
+greenlet==1.1.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+gunicorn==20.1.0
+honcho==1.1.0
+idna==3.3; python_version >= '3'
+itsdangerous==2.1.0; python_version >= '3.7'
+jinja2==3.0.3; python_version >= '3.6'
+kombu==5.2.3; python_version >= '3.7'
+mako==1.1.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+markupsafe==2.1.0; python_version >= '3.7'
+mastodon.py==1.5.1
+packaging==21.3; python_version >= '3.6'
+pillow==9.0.1
+prompt-toolkit==3.0.28; python_full_version >= '3.6.2'
+psycopg2==2.9.3
+pyinotify==0.9.6; sys_platform == 'linux'
+pyparsing==3.0.7; python_version >= '3.6'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-magic==0.4.25; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pytz==2021.3
 raven==6.10.0
-redis==3.2.1
-requests==2.22.0
-six==1.12.0
-sqlalchemy==1.3.5
-twitter==1.18.0
-urllib3==1.25.3
-vine==1.3.0
-werkzeug==0.15.5
-zipp==0.6.0
+redis==4.1.4
+requests==2.27.1
+setuptools==59.6.0; python_version >= '3.6'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sqlalchemy==1.4.31
+twitter==1.19.3
+urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+vine==5.0.0; python_version >= '3.6'
+wcwidth==0.2.5
+werkzeug==2.0.3; python_version >= '3.6'
+wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

rollup.config.js

@@ -7,8 +7,10 @@ export default {
     },
     plugins: [
         svelte({
+            extensions: ['.html'],
             include: 'components/**/*.html',
-            hydratable: true,
+            compilerOptions: {hydratable: true},
+            emitCss: false,
         }),
         node_resolve(),
     ]

routes/__init__.py

@@ -3,13 +3,16 @@ from flask import render_template, url_for, redirect, request, g,\
 from datetime import datetime, timedelta, timezone
 import libforget.twitter
 import libforget.mastodon
+import libforget.misskey
 from libforget.auth import require_auth, csrf,\
                      get_viewer
-from model import Session, TwitterArchive, MastodonApp
+from libforget.session import make_session
+from model import Session, TwitterArchive, MastodonApp, MisskeyApp
 from app import app, db, sentry, imgproxy
 import tasks
 from zipfile import BadZipFile
 from twitter import TwitterError
+from urllib.parse import urlparse
 from urllib.error import URLError
 import libforget.version
 import libforget.settings
@@ -34,10 +37,13 @@ def index():
 
 @app.route('/about/')
 def about():
-    instances = libforget.mastodon.suggested_instances()
+    blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
+    mastodon_instances = libforget.mastodon.suggested_instances(blocklist=blocklist)
+    misskey_instances = libforget.misskey.suggested_instances(blocklist=blocklist)
     return render_template(
             'about.html',
-            mastodon_instances=instances,
+            mastodon_instances=mastodon_instances,
+            misskey_instances=misskey_instances,
             twitter_login_error='twitter_login_error' in request.args)
 
 
@@ -171,6 +177,9 @@ def logout():
     return redirect(url_for('about'))
 
 
+def domain_from_url(url):
+    return urlparse(url).netloc.lower() or urlparse("//"+url).netloc.lower()
+
 @app.route('/login/mastodon', methods=('GET', 'POST'))
 def mastodon_login_step1(instance=None):
 
@@ -178,9 +187,11 @@ def mastodon_login_step1(instance=None):
                     or request.form.get('instance_url', None))
 
     if not instance_url:
+        blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
         instances = libforget.mastodon.suggested_instances(
                 limit=30,
-                min_popularity=1
+                min_popularity=1,
+                blocklist=blocklist,
         )
         return render_template(
                 'mastodon_login.html', instances=instances,
@@ -188,28 +199,21 @@ def mastodon_login_step1(instance=None):
                 generic_error='error' in request.args
                 )
 
-    instance_url = instance_url.lower()
-    # strip protocol
-    instance_url = re.sub('^https?://', '', instance_url,
-                          count=1, flags=re.IGNORECASE)
-    # strip username
-    instance_url = instance_url.split("@")[-1]
-    # strip trailing path
-    instance_url = instance_url.split('/')[0]
+    instance_url = domain_from_url(instance_url)
 
     callback = url_for('mastodon_login_step2',
                        instance_url=instance_url, _external=True)
 
     try:
-        app = libforget.mastodon.get_or_create_app(
+        mastoapp = libforget.mastodon.get_or_create_app(
                 instance_url,
                 callback,
                 url_for('index', _external=True))
-        db.session.merge(app)
+        db.session.merge(mastoapp)
 
         db.session.commit()
 
-        return redirect(libforget.mastodon.login_url(app, callback))
+        return redirect(libforget.mastodon.login_url(mastoapp, callback))
 
     except Exception:
         if sentry:
@@ -220,14 +224,77 @@ def mastodon_login_step1(instance=None):
 @app.route('/login/mastodon/callback/<instance_url>')
 def mastodon_login_step2(instance_url):
     code = request.args.get('code', None)
-    app = MastodonApp.query.get(instance_url)
-    if not code or not app:
+    mastoapp = MastodonApp.query.get(instance_url)
+    if not code or not mastoapp:
         return redirect(url_for('mastodon_login_step1', error=True))
 
     callback = url_for('mastodon_login_step2',
                        instance_url=instance_url, _external=True)
 
-    token = libforget.mastodon.receive_code(code, app, callback)
+    token = libforget.mastodon.receive_code(code, mastoapp, callback)
+    account = token.account
+
+    session = login(account.id)
+
+    db.session.commit()
+
+    g.viewer = session
+
+    resp = redirect(url_for('index', _anchor='bump_instance'))
+    return resp
+
+
+@app.route('/login/misskey', methods=('GET', 'POST'))
+def misskey_login(instance=None):
+    instance_url = (request.args.get('instance_url', None)
+                    or request.form.get('instance_url', None))
+
+    if not instance_url:
+        blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
+        instances = libforget.misskey.suggested_instances(
+            limit = 30,
+            min_popularity = 1,
+            blocklist=blocklist,
+        )
+        return render_template(
+                'misskey_login.html', instances=instances,
+                address_error=request.method == 'POST',
+                generic_error='error' in request.args
+                )
+
+    instance_url = domain_from_url(instance_url)
+
+    callback = url_for('misskey_callback',
+                       instance_url=instance_url, _external=True)
+
+    try:
+        session = make_session()
+        mkapp = libforget.misskey.get_or_create_app(
+                instance_url,
+                callback,
+                url_for('index', _external=True),
+                session)
+        db.session.merge(mkapp)
+
+        db.session.commit()
+
+        return redirect(libforget.misskey.login_url(mkapp, callback, session))
+
+    except Exception:
+        if sentry:
+            sentry.captureException()
+        return redirect(url_for('misskey_login', error=True))
+
+
+@app.route('/login/misskey/callback/<instance_url>')
+def misskey_callback(instance_url):
+    # legacy auth and miauth use different parameter names
+    token = request.args.get('token', None) or request.args.get('session', None)
+    mkapp = MisskeyApp.query.get(instance_url)
+    if not token or not mkapp:
+        return redirect(url_for('misskey_login', error=True))
+
+    token = libforget.misskey.receive_token(token, mkapp)
     account = token.account
 
     session = login(account.id)

routes/api.py

@@ -1,4 +1,4 @@
-from app import app, db
+from app import app, db, imgproxy
 from libforget.auth import require_auth_api, get_viewer
 from flask import jsonify, redirect, make_response, request, Response
 from model import Account
@@ -6,13 +6,20 @@ import libforget.settings
 import libforget.json
 import random
 
-@app.route('/api/health_check')
-def health_check():
+@app.route('/api/health_check') # deprecated 2021-03-12
+@app.route('/api/status_check')
+def api_status_check():
     try:
         db.session.execute('SELECT 1')
-        return 'ok'
     except Exception:
-        return ('bad', 500)
+        return ('PostgreSQL bad', 500)
+
+    try:
+        imgproxy.redis.set('forget-status-check', 'howdy', ex=5)
+    except Exception:
+        return ('Redis bad', 500)
+
+    return 'OK'
 
 
 @app.route('/api/settings', methods=('PUT',))
@@ -60,22 +67,3 @@ def users_badge():
     return redirect(
             "https://img.shields.io/badge/active%20users-{}-blue.svg"
             .format(count))
-
-
-@app.route('/api/known_instances', methods=('GET', 'DELETE'))
-def known_instances():
-    if request.method == 'GET':
-        known = request.cookies.get('forget_known_instances', '')
-        if not known:
-            return Response('[]', 404, mimetype='application/json')
-
-        # pad to avoid oracle attacks
-        for _ in range(random.randint(0, 1000)):
-                known += random.choice((' ', '\t', '\n'))
-
-        return Response(known, mimetype='application/json')
-
-    elif request.method == 'DELETE':
-        resp = Response('', 204)
-        resp.set_cookie('forget_known_instances', '', max_age=0)
-        return resp

routes/misc.py

@@ -1,5 +1,5 @@
 from app import app, db, sentry
-from flask import g, render_template, make_response, redirect
+from flask import g, render_template, make_response, redirect, request
 import version
 import libforget.version
 from libforget.auth import get_viewer_session, set_session_cookie
@@ -48,6 +48,8 @@ def not_found(e):
 
 @app.errorhandler(500)
 def internal_server_error(e):
+    if request.endpoint and request.endpoint.startswith('api_'):
+        return e.get_response()
     return (render_template('500.html', e=e), 500)
 
 

tasks.py

@@ -2,9 +2,10 @@ from celery import Celery, Task
 from app import app as flaskapp
 from app import db
 from model import Session, Account, TwitterArchive, Post, OAuthToken,\
-                  MastodonInstance
+                  MastodonInstance, MisskeyInstance
 import libforget.twitter
 import libforget.mastodon
+import libforget.misskey
 from datetime import timedelta, datetime, timezone
 from time import time
 from zipfile import ZipFile
@@ -22,6 +23,7 @@ app = Celery(
     'tasks',
     broker=flaskapp.config['CELERY_BROKER'],
     task_serializer='pickle',
+    accept_content={'pickle',},
     task_soft_time_limit=600,
     task_time_limit=1200,
 )
@@ -118,10 +120,10 @@ def fetch_acc(id_):
             else:
                 max_id = None
             since_id = None
-        elif account.fetch_current_batch_end:
+        elif account.fetch_current_batch_end_date:
             oldest = (db.session.query(Post)
                       .with_parent(account, 'posts')
-                      .filter(Post.created_at > account.fetch_current_batch_end.created_at)
+                      .filter(Post.created_at > account.fetch_current_batch_end_date)
                       .order_by(db.asc(Post.created_at))
                       .first())
             # ^ None if this is our first fetch of this batch, otherwise oldest of this batch
@@ -129,7 +131,7 @@ def fetch_acc(id_):
                 max_id = oldest.remote_id
             else:
                 max_id = None
-            since_id = account.fetch_current_batch_end.remote_id
+            since_id = account.fetch_current_batch_end_id
         else:
             # we shouldn't get here unless the user had no posts on the service last time we fetched
             max_id = None
@@ -151,6 +153,8 @@ def fetch_acc(id_):
             fetch_posts = libforget.twitter.fetch_posts
         elif (account.service == 'mastodon'):
             fetch_posts = libforget.mastodon.fetch_posts
+        elif (account.service == 'misskey'):
+            fetch_posts = libforget.misskey.fetch_posts
         posts = fetch_posts(account, max_id, since_id)
 
         if posts is None:
@@ -168,9 +172,11 @@ def fetch_acc(id_):
             batch_end = (Post.query.with_parent(account, 'posts').order_by(
                 db.desc(Post.created_at)).first())
             if batch_end:
-                account.fetch_current_batch_end_id = batch_end.id
+                account.fetch_current_batch_end_id = batch_end.remote_id
+                account.fetch_current_batch_end_date = batch_end.created_at
             else:
                 account.fetch_current_batch_end_id = None
+                account.fetch_current_batch_end_date = None
             db.session.commit()
 
         else:
@@ -291,6 +297,10 @@ def delete_from_account(account_id):
                 if refreshed and is_eligible(refreshed[0]):
                     to_delete = refreshed[0]
                     break
+        elif account.service == 'misskey':
+            action = libforget.misskey.delete
+            posts = refresh_posts(posts)
+            to_delete = next(filter(is_eligible, posts), None)
 
         if to_delete:
             print("Deleting {}".format(to_delete))
@@ -317,6 +327,8 @@ def refresh_posts(posts):
         return libforget.twitter.refresh_posts(posts)
     elif posts[0].service == 'mastodon':
         return libforget.mastodon.refresh_posts(posts)
+    elif posts[0].service == 'misskey':
+        return libforget.misskey.refresh_posts(posts)
 
 
 @app.task()
@@ -419,10 +431,16 @@ def queue_deletes():
 @unique
 def refresh_account_with_oldest_post():
     then = time()
-    post = (Post.query.outerjoin(Post.author).join(Account.tokens)
-            .filter(Account.backoff_until < db.func.now())
-            .filter(~Account.dormant).group_by(Post).order_by(
-                db.asc(Post.updated_at)).first())
+    post = db.session.query(Post).from_statement(db.text("""
+    SELECT posts.id, posts.author_id
+        FROM posts, accounts, oauth_tokens
+        WHERE accounts.id = posts.author_id
+        AND accounts.id = oauth_tokens.account_id
+        AND accounts.backoff_until < now()
+        AND NOT accounts.dormant
+        ORDER BY posts.updated_at ASC
+        LIMIT 1;
+    """).columns(Post.id, Post.author_id)).one_or_none()
     if post:
         aid = post.author_id
         refresh_account(aid)
@@ -468,6 +486,33 @@ def update_mastodon_instances_popularity():
     })
     db.session.commit()
 
+@app.task
+def update_misskey_instances_popularity():
+    # bump score for each active account
+    for acct in (Account.query.options(db.joinedload(Account.sessions))
+                 .filter(~Account.dormant).filter(
+                     Account.id.like('misskey:%'))):
+        instance = MisskeyInstance.query.get(acct.misskey_instance)
+        if not instance:
+            instance = MisskeyInstance(
+                instance=acct.misskey_instance, popularity=10)
+            db.session.add(instance)
+        amount = 0.01
+        if acct.policy_enabled:
+            amount = 0.5
+        for _ in acct.sessions:
+            amount += 0.1
+        instance.bump(amount / max(1, instance.popularity))
+
+    # normalise scores so the top is 20
+    top_pop = (db.session.query(db.func.max(MisskeyInstance.popularity))
+               .scalar())
+    MisskeyInstance.query.update({
+        MisskeyInstance.popularity:
+        MisskeyInstance.popularity * 20 / top_pop
+    })
+    db.session.commit()
+
 
 app.add_periodic_task(40, queue_fetch_for_most_stale_accounts)
 app.add_periodic_task(9, queue_deletes)
@@ -475,6 +520,7 @@ app.add_periodic_task(6, refresh_account_with_oldest_post)
 app.add_periodic_task(50, refresh_account_with_longest_time_since_refresh)
 app.add_periodic_task(300, periodic_cleanup)
 app.add_periodic_task(300, update_mastodon_instances_popularity)
+app.add_periodic_task(300, update_misskey_instances_popularity)
 
 if __name__ == '__main__':
     app.worker_main()

templates/about.html

@@ -46,11 +46,37 @@
 </a>
 {% endif %}
 
+</p>
+
+<p id='misskey_instance_buttons'>
+{% for instance in misskey_instances %}
+<a class='btn primary misskey-colored' href="{{ url_for('misskey_login', instance_url=instance) }}">
+{% if loop.first %}
+    {{picture(st, 'misskey', (20,40,80), ('webp', 'png'))}}
+    Log in with
+{% endif %}
+        {{instance}}
+</a>
+{% else %}
+<a class='btn primary misskey-colored' href="{{ url_for('misskey_login') }}">
+    {{picture(st, 'misskey', (20,40,80), ('webp', 'png'))}}
+    Log in with Misskey
+</a>
+{% endfor %}
+
+
+{% if misskey_instances %}
+<a class='btn secondary' href="{{ url_for('misskey_login') }}">
+    Another Misskey instance
+</a>
+{% endif %}
+
 </p>
 </section>
 
+<!-- Mastodon -->
 
-<script type="application/json" id="top_instances">
+<script type="application/json" id="mastodon_top_instances">
 [
 {% for instance in mastodon_instances %}
     {"instance": "{{instance}}"}
@@ -61,7 +87,7 @@
 ]
 </script>
 
-<script type="text/html+template" id="instance_button_template">
+<script type="text/html+template" id="mastodon_instance_button_template">
     <a class='btn primary mastodon-colored'
         href="{{ url_for('mastodon_login_step1') }}?instance_url=${encodeURIComponent(instance)}">
         ${ !first? '' : `
@@ -72,12 +98,43 @@
     </a>
 </script>
 
-<script type="text/html+template" id="another_instance_button_template">
+<script type="text/html+template" id="mastodon_another_instance_button_template">
     <a class='btn secondary' href="{{ url_for('mastodon_login_step1') }}">
         Another Mastodon instance
     </a>
 </script>
 
+<!-- Misskey -->
+
+<script type="application/json" id="misskey_top_instances">
+[
+{% for instance in misskey_instances %}
+    {"instance": "{{instance}}"}
+    {%- if not loop.last -%}
+    ,
+    {%- endif %}
+{% endfor %}
+]
+</script>
+
+<script type="text/html+template" id="misskey_instance_button_template">
+    <a class='btn primary misskey-colored'
+        href="{{ url_for('misskey_login') }}?instance_url=${encodeURIComponent(instance)}">
+        ${ !first? '' : `
+            {{picture(st, 'misskey', (20,40,80), ('webp', 'png'))}}
+            Log in with
+            `}
+        ${ instance }
+    </a>
+</script>
+
+<script type="text/html+template" id="misskey_another_instance_button_template">
+    <a class='btn secondary' href="{{ url_for('misskey_login') }}">
+        Another Misskey instance
+    </a>
+</script>
+
+
 
 {% endif %}
 
@@ -86,7 +143,7 @@
 <ul>
     <li>Delete your posts when they cross an age threshold.</li>
     <li>Or keep your post count in check, deleting old posts when you go over.</li>
-    <li>Preserve old posts that matter by giving them a favourite.</li>
+    <li>Preserve old posts that matter by giving them a favourite or a reaction.</li>
     <li>Set it and <i>forget</i> it. Forget works continuously in the background.</li>
 </ul>
 </section>

templates/logged_in.html

@@ -68,18 +68,19 @@
     {{interval_input(g.viewer.account, 'policy_keep_younger', scales)}}
     old and are not one of your
     <input type=number name=policy_keep_latest min=0 step=1 style='max-width:8ch' value={{g.viewer.account.policy_keep_latest}}>
-    most recent posts will expire
+    most recent posts will be considered for deletion
 </p>
-<p>Keep
+{% if g.viewer.account.service == 'misskey' %}
+<p>…unless you
     <span class="radiostrip">
         <span class="choice">
         <input type=radio name=policy_keep_favourites value=keeponly id=policy_keep_favourites_keeponly {{ "checked" if g.viewer.account.policy_keep_favourites == 'keeponly' }}>
-        <label for=policy_keep_favourites_keeponly>favourited posts</label>
+        <label for=policy_keep_favourites_keeponly>reacted to them</label>
         </span>
 
         <span class="choice">
         <input type=radio name=policy_keep_favourites value=deleteonly id=policy_keep_favourites_deleteonly {{ "checked" if g.viewer.account.policy_keep_favourites == 'deleteonly' }}>
-        <label for=policy_keep_favourites_deleteonly>non-favourited posts</label>
+        <label for=policy_keep_favourites_deleteonly>have not reacted to them</label>
         </span>
 
         <span class="choice">
@@ -88,16 +89,36 @@
         </span>
     </span>
 </p>
-<p>Keep
+{%- else %}
+<p>…unless you
+    <span class="radiostrip">
+        <span class="choice">
+        <input type=radio name=policy_keep_favourites value=keeponly id=policy_keep_favourites_keeponly {{ "checked" if g.viewer.account.policy_keep_favourites == 'keeponly' }}>
+        <label for=policy_keep_favourites_keeponly>favourited them</label>
+        </span>
+
+        <span class="choice">
+        <input type=radio name=policy_keep_favourites value=deleteonly id=policy_keep_favourites_deleteonly {{ "checked" if g.viewer.account.policy_keep_favourites == 'deleteonly' }}>
+        <label for=policy_keep_favourites_deleteonly>have not favourited them</label>
+        </span>
+
+        <span class="choice">
+        <input type=radio name=policy_keep_favourites value=none id=policy_keep_favourites_none {{ "checked" if g.viewer.account.policy_keep_favourites == 'none' }}>
+        <label for=policy_keep_favourites_none>neither</label>
+        </span>
+    </span>
+</p>
+{%- endif %}
+<p>…or unless they
     <span class="radiostrip">
         <span class="choice">
         <input type=radio name=policy_keep_media value=keeponly id=policy_keep_media_keeponly {{ "checked" if g.viewer.account.policy_keep_media == 'keeponly' }}>
-        <label for=policy_keep_media_keeponly>media posts</label>
+        <label for=policy_keep_media_keeponly>have media</label>
         </span>
 
         <span class="choice">
         <input type=radio name=policy_keep_media value=deleteonly id=policy_keep_media_deleteonly {{ "checked" if g.viewer.account.policy_keep_media == 'deleteonly' }}>
-        <label for=policy_keep_media_deleteonly>non-media posts</label>
+        <label for=policy_keep_media_deleteonly>do not have media</label>
         </span>
 
         <span class="choice">
@@ -105,8 +126,8 @@
         <label for=policy_keep_media_none>neither</label>
     </span>
 </p>
-{% if g.viewer.account.service == 'mastodon' %}
-<p>Keep direct messages
+{% if g.viewer.account.service == 'mastodon' or g.viewer.account.service == 'misskey' %}
+<p>Keep direct messages:
     <span class="radiostrip">
         <span class="choice">
         <input type=radio name=policy_keep_direct value=true id=policy_keep_direct_true {{ "checked" if g.viewer.account.policy_keep_direct }}>
@@ -122,7 +143,7 @@
 {% endif %}
 <p>Every
     {{interval_input(g.viewer.account, 'policy_delete_every', scales)}},
-    one expired post will be picked at random and deleted.
+    one post matching these rules will be picked at random and deleted.
 </p>
 <input type=submit value='Save settings'>
 <input type='hidden' name='csrf-token' value='{{g.viewer.csrf_token}}'>

templates/misskey_login.html

@@ -0,0 +1,29 @@
+{% extends 'lib/layout.html' %}
+{% block body %}
+<section>
+    <h2>Log in with Misskey</h2>
+
+{% if generic_error %}
+    <div class='banner error'>Something went wrong while logging in. Try again?</div>
+{% endif %}
+
+{% if address_error %}
+    <div class='banner error'>This doesn't look like a misskey instance url. Try again?</div>
+{% endif %}
+
+    <form method='post'>
+        <label>misskey instance:
+            <input type='text' name='instance_url' list='instances' placeholder='social.example.net'/>
+        </label>
+        <datalist id='instances'>
+            <option value=''>
+            {% for instance in instances %}
+            <option value='{{instance}}'>
+            {% endfor %}
+        </datalist>
+        <input name='confirm' value='Log in' type='submit'/>
+    </form>
+
+</div>
+</section>
+{% endblock %}

templates/privacy.html

@@ -12,13 +12,12 @@
     <li>A unique post identifier</li>
     <li>The post's time and date of publishing</li>
     <li>Whether the post has any media attached</li>
-    <li>Whether the post has been favourited by you</li>
-    <li>How many favourites and reblogs / retweets the post has</li>
-    <li>(Mastodon only) Whether the post is a direct message</li>
+    <li>Whether the post has been favourited by you (only Twitter or Mastodon); or if (not how) you reacted to the post (Misskey only)</li>
+    <li>Whether the post is a direct message (only Mastodon or Misskey)</li>
 </ul>
 <p>No other post metadata and no post contents are stored by Forget.</p>
 
-<p>Last updated on 2017-12-27. <a href="https://github.com/codl/forget/commits/master/templates/privacy.html">History</a>.</p>
+<p>Last updated on 2021-11-11. <a href="https://github.com/codl/forget/commits/master/templates/privacy.html">History</a>.</p>
 </section>
 
 {% endblock %}

test/test_libbrotli.py

@@ -7,10 +7,10 @@ TIMEOUT_TARGET = 0.2
 
 
 @pytest.fixture
-def app(redisdb):
+def app():
     from flask import Flask
     app_ = Flask(__name__)
-    app_.config['REDIS_URI'] = 'redis://localhost:15487'
+    app_.config['REDIS_URI'] = 'redis://localhost:6379'
     app_.debug = True
 
     @app_.route('/')

version.py

@@ -6,7 +6,7 @@
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.18 (https://github.com/warner/python-versioneer)
+# versioneer-0.21 (https://github.com/python-versioneer/python-versioneer)
 
 """Git implementation of _version.py."""
 
@@ -15,6 +15,7 @@ import os
 import re
 import subprocess
 import sys
+from typing import Callable, Dict
 
 
 def get_keywords():
@@ -52,12 +53,12 @@ class NotThisMethod(Exception):
     """Exception raised if a method is not valid for the current scenario."""
 
 
-LONG_VERSION_PY = {}
-HANDLERS = {}
+LONG_VERSION_PY: Dict[str, str] = {}
+HANDLERS: Dict[str, Dict[str, Callable]] = {}
 
 
 def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
+    """Create decorator to mark a method as the handler of a VCS."""
     def decorate(f):
         """Store f in HANDLERS[vcs][method]."""
         if vcs not in HANDLERS:
@@ -71,17 +72,17 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
                 env=None):
     """Call the given command(s)."""
     assert isinstance(commands, list)
-    p = None
-    for c in commands:
+    process = None
+    for command in commands:
         try:
-            dispcmd = str([c] + args)
+            dispcmd = str([command] + args)
             # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
-                                 stdout=subprocess.PIPE,
-                                 stderr=(subprocess.PIPE if hide_stderr
-                                         else None))
+            process = subprocess.Popen([command] + args, cwd=cwd, env=env,
+                                       stdout=subprocess.PIPE,
+                                       stderr=(subprocess.PIPE if hide_stderr
+                                               else None))
             break
-        except EnvironmentError:
+        except OSError:
             e = sys.exc_info()[1]
             if e.errno == errno.ENOENT:
                 continue
@@ -93,15 +94,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
         if verbose:
             print("unable to find command, tried %s" % (commands,))
         return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
-    if p.returncode != 0:
+    stdout = process.communicate()[0].strip().decode()
+    if process.returncode != 0:
         if verbose:
             print("unable to run %s (error)" % dispcmd)
             print("stdout was %s" % stdout)
-        return None, p.returncode
-    return stdout, p.returncode
+        return None, process.returncode
+    return stdout, process.returncode
 
 
 def versions_from_parentdir(parentdir_prefix, root, verbose):
@@ -113,15 +112,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
     """
     rootdirs = []
 
-    for i in range(3):
+    for _ in range(3):
         dirname = os.path.basename(root)
         if dirname.startswith(parentdir_prefix):
             return {"version": dirname[len(parentdir_prefix):],
                     "full-revisionid": None,
                     "dirty": False, "error": None, "date": None}
-        else:
-            rootdirs.append(root)
-            root = os.path.dirname(root)  # up a level
+        rootdirs.append(root)
+        root = os.path.dirname(root)  # up a level
 
     if verbose:
         print("Tried directories %s but none started with prefix %s" %
@@ -138,22 +136,21 @@ def git_get_keywords(versionfile_abs):
     # _version.py.
     keywords = {}
     try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-            if line.strip().startswith("git_date ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["date"] = mo.group(1)
-        f.close()
-    except EnvironmentError:
+        with open(versionfile_abs, "r") as fobj:
+            for line in fobj:
+                if line.strip().startswith("git_refnames ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["refnames"] = mo.group(1)
+                if line.strip().startswith("git_full ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["full"] = mo.group(1)
+                if line.strip().startswith("git_date ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["date"] = mo.group(1)
+    except OSError:
         pass
     return keywords
 
@@ -161,10 +158,14 @@ def git_get_keywords(versionfile_abs):
 @register_vcs_handler("git", "keywords")
 def git_versions_from_keywords(keywords, tag_prefix, verbose):
     """Get version information from git keywords."""
-    if not keywords:
-        raise NotThisMethod("no keywords at all, weird")
+    if "refnames" not in keywords:
+        raise NotThisMethod("Short version file found")
     date = keywords.get("date")
     if date is not None:
+        # Use only the last line.  Previous lines may contain GPG signature
+        # information.
+        date = date.splitlines()[-1]
+
         # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
         # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
         # -like" string, which we must then edit to make compliant), because
@@ -177,11 +178,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         if verbose:
             print("keywords are unexpanded, not using")
         raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    refs = {r.strip() for r in refnames.strip("()").split(",")}
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
     TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    tags = {r[len(TAG):] for r in refs if r.startswith(TAG)}
     if not tags:
         # Either we're using git < 1.8.3, or there really are no tags. We use
         # a heuristic: assume all version tags have a digit. The old git %d
@@ -190,7 +191,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
         # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r'\d', r)])
+        tags = {r for r in refs if re.search(r'\d', r)}
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
@@ -199,6 +200,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # sorting will prefer e.g. "2.0" over "2.0rc1"
         if ref.startswith(tag_prefix):
             r = ref[len(tag_prefix):]
+            # Filter out refs that exactly match prefix or that don't start
+            # with a number once the prefix is stripped (mostly a concern
+            # when prefix is '')
+            if not re.match(r'\d', r):
+                continue
             if verbose:
                 print("picking %s" % r)
             return {"version": r,
@@ -214,7 +220,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
 
 
 @register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command):
     """Get version from 'git describe' in the root of the source tree.
 
     This only gets called if the git-archive 'subst' keywords were *not*
@@ -222,11 +228,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     version string, meaning we're inside a checked out source tree.
     """
     GITS = ["git"]
+    TAG_PREFIX_REGEX = "*"
     if sys.platform == "win32":
         GITS = ["git.cmd", "git.exe"]
+        TAG_PREFIX_REGEX = r"\*"
 
-    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
-                          hide_stderr=True)
+    _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                   hide_stderr=True)
     if rc != 0:
         if verbose:
             print("Directory %s not under git control" % root)
@@ -234,15 +242,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
 
     # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
     # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
-                                          "--always", "--long",
-                                          "--match", "%s*" % tag_prefix],
-                                   cwd=root)
+    describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty",
+                                     "--always", "--long",
+                                     "--match",
+                                     "%s%s" % (tag_prefix, TAG_PREFIX_REGEX)],
+                              cwd=root)
     # --long was added in git-1.5.5
     if describe_out is None:
         raise NotThisMethod("'git describe' failed")
     describe_out = describe_out.strip()
-    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root)
     if full_out is None:
         raise NotThisMethod("'git rev-parse' failed")
     full_out = full_out.strip()
@@ -252,6 +261,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     pieces["short"] = full_out[:7]  # maybe improved later
     pieces["error"] = None
 
+    branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"],
+                             cwd=root)
+    # --abbrev-ref was added in git-1.6.3
+    if rc != 0 or branch_name is None:
+        raise NotThisMethod("'git rev-parse --abbrev-ref' returned error")
+    branch_name = branch_name.strip()
+
+    if branch_name == "HEAD":
+        # If we aren't exactly on a branch, pick a branch which represents
+        # the current commit. If all else fails, we are on a branchless
+        # commit.
+        branches, rc = runner(GITS, ["branch", "--contains"], cwd=root)
+        # --contains was added in git-1.5.4
+        if rc != 0 or branches is None:
+            raise NotThisMethod("'git branch --contains' returned error")
+        branches = branches.split("\n")
+
+        # Remove the first line if we're running detached
+        if "(" in branches[0]:
+            branches.pop(0)
+
+        # Strip off the leading "* " from the list of branches.
+        branches = [branch[2:] for branch in branches]
+        if "master" in branches:
+            branch_name = "master"
+        elif not branches:
+            branch_name = None
+        else:
+            # Pick the first branch that is returned. Good or bad.
+            branch_name = branches[0]
+
+    pieces["branch"] = branch_name
+
     # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
     # TAG might have hyphens.
     git_describe = describe_out
@@ -268,7 +310,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
         # TAG-NUM-gHEX
         mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
         if not mo:
-            # unparseable. Maybe git-describe is misbehaving?
+            # unparsable. Maybe git-describe is misbehaving?
             pieces["error"] = ("unable to parse git-describe output: '%s'"
                                % describe_out)
             return pieces
@@ -293,13 +335,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     else:
         # HEX: no tags
         pieces["closest-tag"] = None
-        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
-                                    cwd=root)
+        count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root)
         pieces["distance"] = int(count_out)  # total number of commits
 
     # commit date: see ISO-8601 comment in git_versions_from_keywords()
-    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
-                       cwd=root)[0].strip()
+    date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip()
+    # Use only the last line.  Previous lines may contain GPG signature
+    # information.
+    date = date.splitlines()[-1]
     pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
 
     return pieces
@@ -337,19 +380,67 @@ def render_pep440(pieces):
     return rendered
 
 
-def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
+def render_pep440_branch(pieces):
+    """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] .
+
+    The ".dev0" means not master branch. Note that .dev0 sorts backwards
+    (a feature branch will appear "older" than the master branch).
 
     Exceptions:
-    1: no tags. 0.post.devDISTANCE
+    1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty]
     """
     if pieces["closest-tag"]:
         rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
+        if pieces["distance"] or pieces["dirty"]:
+            if pieces["branch"] != "master":
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
     else:
         # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
+        rendered = "0"
+        if pieces["branch"] != "master":
+            rendered += ".dev0"
+        rendered += "+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def pep440_split_post(ver):
+    """Split pep440 version string at the post-release segment.
+
+    Returns the release segments before the post-release and the
+    post-release version number (or -1 if no post-release segment is present).
+    """
+    vc = str.split(ver, ".post")
+    return vc[0], int(vc[1] or 0) if len(vc) == 2 else None
+
+
+def render_pep440_pre(pieces):
+    """TAG[.postN.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post0.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        if pieces["distance"]:
+            # update the post release segment
+            tag_version, post_version = pep440_split_post(pieces["closest-tag"])
+            rendered = tag_version
+            if post_version is not None:
+                rendered += ".post%d.dev%d" % (post_version+1, pieces["distance"])
+            else:
+                rendered += ".post0.dev%d" % (pieces["distance"])
+        else:
+            # no commits, use the tag as the version
+            rendered = pieces["closest-tag"]
+    else:
+        # exception #1
+        rendered = "0.post0.dev%d" % pieces["distance"]
     return rendered
 
 
@@ -380,12 +471,41 @@ def render_pep440_post(pieces):
     return rendered
 
 
+def render_pep440_post_branch(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] .
+
+    The ".dev0" means not master branch.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["branch"] != "master":
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["branch"] != "master":
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
 def render_pep440_old(pieces):
     """TAG[.postDISTANCE[.dev0]] .
 
     The ".dev0" means dirty.
 
-    Eexceptions:
+    Exceptions:
     1: no tags. 0.postDISTANCE[.dev0]
     """
     if pieces["closest-tag"]:
@@ -456,10 +576,14 @@ def render(pieces, style):
 
     if style == "pep440":
         rendered = render_pep440(pieces)
+    elif style == "pep440-branch":
+        rendered = render_pep440_branch(pieces)
     elif style == "pep440-pre":
         rendered = render_pep440_pre(pieces)
     elif style == "pep440-post":
         rendered = render_pep440_post(pieces)
+    elif style == "pep440-post-branch":
+        rendered = render_pep440_post_branch(pieces)
     elif style == "pep440-old":
         rendered = render_pep440_old(pieces)
     elif style == "git-describe":
@@ -495,7 +619,7 @@ def get_versions():
         # versionfile_source is the relative path from the top of the source
         # tree (where the .git directory might live) to this file. Invert
         # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
+        for _ in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
     except NameError:
         return {"version": "0+unknown", "full-revisionid": None,