Delete dependabot.yml

it's dirty but it's needed for versioneer

I will figure out a better way to do this later

.codecov.yml

@@ -0,0 +1,3 @@
+ignore:
+    - version.py
+comment: off

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit =
+    */site-packages/*

.dockerignore

@@ -0,0 +1,22 @@
+.envrc.example
+.gitignore
+.tool-versions
+*.md
+Dockerfile
+docker-compose.yml
+#.git
+.github
+.codecov.yml
+.coveragerc
+.env
+.eslintrc.yml
+.gitattributes
+LICENSE
+CHANGELOG.markdown
+README.markdown
+config.example.py
+config.docker.py
+forget.example.service
+requirements-dev.txt
+data
+config.py

.env

@@ -0,0 +1 @@
+FLASK_APP=forget.py

.eslintrc.yml

@@ -0,0 +1,17 @@
+env:
+  browser: true
+  es6: true
+extends: 'eslint:recommended'
+rules:
+  indent:
+    - error
+    - 4
+  linebreak-style:
+    - error
+    - unix
+  quotes:
+    - error
+    - single
+  semi:
+    - error
+    - always

.gitattributes

@@ -0,0 +1 @@
+version.py export-subst

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: codl
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/stale.yml

@@ -0,0 +1,15 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 365
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels: []
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/docker-publish.yml

@@ -0,0 +1,102 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  schedule:
+    - cron: '18 10 * * *'
+  push:
+    branches: [ "master" ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ "master" ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@7e0881f8fe90b25e305bbf0309761e9314607e25
+        with:
+          cosign-release: 'v1.9.0'
+
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          COSIGN_EXPERIMENTAL: "true"
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}

.github/workflows/run-tests.yml

@@ -0,0 +1,39 @@
+name: Run tests on pushes and on PRs
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  run_tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.6"
+          cache: 'pipenv'
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "10"
+          cache: 'npm'
+
+      - name: Install pipenv and python dependencies
+        run: |
+          pip install -U pip pipenv
+          pipenv sync -d
+      - name: Install node dependencies
+        run: npm install
+
+      - name: Start Redis
+        run: docker run --name redis --publish 6379:6379 --detach redis
+
+
+      - name: Run tests with pytest
+        run: pipenv run pytest --cov=.
+
+      - uses: codecov/codecov-action@v2

.gitignore

@@ -5,4 +5,11 @@ celerybeat-schedule
 .doit.db
 static/*
 !static/.keep
+.cache/
+.coverage
+.pytest_cache
 
+data/*
+!data/.keep
+
+docker-compose.override.yml

CHANGELOG.markdown

@@ -0,0 +1,196 @@
+## v2.2.0
+
+Released 2022-08-10
+
+* add: instance hidelist
+  <https://github.com/codl/forget/pull/590>
+* add: docker deployment support (Thanks @shibaobun !)
+  <https://github.com/codl/forget/pull/612>
+* removed: migration path for known instances list from cookie to localstorage
+  <https://github.com/codl/forget/pull/545>
+
+## v2.1.0
+
+Released 2022-03-04
+
+* add: Misskey support (Thanks @Johann150 !)
+  <https://github.com/codl/forget/pull/544>
+* fix: lowered database impact of a background task
+  <https://github.com/codl/forget/issue/166>
+* fix: wording on "favourited posts" is unclear
+  <https://github.com/codl/forget/pull/366>
+* fix: failing to fetch some posts in some circumstances
+  <https://github.com/codl/forget/issue/584>
+
+## v2.0.0
+
+Released 2019-09-13
+
+* fix: newer twitter accounts not fetching new posts past the initial historical fetch
+  <https://github.com/codl/forget/pull/254>
+* fix: fetching task not getting rescheduled properly when doing the initial historic fetch
+  <https://github.com/codl/forget/pull/264>
+* BREAKING: disabled tweet archive upload since twitter has dropped support for them
+  <https://github.com/codl/forget/pull/265>
+
+## v1.6.1
+
+Released 2019-07-23
+
+* increased frequency of refresh jobs
+* version number in footer now links to changelog instead of commit log
+* updated about page. less negative, more succint, clarifies that forget is not a purging tool
+
+## v1.5.3
+
+Released 2019-07-11
+
+* fix: everything related to mastodon broken because of a typo
+
+## v1.5.2
+
+Released 2019-07-11
+
+* fix: stock user agent when querying mastodon servers
+
+## v1.5.1
+
+Released 2019-05-02
+
+* fix: proxied avatars not working in some environments
+* dependency updates
+
+## v1.5.0
+
+Released 2019-03-15
+
+* back off before hitting rate limit on mastodon instances
+* tracking of used instances for the login buttons on the front page is now entirely client-side,
+  avoiding a potential information disclosure vulnerability ([GH-175](https://github.com/codl/forget/issues/175))
+* fix: fetch\_acc running multiple copies fetching the same posts
+* internals: increased frequency of refresh jobs, decreased frequency of bookkeeping jobs
+
+## v1.4.3
+
+Released 2019-03-11
+
+* documentation improvements
+* fix: deadlock when refreshing or deleting from mastodon accounts ([GH-19](https://github.com/codl/forget/issues/19))
+* fix: crash in fetch\_acc when user has no posts
+* fix: not backing off if something crashes in refresh\_account
+* fix: crashes when trying to refresh but no accounts have been created yet
+
+## v1.4.2
+
+Released 2019-02-24
+
+* fix: implemented a more robust fetching algorithm, which should prevent accounts getting stuck with only a fraction of their posts fetched ([GH-13](https://github.com/codl/forget/issues/13))
+* fix: picture tags having an extra comma
+* fix: outdated joke in about page
+* fix: posts' status not getting refreshed (ie whether or not they were faved, or deleted externally)
+* internals: removed `x-` prefix from custom headers, as per [section 8.3.1 of RFC7231](https://httpwg.org/specs/rfc7231.html#considerations.for.new.header.fields)
+
+## v1.4.1 (security update)
+
+Released 2018-10-29
+
+* updated requests to 2.20.0 ([CVE-2018-18074](https://nvd.nist.gov/vuln/detail/CVE-2018-18074))
+
+## v1.4.0
+
+Released 2018-10-06
+
+* added warning when it looks like an archive is a full "Your Twitter data" archive
+
+## v1.3.0
+
+Released 2018-07-06
+
+* implement exponential backoff
+
+## v1.2.1
+
+Released 2018-05-08
+
+* limit number of log-in buttons to 5, and show up to 5 known instances
+
+## v1.2.0
+
+Released 2018-05-08
+
+* remember a user's mastodon instances and let them log in in one click ([GH-36](https://github.com/codl/forget/issues/36))
+
+## v1.1.3
+
+Released 2018-04-25
+
+* made radio strips more accessible
+* unified button looks
+* updated and cleaned up markup in README
+
+## v1.1.2
+
+Released 2018-04-25
+
+* fixed crash when saving settings with JS disabled
+
+## v1.1.1
+
+Released 2018-04-19
+
+* rewrote post-receive hook so it would play nice with versioneer
+
+## v1.1.0
+
+Released 2018-01-31
+
+* three types of policies are now available for favs and media (keep only, delete only, ignore)
+* a new input type was introduced to avoid having messy inline radio buttons
+* sentry js init file now has 1 hour of caching
+* fav and reblog count are now stored, for GH-7
+* GH-17 reblogs are deleted regardless of media and favs
+* mastodon instance popularity scoring has been simplified
+
+## v1.0.0
+
+* image proxy now respects max-age from cache-control header
+* image proxy now stores a handful of whitelisted headers
+* privacy policy moved to its own page
+* only one copy of each task+args can run at once
+* fix Error returning to forget after cancelling authorization #14
+* a whole lot of trying to not hit rate limits
+* removed flask-limiter
+* a whole buncha minor changes and fixes that i don't remember because i'm writing this after the fact 🤷
+
+## v0.0.10
+
+* a test suite (it only tests libbrotli for now)
+* an image proxy for those avatars that are served over http not https
+* show a message to the user when their account has been
+  administratively disabled to explain why
+* whjole lot of quality of life improvements
+* whole lot of bug fixes
+* some stylistic changes
+
+## v0.0.9
+
+* logged in page now shows time of last delete and next delete
+* enabling/disabling doesnt require a refresh anymore
+* security enhancements (A+ on moz observatory binchhhhh)
+* bug fixes etc
+
+## v0.0.8
+
+* quick log-in buttons for popular mastodon instances
+* add csrf tokens
+* bug fixes
+
+## v0.0.7
+
+* add option for mastodon users to preserve direct messages (enabled by default)
+* removed storing the posts' bodies. it was convenient for debugging early on but now it's kinda iffy privacy wise
+* various fixes for mastodon
+
+## before v0.0.7
+
+idk

Dockerfile

@@ -0,0 +1,51 @@
+# syntax=docker/dockerfile:1.4.2
+
+FROM python:3.10.6-bullseye AS pydeps
+WORKDIR /usr/src/app
+
+RUN --mount=type=cache,target=/root/.cache/pip/http pip install --upgrade pip==22.2.2
+
+COPY requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/pip/http pip install -r requirements.txt
+
+
+FROM pydeps AS pynodedeps
+
+RUN rm -f /etc/apt/apt.conf.d/docker-clean &&\
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' \
+    > /etc/apt/apt.conf.d/keep-cache
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt-get update -qq && apt-get install -qq nodejs npm
+
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,target=/root/.npm npm clean-install
+
+
+FROM scratch AS layer-cake
+WORKDIR /
+
+COPY *.py setup.cfg rollup.config.js ./
+COPY assets assets
+COPY components components
+COPY libforget libforget
+COPY migrations migrations
+COPY routes routes
+COPY static static
+COPY templates templates
+
+
+FROM pynodedeps AS build
+
+COPY --from=layer-cake / ./
+RUN doit
+
+FROM pydeps
+
+COPY --from=build /usr/src/app ./
+
+COPY .git/ .git/
+
+ENV FLASK_APP=forget.py
+
+VOLUME ["/var/run/celery"]

MANIFEST.in

@@ -0,0 +1,2 @@
+include versioneer.py
+include version.py

Pipfile

@@ -0,0 +1,38 @@
+[[source]]
+
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+
+[packages]
+
+alembic = "*"
+brotli = ">=1.0.1"
+#celery = "~=4.4.2"
+celery = "*"
+csscompressor = "*"
+doit = "*"
+flask = ">=1.1"
+flask-migrate = "*"
+flask-sqlalchemy = "*"
+gunicorn = ">=19.8"
+honcho = "*"
+pillow = "*"
+"psycopg2" = "*"
+raven = "*"
+redis = "*"
+requests = "*"
+sqlalchemy = "*"
+twitter = "*"
+"mastodon.py" = ">=1.2"
+blinker = "*"
+
+
+[dev-packages]
+
+coverage = "*"
+codecov = "*"
+pytest = "*"
+pytest-cov = "*"
+versioneer = "*"

Pipfile.lock

@@ -0,0 +1,876 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "7247d712fbaff173d8cc9dd7d1bd235eb1f45354c550090b0e96358beee58ea7"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "alembic": {
+            "hashes": [
+                "sha256:6c0c05e9768a896d804387e20b299880fe01bc56484246b0dffe8075d6d3d847",
+                "sha256:ad842f2c3ab5c5d4861232730779c05e33db4ba880a08b85eb505e87c01095bc"
+            ],
+            "index": "pypi",
+            "version": "==1.7.6"
+        },
+        "amqp": {
+            "hashes": [
+                "sha256:1e5f707424e544078ca196e72ae6a14887ce74e02bd126be54b7c03c971bef18",
+                "sha256:9cd81f7b023fc04bbb108718fbac674f06901b77bfcdce85b10e2a5d0ee91be5"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==5.0.9"
+        },
+        "billiard": {
+            "hashes": [
+                "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547",
+                "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b"
+            ],
+            "version": "==3.6.4.0"
+        },
+        "blinker": {
+            "hashes": [
+                "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"
+            ],
+            "index": "pypi",
+            "version": "==1.4"
+        },
+        "blurhash": {
+            "hashes": [
+                "sha256:7611c1bc41383d2349b6129208587b5d61e8792ce953893cb49c38beeb400d1d",
+                "sha256:da56b163e5a816e4ad07172f5639287698e09d7f3dc38d18d9726d9c1dbc4cee"
+            ],
+            "version": "==1.1.4"
+        },
+        "brotli": {
+            "hashes": [
+                "sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
+                "sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
+                "sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
+                "sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c",
+                "sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c",
+                "sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70",
+                "sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f",
+                "sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
+                "sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
+                "sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
+                "sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
+                "sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
+                "sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
+                "sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
+                "sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
+                "sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
+                "sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6",
+                "sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438",
+                "sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f",
+                "sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
+                "sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
+                "sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
+                "sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
+                "sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
+                "sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
+                "sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430",
+                "sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
+                "sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
+                "sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
+                "sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
+                "sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
+                "sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
+                "sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c",
+                "sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761",
+                "sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649",
+                "sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
+                "sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
+                "sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
+                "sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
+                "sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
+                "sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
+                "sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5",
+                "sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7",
+                "sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d",
+                "sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
+                "sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
+                "sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
+                "sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
+                "sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
+                "sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
+                "sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
+                "sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
+                "sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
+                "sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
+                "sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
+                "sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
+                "sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
+                "sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
+                "sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
+                "sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806",
+                "sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3",
+                "sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1"
+            ],
+            "index": "pypi",
+            "version": "==1.0.9"
+        },
+        "celery": {
+            "hashes": [
+                "sha256:8aacd02fc23a02760686d63dde1eb0daa9f594e735e73ea8fb15c2ff15cb608c",
+                "sha256:e2cd41667ad97d4f6a2f4672d1c6a6ebada194c619253058b5f23704aaadaa82"
+            ],
+            "index": "pypi",
+            "version": "==5.2.3"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
+                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
+            ],
+            "version": "==2021.10.8"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597",
+                "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.12"
+        },
+        "click": {
+            "hashes": [
+                "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1",
+                "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==8.0.4"
+        },
+        "click-didyoumean": {
+            "hashes": [
+                "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667",
+                "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"
+            ],
+            "markers": "python_version < '4' and python_full_version >= '3.6.2'",
+            "version": "==0.3.0"
+        },
+        "click-plugins": {
+            "hashes": [
+                "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b",
+                "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"
+            ],
+            "version": "==1.1.1"
+        },
+        "click-repl": {
+            "hashes": [
+                "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b",
+                "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8"
+            ],
+            "version": "==0.2.0"
+        },
+        "cloudpickle": {
+            "hashes": [
+                "sha256:5cd02f3b417a783ba84a4ec3e290ff7929009fe51f6405423cfccfadd43ba4a4",
+                "sha256:6b2df9741d06f43839a3275c4e6632f7df6487a1f181f5f46a052d3c917c3d11"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.0.0"
+        },
+        "csscompressor": {
+            "hashes": [
+                "sha256:afa22badbcf3120a4f392e4d22f9fff485c044a1feda4a950ecc5eba9dd31a05"
+            ],
+            "index": "pypi",
+            "version": "==0.9.5"
+        },
+        "decorator": {
+            "hashes": [
+                "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330",
+                "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==5.1.1"
+        },
+        "deprecated": {
+            "hashes": [
+                "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d",
+                "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.2.13"
+        },
+        "doit": {
+            "hashes": [
+                "sha256:388111f8a6a5b3b44620ceeb287d0585927c85d0d6d3f8790d34fbb45c0cf6c3",
+                "sha256:a7f414de4c596a96a727890e0792709a103f1af48e8180e27e07862fff781238"
+            ],
+            "index": "pypi",
+            "version": "==0.34.2"
+        },
+        "flask": {
+            "hashes": [
+                "sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f",
+                "sha256:e1120c228ca2f553b470df4a5fa927ab66258467526069981b3eb0a91902687d"
+            ],
+            "index": "pypi",
+            "version": "==2.0.3"
+        },
+        "flask-migrate": {
+            "hashes": [
+                "sha256:57d6060839e3a7f150eaab6fe4e726d9e3e7cffe2150fb223d73f92421c6d1d9",
+                "sha256:a6498706241aba6be7a251078de9cf166d74307bca41a4ca3e403c9d39e2f897"
+            ],
+            "index": "pypi",
+            "version": "==3.1.0"
+        },
+        "flask-sqlalchemy": {
+            "hashes": [
+                "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912",
+                "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390"
+            ],
+            "index": "pypi",
+            "version": "==2.5.1"
+        },
+        "greenlet": {
+            "hashes": [
+                "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3",
+                "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711",
+                "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd",
+                "sha256:049fe7579230e44daef03a259faa24511d10ebfa44f69411d99e6a184fe68073",
+                "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708",
+                "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67",
+                "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23",
+                "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1",
+                "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08",
+                "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd",
+                "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2",
+                "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa",
+                "sha256:356b3576ad078c89a6107caa9c50cc14e98e3a6c4874a37c3e0273e4baf33de8",
+                "sha256:40b951f601af999a8bf2ce8c71e8aaa4e8c6f78ff8afae7b808aae2dc50d4c40",
+                "sha256:572e1787d1460da79590bf44304abbc0a2da944ea64ec549188fa84d89bba7ab",
+                "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6",
+                "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc",
+                "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b",
+                "sha256:7418b6bfc7fe3331541b84bb2141c9baf1ec7132a7ecd9f375912eca810e714e",
+                "sha256:7cbd7574ce8e138bda9df4efc6bf2ab8572c9aff640d8ecfece1b006b68da963",
+                "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3",
+                "sha256:833e1551925ed51e6b44c800e71e77dacd7e49181fdc9ac9a0bf3714d515785d",
+                "sha256:8639cadfda96737427330a094476d4c7a56ac03de7265622fcf4cfe57c8ae18d",
+                "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe",
+                "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28",
+                "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3",
+                "sha256:903bbd302a2378f984aef528f76d4c9b1748f318fe1294961c072bdc7f2ffa3e",
+                "sha256:93f81b134a165cc17123626ab8da2e30c0455441d4ab5576eed73a64c025b25c",
+                "sha256:95e69877983ea39b7303570fa6760f81a3eec23d0e3ab2021b7144b94d06202d",
+                "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0",
+                "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497",
+                "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee",
+                "sha256:aa5b467f15e78b82257319aebc78dd2915e4c1436c3c0d1ad6f53e47ba6e2713",
+                "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58",
+                "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a",
+                "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06",
+                "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88",
+                "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965",
+                "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f",
+                "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4",
+                "sha256:be5f425ff1f5f4b3c1e33ad64ab994eed12fc284a6ea71c5243fd564502ecbe5",
+                "sha256:dd0b1e9e891f69e7675ba5c92e28b90eaa045f6ab134ffe70b52e948aa175b3c",
+                "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a",
+                "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1",
+                "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43",
+                "sha256:eb6ea6da4c787111adf40f697b4e58732ee0942b5d3bd8f435277643329ba627",
+                "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b",
+                "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168",
+                "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d",
+                "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5",
+                "sha256:f3acda1924472472ddd60c29e5b9db0cec629fbe3c5c5accb74d6d6d14773478",
+                "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf",
+                "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce",
+                "sha256:fa877ca7f6b48054f847b61d6fa7bed5cebb663ebc55e018fda12db09dcc664c",
+                "sha256:fdcec0b8399108577ec290f55551d926d9a1fa6cad45882093a7a07ac5ec147b"
+            ],
+            "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
+            "version": "==1.1.2"
+        },
+        "gunicorn": {
+            "hashes": [
+                "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e",
+                "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"
+            ],
+            "index": "pypi",
+            "version": "==20.1.0"
+        },
+        "honcho": {
+            "hashes": [
+                "sha256:a4d6e3a88a7b51b66351ecfc6e9d79d8f4b87351db9ad7e923f5632cc498122f",
+                "sha256:c5eca0bded4bef6697a23aec0422fd4f6508ea3581979a3485fc4b89357eb2a9"
+            ],
+            "index": "pypi",
+            "version": "==1.1.0"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.3"
+        },
+        "itsdangerous": {
+            "hashes": [
+                "sha256:29285842166554469a56d427addc0843914172343784cb909695fdbe90a3e129",
+                "sha256:d848fcb8bc7d507c4546b448574e8a44fc4ea2ba84ebf8d783290d53e81992f5"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.0"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8",
+                "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.3"
+        },
+        "kombu": {
+            "hashes": [
+                "sha256:81a90c1de97e08d3db37dbf163eaaf667445e1068c98bfd89f051a40e9f6dbbd",
+                "sha256:eeaeb8024f3a5cfc71c9250e45cddb8493f269d74ada2f74909a93c59c4b4179"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.2.3"
+        },
+        "mako": {
+            "hashes": [
+                "sha256:4e9e345a41924a954251b95b4b28e14a301145b544901332e658907a7464b6b2",
+                "sha256:afaf8e515d075b22fad7d7b8b30e4a1c90624ff2f3733a06ec125f5a5f043a57"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.1.6"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:023af8c54fe63530545f70dd2a2a7eed18d07a9a77b94e8bf1e2ff7f252db9a3",
+                "sha256:09c86c9643cceb1d87ca08cdc30160d1b7ab49a8a21564868921959bd16441b8",
+                "sha256:142119fb14a1ef6d758912b25c4e803c3ff66920635c44078666fe7cc3f8f759",
+                "sha256:1d1fb9b2eec3c9714dd936860850300b51dbaa37404209c8d4cb66547884b7ed",
+                "sha256:204730fd5fe2fe3b1e9ccadb2bd18ba8712b111dcabce185af0b3b5285a7c989",
+                "sha256:24c3be29abb6b34052fd26fc7a8e0a49b1ee9d282e3665e8ad09a0a68faee5b3",
+                "sha256:290b02bab3c9e216da57c1d11d2ba73a9f73a614bbdcc027d299a60cdfabb11a",
+                "sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c",
+                "sha256:30c653fde75a6e5eb814d2a0a89378f83d1d3f502ab710904ee585c38888816c",
+                "sha256:3cace1837bc84e63b3fd2dfce37f08f8c18aeb81ef5cf6bb9b51f625cb4e6cd8",
+                "sha256:4056f752015dfa9828dce3140dbadd543b555afb3252507348c493def166d454",
+                "sha256:454ffc1cbb75227d15667c09f164a0099159da0c1f3d2636aa648f12675491ad",
+                "sha256:598b65d74615c021423bd45c2bc5e9b59539c875a9bdb7e5f2a6b92dfcfc268d",
+                "sha256:599941da468f2cf22bf90a84f6e2a65524e87be2fce844f96f2dd9a6c9d1e635",
+                "sha256:5ddea4c352a488b5e1069069f2f501006b1a4362cb906bee9a193ef1245a7a61",
+                "sha256:62c0285e91414f5c8f621a17b69fc0088394ccdaa961ef469e833dbff64bd5ea",
+                "sha256:679cbb78914ab212c49c67ba2c7396dc599a8479de51b9a87b174700abd9ea49",
+                "sha256:6e104c0c2b4cd765b4e83909cde7ec61a1e313f8a75775897db321450e928cce",
+                "sha256:736895a020e31b428b3382a7887bfea96102c529530299f426bf2e636aacec9e",
+                "sha256:75bb36f134883fdbe13d8e63b8675f5f12b80bb6627f7714c7d6c5becf22719f",
+                "sha256:7d2f5d97fcbd004c03df8d8fe2b973fe2b14e7bfeb2cfa012eaa8759ce9a762f",
+                "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f",
+                "sha256:84ad5e29bf8bab3ad70fd707d3c05524862bddc54dc040982b0dbcff36481de7",
+                "sha256:8da5924cb1f9064589767b0f3fc39d03e3d0fb5aa29e0cb21d43106519bd624a",
+                "sha256:961eb86e5be7d0973789f30ebcf6caab60b844203f4396ece27310295a6082c7",
+                "sha256:96de1932237abe0a13ba68b63e94113678c379dca45afa040a17b6e1ad7ed076",
+                "sha256:a0a0abef2ca47b33fb615b491ce31b055ef2430de52c5b3fb19a4042dbc5cadb",
+                "sha256:b2a5a856019d2833c56a3dcac1b80fe795c95f401818ea963594b345929dffa7",
+                "sha256:b8811d48078d1cf2a6863dafb896e68406c5f513048451cd2ded0473133473c7",
+                "sha256:c532d5ab79be0199fa2658e24a02fce8542df196e60665dd322409a03db6a52c",
+                "sha256:d3b64c65328cb4cd252c94f83e66e3d7acf8891e60ebf588d7b493a55a1dbf26",
+                "sha256:d4e702eea4a2903441f2735799d217f4ac1b55f7d8ad96ab7d4e25417cb0827c",
+                "sha256:d5653619b3eb5cbd35bfba3c12d575db2a74d15e0e1c08bf1db788069d410ce8",
+                "sha256:d66624f04de4af8bbf1c7f21cc06649c1c69a7f84109179add573ce35e46d448",
+                "sha256:e67ec74fada3841b8c5f4c4f197bea916025cb9aa3fe5abf7d52b655d042f956",
+                "sha256:e6f7f3f41faffaea6596da86ecc2389672fa949bd035251eab26dc6697451d05",
+                "sha256:f02cf7221d5cd915d7fa58ab64f7ee6dd0f6cddbb48683debf5d04ae9b1c2cc1",
+                "sha256:f0eddfcabd6936558ec020130f932d479930581171368fd728efcfb6ef0dd357",
+                "sha256:fabbe18087c3d33c5824cb145ffca52eccd053061df1d79d4b66dafa5ad2a5ea",
+                "sha256:fc3150f85e2dbcf99e65238c842d1cfe69d3e7649b19864c1cc043213d9cd730"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.0"
+        },
+        "mastodon.py": {
+            "hashes": [
+                "sha256:2afddbad8b5d7326fcc8a8f8c62bfe956e34627f516b06c6694fc8c8fedc33ee",
+                "sha256:cc454cac0ed1ae4f105f7399ea53f5b31a1be5075d1882f47162d2e78a9e4064"
+            ],
+            "index": "pypi",
+            "version": "==1.5.1"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.3"
+        },
+        "pillow": {
+            "hashes": [
+                "sha256:011233e0c42a4a7836498e98c1acf5e744c96a67dd5032a6f666cc1fb97eab97",
+                "sha256:0f29d831e2151e0b7b39981756d201f7108d3d215896212ffe2e992d06bfe049",
+                "sha256:12875d118f21cf35604176872447cdb57b07126750a33748bac15e77f90f1f9c",
+                "sha256:14d4b1341ac07ae07eb2cc682f459bec932a380c3b122f5540432d8977e64eae",
+                "sha256:1c3c33ac69cf059bbb9d1a71eeaba76781b450bc307e2291f8a4764d779a6b28",
+                "sha256:1d19397351f73a88904ad1aee421e800fe4bbcd1aeee6435fb62d0a05ccd1030",
+                "sha256:253e8a302a96df6927310a9d44e6103055e8fb96a6822f8b7f514bb7ef77de56",
+                "sha256:2632d0f846b7c7600edf53c48f8f9f1e13e62f66a6dbc15191029d950bfed976",
+                "sha256:335ace1a22325395c4ea88e00ba3dc89ca029bd66bd5a3c382d53e44f0ccd77e",
+                "sha256:413ce0bbf9fc6278b2d63309dfeefe452835e1c78398efb431bab0672fe9274e",
+                "sha256:5100b45a4638e3c00e4d2320d3193bdabb2d75e79793af7c3eb139e4f569f16f",
+                "sha256:514ceac913076feefbeaf89771fd6febde78b0c4c1b23aaeab082c41c694e81b",
+                "sha256:528a2a692c65dd5cafc130de286030af251d2ee0483a5bf50c9348aefe834e8a",
+                "sha256:6295f6763749b89c994fcb6d8a7f7ce03c3992e695f89f00b741b4580b199b7e",
+                "sha256:6c8bc8238a7dfdaf7a75f5ec5a663f4173f8c367e5a39f87e720495e1eed75fa",
+                "sha256:718856856ba31f14f13ba885ff13874be7fefc53984d2832458f12c38205f7f7",
+                "sha256:7f7609a718b177bf171ac93cea9fd2ddc0e03e84d8fa4e887bdfc39671d46b00",
+                "sha256:80ca33961ced9c63358056bd08403ff866512038883e74f3a4bf88ad3eb66838",
+                "sha256:80fe64a6deb6fcfdf7b8386f2cf216d329be6f2781f7d90304351811fb591360",
+                "sha256:81c4b81611e3a3cb30e59b0cf05b888c675f97e3adb2c8672c3154047980726b",
+                "sha256:855c583f268edde09474b081e3ddcd5cf3b20c12f26e0d434e1386cc5d318e7a",
+                "sha256:9bfdb82cdfeccec50aad441afc332faf8606dfa5e8efd18a6692b5d6e79f00fd",
+                "sha256:a5d24e1d674dd9d72c66ad3ea9131322819ff86250b30dc5821cbafcfa0b96b4",
+                "sha256:a9f44cd7e162ac6191491d7249cceb02b8116b0f7e847ee33f739d7cb1ea1f70",
+                "sha256:b5b3f092fe345c03bca1e0b687dfbb39364b21ebb8ba90e3fa707374b7915204",
+                "sha256:b9618823bd237c0d2575283f2939655f54d51b4527ec3972907a927acbcc5bfc",
+                "sha256:cef9c85ccbe9bee00909758936ea841ef12035296c748aaceee535969e27d31b",
+                "sha256:d21237d0cd37acded35154e29aec853e945950321dd2ffd1a7d86fe686814669",
+                "sha256:d3c5c79ab7dfce6d88f1ba639b77e77a17ea33a01b07b99840d6ed08031cb2a7",
+                "sha256:d9d7942b624b04b895cb95af03a23407f17646815495ce4547f0e60e0b06f58e",
+                "sha256:db6d9fac65bd08cea7f3540b899977c6dee9edad959fa4eaf305940d9cbd861c",
+                "sha256:ede5af4a2702444a832a800b8eb7f0a7a1c0eed55b644642e049c98d589e5092",
+                "sha256:effb7749713d5317478bb3acb3f81d9d7c7f86726d41c1facca068a04cf5bb4c",
+                "sha256:f154d173286a5d1863637a7dcd8c3437bb557520b01bddb0be0258dcb72696b5",
+                "sha256:f25ed6e28ddf50de7e7ea99d7a976d6a9c415f03adcaac9c41ff6ff41b6d86ac"
+            ],
+            "index": "pypi",
+            "version": "==9.0.1"
+        },
+        "prompt-toolkit": {
+            "hashes": [
+                "sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c",
+                "sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650"
+            ],
+            "markers": "python_full_version >= '3.6.2'",
+            "version": "==3.0.28"
+        },
+        "psycopg2": {
+            "hashes": [
+                "sha256:06f32425949bd5fe8f625c49f17ebb9784e1e4fe928b7cce72edc36fb68e4c0c",
+                "sha256:0762c27d018edbcb2d34d51596e4346c983bd27c330218c56c4dc25ef7e819bf",
+                "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362",
+                "sha256:34b33e0162cfcaad151f249c2649fd1030010c16f4bbc40a604c1cb77173dcf7",
+                "sha256:4295093a6ae3434d33ec6baab4ca5512a5082cc43c0505293087b8a46d108461",
+                "sha256:8cf3878353cc04b053822896bc4922b194792df9df2f1ad8da01fb3043602126",
+                "sha256:8e841d1bf3434da985cc5ef13e6f75c8981ced601fd70cc6bf33351b91562981",
+                "sha256:9572e08b50aed176ef6d66f15a21d823bb6f6d23152d35e8451d7d2d18fdac56",
+                "sha256:a81e3866f99382dfe8c15a151f1ca5fde5815fde879348fe5a9884a7c092a305",
+                "sha256:cb10d44e6694d763fa1078a26f7f6137d69f555a78ec85dc2ef716c37447e4b2",
+                "sha256:d3ca6421b942f60c008f81a3541e8faf6865a28d5a9b48544b0ee4f40cac7fca"
+            ],
+            "index": "pypi",
+            "version": "==2.9.3"
+        },
+        "pyinotify": {
+            "hashes": [
+                "sha256:9c998a5d7606ca835065cdabc013ae6c66eb9ea76a00a1e3bc6e0cfe2b4f71f4"
+            ],
+            "markers": "sys_platform == 'linux'",
+            "version": "==0.9.6"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
+                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.7"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.8.2"
+        },
+        "python-magic": {
+            "hashes": [
+                "sha256:1a2c81e8f395c744536369790bd75094665e9644110a6623bcc3bbea30f03973",
+                "sha256:21f5f542aa0330f5c8a64442528542f6215c8e18d2466b399b0d9d39356d83fc"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.4.25"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c",
+                "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"
+            ],
+            "version": "==2021.3"
+        },
+        "raven": {
+            "hashes": [
+                "sha256:3fa6de6efa2493a7c827472e984ce9b020797d0da16f1db67197bcc23c8fae54",
+                "sha256:44a13f87670836e153951af9a3c80405d36b43097db869a36e92809673692ce4"
+            ],
+            "index": "pypi",
+            "version": "==6.10.0"
+        },
+        "redis": {
+            "hashes": [
+                "sha256:04629f8e42be942c4f7d1812f2094568f04c612865ad19ad3ace3005da70631a",
+                "sha256:1d9a0cdf89fdd93f84261733e24f55a7bbd413a9b219fdaf56e3e728ca9a2306"
+            ],
+            "index": "pypi",
+            "version": "==4.1.4"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
+                "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
+            ],
+            "index": "pypi",
+            "version": "==2.27.1"
+        },
+        "setuptools": {
+            "hashes": [
+                "sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373",
+                "sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==59.6.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.16.0"
+        },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:05fa14f279d43df68964ad066f653193187909950aa0163320b728edfc400167",
+                "sha256:0ddc5e5ccc0160e7ad190e5c61eb57560f38559e22586955f205e537cda26034",
+                "sha256:15a03261aa1e68f208e71ae3cd845b00063d242cbf8c87348a0c2c0fc6e1f2ac",
+                "sha256:289465162b1fa1e7a982f8abe59d26a8331211cad4942e8031d2b7db1f75e649",
+                "sha256:2e216c13ecc7fcdcbb86bb3225425b3ed338e43a8810c7089ddb472676124b9b",
+                "sha256:2fd4d3ca64c41dae31228b80556ab55b6489275fb204827f6560b65f95692cf3",
+                "sha256:330eb45395874cc7787214fdd4489e2afb931bc49e0a7a8f9cd56d6e9c5b1639",
+                "sha256:3c7ed6c69debaf6198fadb1c16ae1253a29a7670bbf0646f92582eb465a0b999",
+                "sha256:4ad31cec8b49fd718470328ad9711f4dc703507d434fd45461096da0a7135ee0",
+                "sha256:57205844f246bab9b666a32f59b046add8995c665d9ecb2b7b837b087df90639",
+                "sha256:582b59d1e5780a447aada22b461e50b404a9dc05768da1d87368ad8190468418",
+                "sha256:5e9c7b3567edbc2183607f7d9f3e7e89355b8f8984eec4d2cd1e1513c8f7b43f",
+                "sha256:6a01ec49ca54ce03bc14e10de55dfc64187a2194b3b0e5ac0fdbe9b24767e79e",
+                "sha256:6f22c040d196f841168b1456e77c30a18a3dc16b336ddbc5a24ce01ab4e95ae0",
+                "sha256:81f2dd355b57770fdf292b54f3e0a9823ec27a543f947fa2eb4ec0df44f35f0d",
+                "sha256:85e4c244e1de056d48dae466e9baf9437980c19fcde493e0db1a0a986e6d75b4",
+                "sha256:8d0949b11681380b4a50ac3cd075e4816afe9fa4a8c8ae006c1ca26f0fa40ad8",
+                "sha256:975f5c0793892c634c4920057da0de3a48bbbbd0a5c86f5fcf2f2fedf41b76da",
+                "sha256:9e4fb2895b83993831ba2401b6404de953fdbfa9d7d4fa6a4756294a83bbc94f",
+                "sha256:b35dca159c1c9fa8a5f9005e42133eed82705bf8e243da371a5e5826440e65ca",
+                "sha256:b7b20c88873675903d6438d8b33fba027997193e274b9367421e610d9da76c08",
+                "sha256:bb4b15fb1f0aafa65cbdc62d3c2078bea1ceecbfccc9a1f23a2113c9ac1191fa",
+                "sha256:c0c7171aa5a57e522a04a31b84798b6c926234cb559c0939840c3235cf068813",
+                "sha256:c317ddd7c586af350a6aef22b891e84b16bff1a27886ed5b30f15c1ed59caeaa",
+                "sha256:c3abc34fed19fdeaead0ced8cf56dd121f08198008c033596aa6aae7cc58f59f",
+                "sha256:ca68c52e3cae491ace2bf39b35fef4ce26c192fd70b4cd90f040d419f70893b5",
+                "sha256:cf2cd387409b12d0a8b801610d6336ee7d24043b6dd965950eaec09b73e7262f",
+                "sha256:d046a9aeba9bc53e88a41e58beb72b6205abb9a20f6c136161adf9128e589db5",
+                "sha256:d5c20c8415173b119762b6110af64448adccd4d11f273fb9f718a9865b88a99c",
+                "sha256:d86132922531f0dc5a4f424c7580a472a924dd737602638e704841c9cb24aea2",
+                "sha256:dccff41478050e823271642837b904d5f9bda3f5cf7d371ce163f00a694118d6",
+                "sha256:de85c26a5a1c72e695ab0454e92f60213b4459b8d7c502e0be7a6369690eeb1a",
+                "sha256:e3a86b59b6227ef72ffc10d4b23f0fe994bef64d4667eab4fb8cd43de4223bec",
+                "sha256:e79e73d5ee24196d3057340e356e6254af4d10e1fc22d3207ea8342fc5ffb977",
+                "sha256:ea8210090a816d48a4291a47462bac750e3bc5c2442e6d64f7b8137a7c3f9ac5",
+                "sha256:f3b7ec97e68b68cb1f9ddb82eda17b418f19a034fa8380a0ac04e8fe01532875"
+            ],
+            "index": "pypi",
+            "version": "==1.4.31"
+        },
+        "twitter": {
+            "hashes": [
+                "sha256:06eac7ee7f2a14ddeb680671ff07450984f6d254334f5db8dd69547dd1e179c5",
+                "sha256:a56ff9575fbd50a51ce91107dcb5a4c3fd00c2ba1bcb172ce538b0948d3626e6"
+            ],
+            "index": "pypi",
+            "version": "==1.19.3"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
+                "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.8"
+        },
+        "vine": {
+            "hashes": [
+                "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30",
+                "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==5.0.0"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
+                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+            ],
+            "version": "==0.2.5"
+        },
+        "werkzeug": {
+            "hashes": [
+                "sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8",
+                "sha256:b863f8ff057c522164b6067c9e28b041161b4be5ba4d0daceeaa50a163822d3c"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.0.3"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:086218a72ec7d986a3eddb7707c8c4526d677c7b35e355875a0fe2918b059179",
+                "sha256:0877fe981fd76b183711d767500e6b3111378ed2043c145e21816ee589d91096",
+                "sha256:0a017a667d1f7411816e4bf214646d0ad5b1da2c1ea13dec6c162736ff25a374",
+                "sha256:0cb23d36ed03bf46b894cfec777eec754146d68429c30431c99ef28482b5c1df",
+                "sha256:1fea9cd438686e6682271d36f3481a9f3636195578bab9ca3382e2f5f01fc185",
+                "sha256:220a869982ea9023e163ba915077816ca439489de6d2c09089b219f4e11b6785",
+                "sha256:25b1b1d5df495d82be1c9d2fad408f7ce5ca8a38085e2da41bb63c914baadff7",
+                "sha256:2dded5496e8f1592ec27079b28b6ad2a1ef0b9296d270f77b8e4a3a796cf6909",
+                "sha256:2ebdde19cd3c8cdf8df3fc165bc7827334bc4e353465048b36f7deeae8ee0918",
+                "sha256:43e69ffe47e3609a6aec0fe723001c60c65305784d964f5007d5b4fb1bc6bf33",
+                "sha256:46f7f3af321a573fc0c3586612db4decb7eb37172af1bc6173d81f5b66c2e068",
+                "sha256:47f0a183743e7f71f29e4e21574ad3fa95676136f45b91afcf83f6a050914829",
+                "sha256:498e6217523111d07cd67e87a791f5e9ee769f9241fcf8a379696e25806965af",
+                "sha256:4b9c458732450ec42578b5642ac53e312092acf8c0bfce140ada5ca1ac556f79",
+                "sha256:51799ca950cfee9396a87f4a1240622ac38973b6df5ef7a41e7f0b98797099ce",
+                "sha256:5601f44a0f38fed36cc07db004f0eedeaadbdcec90e4e90509480e7e6060a5bc",
+                "sha256:5f223101f21cfd41deec8ce3889dc59f88a59b409db028c469c9b20cfeefbe36",
+                "sha256:610f5f83dd1e0ad40254c306f4764fcdc846641f120c3cf424ff57a19d5f7ade",
+                "sha256:6a03d9917aee887690aa3f1747ce634e610f6db6f6b332b35c2dd89412912bca",
+                "sha256:705e2af1f7be4707e49ced9153f8d72131090e52be9278b5dbb1498c749a1e32",
+                "sha256:766b32c762e07e26f50d8a3468e3b4228b3736c805018e4b0ec8cc01ecd88125",
+                "sha256:77416e6b17926d953b5c666a3cb718d5945df63ecf922af0ee576206d7033b5e",
+                "sha256:778fd096ee96890c10ce96187c76b3e99b2da44e08c9e24d5652f356873f6709",
+                "sha256:78dea98c81915bbf510eb6a3c9c24915e4660302937b9ae05a0947164248020f",
+                "sha256:7dd215e4e8514004c8d810a73e342c536547038fb130205ec4bba9f5de35d45b",
+                "sha256:7dde79d007cd6dfa65afe404766057c2409316135cb892be4b1c768e3f3a11cb",
+                "sha256:81bd7c90d28a4b2e1df135bfbd7c23aee3050078ca6441bead44c42483f9ebfb",
+                "sha256:85148f4225287b6a0665eef08a178c15097366d46b210574a658c1ff5b377489",
+                "sha256:865c0b50003616f05858b22174c40ffc27a38e67359fa1495605f96125f76640",
+                "sha256:87883690cae293541e08ba2da22cacaae0a092e0ed56bbba8d018cc486fbafbb",
+                "sha256:8aab36778fa9bba1a8f06a4919556f9f8c7b33102bd71b3ab307bb3fecb21851",
+                "sha256:8c73c1a2ec7c98d7eaded149f6d225a692caa1bd7b2401a14125446e9e90410d",
+                "sha256:936503cb0a6ed28dbfa87e8fcd0a56458822144e9d11a49ccee6d9a8adb2ac44",
+                "sha256:944b180f61f5e36c0634d3202ba8509b986b5fbaf57db3e94df11abee244ba13",
+                "sha256:96b81ae75591a795d8c90edc0bfaab44d3d41ffc1aae4d994c5aa21d9b8e19a2",
+                "sha256:981da26722bebb9247a0601e2922cedf8bb7a600e89c852d063313102de6f2cb",
+                "sha256:ae9de71eb60940e58207f8e71fe113c639da42adb02fb2bcbcaccc1ccecd092b",
+                "sha256:b73d4b78807bd299b38e4598b8e7bd34ed55d480160d2e7fdaabd9931afa65f9",
+                "sha256:d4a5f6146cfa5c7ba0134249665acd322a70d1ea61732723c7d3e8cc0fa80755",
+                "sha256:dd91006848eb55af2159375134d724032a2d1d13bcc6f81cd8d3ed9f2b8e846c",
+                "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a",
+                "sha256:e6906d6f48437dfd80464f7d7af1740eadc572b9f7a4301e7dd3d65db285cacf",
+                "sha256:e92d0d4fa68ea0c02d39f1e2f9cb5bc4b4a71e8c442207433d8db47ee79d7aa3",
+                "sha256:e94b7d9deaa4cc7bac9198a58a7240aaf87fe56c6277ee25fa5b3aa1edebd229",
+                "sha256:ea3e746e29d4000cd98d572f3ee2a6050a4f784bb536f4ac1f035987fc1ed83e",
+                "sha256:ec7e20258ecc5174029a0f391e1b948bf2906cd64c198a9b8b281b811cbc04de",
+                "sha256:ec9465dd69d5657b5d2fa6133b3e1e989ae27d29471a672416fd729b429eb554",
+                "sha256:f122ccd12fdc69628786d0c947bdd9cb2733be8f800d88b5a37c57f1f1d73c10",
+                "sha256:f99c0489258086308aad4ae57da9e8ecf9e1f3f30fa35d5e170b4d4896554d80",
+                "sha256:f9c51d9af9abb899bd34ace878fbec8bf357b3194a10c4e8e0a25512826ef056",
+                "sha256:fd76c47f20984b43d93de9a82011bb6e5f8325df6c9ed4d8310029a55fa361ea"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.13.3"
+        }
+    },
+    "develop": {
+        "attrs": {
+            "hashes": [
+                "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
+                "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==21.4.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
+                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
+            ],
+            "version": "==2021.10.8"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597",
+                "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.12"
+        },
+        "codecov": {
+            "hashes": [
+                "sha256:585dc217dc3d8185198ceb402f85d5cb5dbfa0c5f350a5abcdf9e347776a5b47",
+                "sha256:782a8e5352f22593cbc5427a35320b99490eb24d9dcfa2155fd99d2b75cfb635",
+                "sha256:a0da46bb5025426da895af90938def8ee12d37fcbcbbbc15b6dc64cf7ebc51c1"
+            ],
+            "index": "pypi",
+            "version": "==2.1.12"
+        },
+        "coverage": {
+            "hashes": [
+                "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9",
+                "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d",
+                "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf",
+                "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7",
+                "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6",
+                "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4",
+                "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059",
+                "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39",
+                "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536",
+                "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac",
+                "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c",
+                "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903",
+                "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d",
+                "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05",
+                "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684",
+                "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1",
+                "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f",
+                "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7",
+                "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca",
+                "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad",
+                "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca",
+                "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d",
+                "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92",
+                "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4",
+                "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf",
+                "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6",
+                "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1",
+                "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4",
+                "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359",
+                "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3",
+                "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620",
+                "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512",
+                "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69",
+                "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2",
+                "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518",
+                "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0",
+                "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa",
+                "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4",
+                "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e",
+                "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1",
+                "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2"
+            ],
+            "index": "pypi",
+            "version": "==6.3.2"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.3"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.3"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.11.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
+                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db",
+                "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171"
+            ],
+            "index": "pypi",
+            "version": "==7.0.1"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6",
+                "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"
+            ],
+            "index": "pypi",
+            "version": "==3.0.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
+                "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
+            ],
+            "index": "pypi",
+            "version": "==2.27.1"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
+                "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.8"
+        },
+        "versioneer": {
+            "hashes": [
+                "sha256:1b4900f17b84ce76dbc5d462fe06522ea2ec400945c46dc751abad12db2e7ca6",
+                "sha256:64f2dbcbbed15f9a6da2b85f643997db729cf496cafdb97670fb2fa73a7d8e20"
+            ],
+            "index": "pypi",
+            "version": "==0.21"
+        }
+    }
+}

Procfile

@@ -1,2 +1,3 @@
 web: gunicorn -w 9 -t 3600 -b 127.0.0.1:42157 forget:app
-worker: python tasks.py -B -Ofair
+worker: celery -A tasks worker --autoscale=64,8
+beat: celery -A tasks beat

Procfile.dev

@@ -0,0 +1,3 @@
+web: gunicorn -b 127.0.0.1:5000 --reload --reload-extra-file templates/ -w 4 forget:app
+worker: python tasks.py -B -Ofair
+build: doit auto

README.markdown

@@ -1,38 +1,150 @@
-uhh frick i forgot to write a readme hang on uhh
+![Forget](assets/promo.gif)
 
-# forget
+![Maintenance status](https://img.shields.io/maintenance/no/2022.svg)
 
-its a thing that deletes your posts
+[![Build status](https://img.shields.io/travis/codl/forget.svg)](https://travis-ci.org/codl/forget/)
+[![Test coverage](https://img.shields.io/codecov/c/github/codl/forget.svg)](https://codecov.io/gh/codl/forget)
 
-it works with twitter and maybe sometime in the future it will work with other services
+Forget is a post deleting service for Twitter, Mastodon, and Misskey.
 
-it lives at <https://forget.codl.fr>
 
-you can run your own if you want to, youll need postgresql and redis and python 3.6+
+## Features
+
+  * Delete your posts when they cross an age threshold.
+  * Or keep your post count in check, deleting old posts when you go over.
+  * Preserve old posts that matter by giving them a favourite or a reaction.
+  * Set it and <i>forget</i> it. Forget works continuously in the background.
+
+## Non-features
+
+Forget is not a one-time purging tool. It is designed to prune your account
+continuously, not quickly. If you need a lot of posts gone fast, you may want
+to look for another more-suited tool.
+
+## Running your own
+
+### Requirements
+
+* Postgresql
+* Redis
+* Python 3.6+
+* Node.js 10+
+
+
+### Set up venv
+
+Setting up a venv will isolate Forget from your system's libraries and allow you to install
+dependencies locally as a normal user. It's not necessary but it is recommended!
 
 ```
-$ # set up virtualenv (recommended)
-$ virtualenv venv
+$ python -m venv venv
 $ source venv/bin/activate
+```
 
-$ # install requirements and set up config file
+If you're using `zsh` or `fish` as a shell, substitute `venv/bin/activate` with
+`venv/bin/activate.zsh` or `venv/bin/activate.fish`, respectively.
+
+You will need to "activate" the venv in every new terminal before you can use
+pip or any python tools included in dependencies (honcho, flask...)
+
+### Download and install dependencies
+
+```
 $ pip install -r requirements.txt
+$ npm install
+```
+
+Wow!! Exciting
+
+### Create and complete config file
+
+Gotta set up those, paths, and stuff.
+
+```
 $ cp config.example.py config.py
 $ $EDITOR config.py
+```
 
-$ # set up database schema
-$ createdb forget
+### Set up database schema
+
+If you haven't started postgresql yet now would be a great time to do that.
+
+```
+$ createdb forget # if you havent created the DB yet
 $ env FLASK_APP=forget.py flask db upgrade
+```
 
-$ # build assets
+### Build static assets
+
+Gonna do it...!
+
+```
 $ doit
+```
 
-$ # start web server and background worker
+Done did it.
+
+### Running
+
+The included `Procfile` will run the app server and the background worker.
+`honcho`, a `Procfile` runner, is included as a dependency:
+
+```
 $ honcho start
 ```
 
-the web server will listen on `127.0.0.1:42157`, you'll probably want to proxy with nginx or apache or what have you
+The application server will listen on `http://127.0.0.1:42157`.
+You'll want to use your favourite web server to proxy traffic to it.
 
-sorry this readme sucks i forgot to write one before release
+### Development
 
-send me a tweet [@codl](https://twitter.com/codl) if you're having trouble or, to tell me you like it
+For development, you may want to use `Procfile.dev`, which starts flask in
+debug mode and rebuilds the static assets automatically when they change
+
+```
+$ honcho -f Procfile.dev start
+```
+
+Or you could just look at `Procfile.dev` and run those things manually. It's up
+to you.
+
+You can run the (currently very incomplete) test suite by running `pytest`.
+You'll need redis installed on your development machine, a temporary redis
+server will be started and shut down automatically by the test suite.
+
+## Docker
+
+This project is also able to be deployed through Docker.
+
+1. Copy `config.docker.py` to `config.py` and add additional configurations to
+   your liking.
+1. By default, the webapp container will be listening on `127.0.0.1:42157`,
+   which you can point a reverse proxy at.
+   * If your reverse proxy is in another docker network then you'll need a
+     `docker-compose.override.yml` file to attach the `www` service to the
+     right network and not publish any ports. An example override file is
+     provided. The web app will be listening on `http://forget-www-1:42157`.
+1. By default, the `docker-compose.yml` creates relative mounts `./redis`,
+   `./postgres`, and `./celery` relative to the `docker-compose.yml` location.
+   An example `docker-compose.override.yml` file is provided that shows how to
+   change this.
+1. Run `docker-compose build` to build the image.
+1. Run `docker-compose up` to start or `docker-compose up -d` to start in the
+   background, and use `docker-compose down` to stop.
+
+## Contact
+
+If you're having trouble with Forget, or if you're not having trouble but you
+just want to tell me you like it, you can drop me a note at
+[@codl@chitter.xyz](https://chitter.xyz/@codl) or
+[codl@codl.fr](mailto:codl@codl.fr).
+
+## Greetz
+
+Thank you bea, for making ephemeral, inspiring me to make [limiter][], then this,
+in an attempt to bring ephemeral with me everywhere. ☕
+
+[limiter]: https://github.com/codl/limiter
+
+Thank you to the kind folks who have emailed me to tell me Forget has made their
+time on social media less stressful. 🌻

app.py

@@ -1,30 +1,31 @@
-from flask import Flask, request
+from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy import MetaData
 from flask_migrate import Migrate
 import version
-from lib import cachebust
-from flask_limiter import Limiter
-from flask_limiter.util import get_remote_address
-from lib import get_viewer
+from libforget.cachebust import cachebust
+import mimetypes
+import libforget.brotli
+import libforget.img_proxy
+from werkzeug.middleware.proxy_fix import ProxyFix
 
 app = Flask(__name__)
 
 default_config = {
         "SQLALCHEMY_TRACK_MODIFICATIONS": False,
         "SQLALCHEMY_DATABASE_URI": "postgresql+psycopg2:///forget",
-        "SECRET_KEY": "hunter2",
-        "CELERY_BROKER": "redis://",
         "HTTPS": True,
         "SENTRY_CONFIG": {},
-        "RATELIMIT_STORAGE_URL": "redis://",
+        "REPO_URL": "https://github.com/codl/forget",
+        "CHANGELOG_URL": "https://github.com/codl/forget/blob/{hash}/CHANGELOG.markdown",
+        "REDIS_URI": "redis://",
 }
 
 app.config.update(default_config)
 
 app.config.from_pyfile('config.py', True)
 
-metadata = MetaData(naming_convention = {
+metadata = MetaData(naming_convention={
     "ix": 'ix_%(column_0_label)s',
     "uq": "uq_%(table_name)s_%(column_0_name)s",
     "ck": "ck_%(table_name)s_%(constraint_name)s",
@@ -35,28 +36,64 @@ metadata = MetaData(naming_convention = {
 db = SQLAlchemy(app, metadata=metadata)
 migrate = Migrate(app, db)
 
+if 'CELERY_BROKER' not in app.config:
+    uri = app.config['REDIS_URI']
+    if uri.startswith('unix://'):
+        uri = uri.replace('unix', 'redis+socket', 1)
+    app.config['CELERY_BROKER'] = uri
+
 sentry = None
 if 'SENTRY_DSN' in app.config:
     from raven.contrib.flask import Sentry
-    app.config['SENTRY_CONFIG']['release'] = version.version
+    app.config['SENTRY_CONFIG']['release'] = version.get_versions()['version']
     sentry = Sentry(app, dsn=app.config['SENTRY_DSN'])
 
 url_for = cachebust(app)
 
+
 @app.context_processor
 def inject_static():
     def static(filename, **kwargs):
         return url_for('static', filename=filename, **kwargs)
     return {'st': static}
 
-def rate_limit_key():
-    viewer = get_viewer()
-    if viewer:
-        return viewer.id
-    for address in request.access_route:
-        if address != '127.0.0.1':
-            print(address)
-            return address
-    return request.remote_addr
 
-limiter = Limiter(app, key_func=rate_limit_key)
+@app.after_request
+def install_security_headers(resp):
+    csp = ("default-src 'none';"
+           "img-src 'self';"
+           "style-src 'self' 'unsafe-inline';"
+           "frame-ancestors 'none';"
+           )
+    if 'SENTRY_DSN' in app.config:
+        csp += "script-src 'self' https://cdn.ravenjs.com/;"
+        csp += "connect-src 'self' https://sentry.io/;"
+    else:
+        csp += "script-src 'self' 'unsafe-eval';"
+        csp += "connect-src 'self';"
+
+    if 'CSP_REPORT_URI' in app.config:
+        csp += "report-uri " + app.config.get('CSP_REPORT_URI')
+
+    if app.config.get('HTTPS'):
+        resp.headers.set('strict-transport-security',
+                         'max-age={}'.format(60*60*24*365))
+        csp += "; upgrade-insecure-requests"
+
+    resp.headers.set('Content-Security-Policy', csp)
+    resp.headers.set('referrer-policy', 'no-referrer')
+    resp.headers.set('x-content-type-options', 'nosniff')
+    resp.headers.set('x-frame-options', 'DENY')
+    resp.headers.set('x-xss-protection', '1')
+
+    return resp
+
+
+mimetypes.add_type('image/webp', '.webp')
+
+libforget.brotli.brotli(app)
+
+imgproxy = (
+    libforget.img_proxy.ImgProxyCache(redis_uri=app.config.get('REDIS_URI')))
+
+app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)

assets/icon.svg

@@ -16,7 +16,7 @@
    id="svg8"
    inkscape:version="0.92.1 r"
    sodipodi:docname="icon.svg"
-   inkscape:export-filename="/home/codl/dev/forget/static/icon.png"
+   inkscape:export-filename="/home/codl/dev/forget/assets/icon.png"
    inkscape:export-xdpi="96"
    inkscape:export-ydpi="96">
   <defs
@@ -26,11 +26,11 @@
      pagecolor="#ffffff"
      bordercolor="#666666"
      borderopacity="1.0"
-     inkscape:pageopacity="0.0"
+     inkscape:pageopacity="0"
      inkscape:pageshadow="2"
-     inkscape:zoom="2.8567114"
-     inkscape:cx="34.405784"
-     inkscape:cy="62.165837"
+     inkscape:zoom="1.4283557"
+     inkscape:cx="184.99523"
+     inkscape:cy="-92.562755"
      inkscape:document-units="mm"
      inkscape:current-layer="layer1"
      showgrid="false"
@@ -43,7 +43,8 @@
      inkscape:window-height="1020"
      inkscape:window-x="0"
      inkscape:window-y="31"
-     inkscape:window-maximized="1" />
+     inkscape:window-maximized="1"
+     inkscape:pagecheckerboard="true" />
   <metadata
      id="metadata5">
     <rdf:RDF>
@@ -61,13 +62,13 @@
      inkscape:groupmode="layer"
      id="layer1"
      transform="translate(84.097282,-171.40979)">
-    <rect
-       style="opacity:1;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.26458332;stroke-linecap:butt;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
-       id="rect4489"
-       width="37.139881"
-       height="37.139881"
-       x="-84.097282"
-       y="171.40979" />
+    <path
+       transform="matrix(0.26458333,0,0,0.26458333,-84.097282,171.40979)"
+       style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.99999994px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="M 32.939453,3.1972656 C 24.168569,3.0451495 15.079238,7.6549961 11.033203,16.685547 c -8.4731529,18.911677 0.08466,41.03752 7.3125,60.478515 3.631125,9.766774 7.312103,19.040448 9.494141,26.453128 1.07333,3.64625 1.715203,6.97656 1.914062,9.47461 0.780434,3.12173 0.687876,5.12719 -0.767152,6.9847 l 10.287032,18.50112 c 1.781562,-0.48657 1.102772,-0.30657 2.903948,-1.09715 1.500393,-0.6583 2.781027,-0.97068 5.667969,-3.40235 6.953155,-5.85528 8.501804,-14.76179 8.296875,-22.9746 -0.106144,-4.25391 -0.825456,-8.66099 -1.697266,-13.099614 3.56439,-0.364764 7.038375,-1.236687 10.357422,-2.53125 0.595555,6.809914 3.787703,12.887944 8.369141,17.191404 5.951744,5.59062 13.747643,8.65249 21.671875,9.53711 7.93258,0.88555 16.43751,-0.39549 23.59375,-5.08789 7.29046,-4.78041 12.4389,-13.30017 12.90039,-23.757811 0.43456,-9.847238 -2.68108,-18.233863 -8.64844,-23.457031 -5.8482,-5.11886 -13.11537,-6.604703 -19.62304,-6.617188 -4.002224,-0.0077 -8.038878,0.399729 -11.974613,1.019531 1.060396,-6.714583 0.246984,-13.93875 -3.353516,-20.873047 C 82.118522,32.604501 73.360599,23.04592 63.75,15.826172 54.172795,8.6315103 43.653188,3.3830772 32.939453,3.1972656 Z"
+       id="path4571"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="sssscccccsccsssssccsss" />
     <path
        style="fill:#000000;fill-rule:nonzero;stroke:#0b0b0b;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;fill-opacity:1"
        d="m -74.470561,193.73092 c 5.501238,4.06044 14.931932,-2.8816 11.395421,-9.69266 -3.53651,-6.81105 -13.448034,-12.5624 -15.848803,-7.204 -3.258985,7.2739 6.922683,21.59014 4.97731,26.45834 l 0.916874,1.70276 c 4.97731,-4.19142 -5.89899,-23.53379 -3.274548,-26.45834 2.591897,-2.88827 10.511924,4.25244 11.264439,7.59695 1.178837,5.23928 -5.108291,4.97731 -9.430693,7.59695 z"

assets/instance_buttons.js

@@ -0,0 +1,74 @@
+import {SLOTS, normalize_known, known_load, known_save} from './known_instances.js';
+
+(function instance_buttons(){
+
+    const mastodon_container = document.querySelector('#mastodon_instance_buttons');
+    const mastodon_button_template = Function('first', 'instance',
+        'return `' + document.querySelector('#mastodon_instance_button_template').innerHTML + '`;');
+    const mastodon_another_button_template = Function(
+        'return `' +
+            document.querySelector('#mastodon_another_instance_button_template').innerHTML + '`;');
+    const mastodon_top_instances =
+        Function('return JSON.parse(`' + document.querySelector('#mastodon_top_instances').innerHTML + '`);')();
+
+    const misskey_container = document.querySelector('#misskey_instance_buttons');
+    const misskey_button_template = Function('first', 'instance',
+        'return `' + document.querySelector('#misskey_instance_button_template').innerHTML + '`;');
+    const misskey_another_button_template = Function(
+        'return `' +
+            document.querySelector('#misskey_another_instance_button_template').innerHTML + '`;');
+    const misskey_top_instances =
+        Function('return JSON.parse(`' + document.querySelector('#misskey_top_instances').innerHTML + '`);')();
+
+    async function replace_buttons(){
+        let known = known_load();
+
+        known = normalize_known(known);
+        known_save(known);
+
+        let filtered_top_instances = []
+        for(let instance of top_instances){
+            let found = false;
+            for(let k of known_instances){
+                if(k['instance'] == instance['instance']){
+                    found = true;
+                    break;
+                }
+            }
+            if(!found){
+                filtered_top_instances.push(instance)
+            }
+        }
+
+        let instances = known_instances.concat(filtered_top_instances).slice(0, SLOTS);
+
+        let html = '';
+
+        let first = true;
+        for(let instance of instances){
+            html += template(first, instance['instance'])
+            first = false;
+        }
+
+        html += template_another_instance();
+
+        container.innerHTML = html;
+    }
+
+    async function init_buttons(){
+        let known = await get_known();
+
+        known.mastodon = normalize_known(known.mastodon);
+        known.misskey = normalize_known(known.misskey);
+        known_save(known);
+
+        replace_buttons(mastodon_top_instances, known.mastodon,
+            mastodon_container, mastodon_button_template,
+            mastodon_another_button_template);
+        replace_buttons(misskey_top_instances, known.misskey,
+            misskey_container, misskey_button_template,
+            misskey_another_button_template);
+    }
+
+    init_buttons();
+})();

assets/known_instances.js

@@ -0,0 +1,72 @@
+const STORAGE_KEY = 'forget_known_instances@2021-12-09';
+export const SLOTS = 5;
+
+function load_and_migrate_old(){
+    const OLD_KEY = "forget_known_instances";
+    let olddata = localStorage.getItem(OLD_KEY);
+    if(olddata != null){
+        olddata = JSON.parse(olddata)
+        let newdata = {
+            mastodon: olddata,
+            misskey: [{
+                "instance": "misskey.io",
+                "hits": 0
+            }]
+        };
+        known_save(newdata);
+        localStorage.removeItem(OLD_KEY);
+        return newdata;
+    }
+}
+
+export function known_save(known){
+    localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
+}
+
+export function known_load(){
+    const default_ = {
+        mastodon:[{ "instance": "mastodon.social", "hits": 0 }],
+        misskey:[{ "instance": "misskey.io", "hits": 0 }],
+    };
+    // this makes mastodon.social and misskey.io show up on respective first
+    // buttons by default even if they are not the most popular instance
+    // according to the server
+
+    let known = localStorage.getItem(STORAGE_KEY);
+    if(known){
+        known = JSON.parse(known);
+    } else {
+        known = load_and_migrate_old();
+    }
+    return known || default_;
+}
+
+export function normalize_known(known){
+    /*
+
+    move instances with the most hits to the top SLOTS slots,
+    making sure not to reorder anything that is already there
+
+    */
+    let head = known.slice(0, SLOTS);
+    let tail = known.slice(SLOTS);
+
+    if(tail.length == 0){
+        return known;
+    }
+
+    for(let i = 0; i < SLOTS; i++){
+        let head_min = head.reduce((acc, cur) => acc.hits < cur.hits ? acc : cur);
+        let tail_max = tail.reduce((acc, cur) => acc.hits > cur.hits ? acc : cur);
+        if(head_min.hits < tail_max.hits){
+            // swappy
+            let i = head.indexOf(head_min);
+            let j = tail.indexOf(tail_max);
+            let buf = head[i];
+            head[i] = tail[j];
+            tail[j] = buf;
+        }
+    }
+
+    return head.concat(tail)
+}

assets/mastodon.svg

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   viewBox="0 0 101.25 101.24999"
+   height="108"
+   width="108"
+   version="1.1"
+   id="svg4"
+   sodipodi:docname="mastodon.svg"
+   inkscape:version="0.92.1 r"
+   inkscape:export-filename="/home/codl/dev/forget/assets/mastodon.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96">
+  <metadata
+     id="metadata10">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs8" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1219"
+     inkscape:window-height="900"
+     id="namedview6"
+     showgrid="false"
+     inkscape:zoom="2.194861"
+     inkscape:cx="50.601219"
+     inkscape:cy="53.761946"
+     inkscape:window-x="0"
+     inkscape:window-y="31"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4"
+     units="px"
+     inkscape:pagecheckerboard="true" />
+  <path
+     d="m 76.184258,49.229421 c -3.9125,0 -7.085,-3.1825 -7.085,-7.095 0,-3.91125 3.1725,-7.09375 7.085,-7.09375 3.92125,0 7.09375,3.1825 7.09375,7.09375 0,3.9125 -3.1725,7.095 -7.09375,7.095 m -25.55875,0 c -3.9225,0 -7.095,-3.1825 -7.095,-7.095 0,-3.91125 3.1725,-7.09375 7.095,-7.09375 3.91125,0 7.09375,3.1825 7.09375,7.09375 0,3.9125 -3.1825,7.095 -7.09375,7.095 m -25.57,0 c -3.91125,0 -7.08375,-3.1825 -7.08375,-7.095 0,-3.91125 3.1725,-7.09375 7.08375,-7.09375 3.92125,0 7.09375,3.1825 7.09375,7.09375 0,3.9125 -3.1725,7.095 -7.09375,7.095 m 72.5775,-15.905 c 0,-21.86625 -14.32375,-28.2737494 -14.32375,-28.2737494 -7.23,-3.31875 -19.63,-4.71250004 -32.5175,-4.82750004 h -0.3125 c -12.88875,0.115 -25.28875,1.50875004 -32.5075,4.82750004 0,0 -14.3237495,6.4074994 -14.3237495,28.2737494 0,5.00375 -0.105,10.995 0.05125,17.34 0.52,21.38875 3.92125,42.46375 23.6974995,47.69625 9.1125,2.412499 16.945,2.912499 23.24875,2.568749 11.4225,-0.63375 17.84,-4.076249 17.84,-4.076249 l -0.37375,-8.3025 c 0,0 -8.16625,2.58 -17.34125,2.2675 -9.09125,-0.3125 -18.6825,-0.9775 -20.16,-12.13875 -0.135,-0.97875 -0.1975,-2.02875 -0.1975,-3.13125 0,0 8.915,2.185 20.2325,2.69375 6.9075,0.3225 13.39875,-0.39375 19.98375,-1.185 12.6275,-1.50875 23.62375,-9.29 25.0075,-16.405 2.17375,-11.1925 1.99625,-27.3275 1.99625,-27.3275"
+     id="path2"
+     style="fill:#ffffff"
+     inkscape:connector-curvature="0" />
+</svg>

assets/settings.js

@@ -0,0 +1,224 @@
+import Banner from '../components/Banner.html';
+import {known_load, known_save} from './known_instances.js'
+
+(function settings_init(){
+    if(!('fetch' in window)){
+        return;
+    }
+
+    let status_timeout = null;
+
+    let settings_section = document.querySelector('#settings-section');
+    let form = document.querySelector('form[name=settings]');
+    let inputs = Array.from(form.elements)
+    let backoff_level = 0;
+
+    let banner_el = document.querySelector('.main-banner');
+    banner_el.innerHTML = '';
+    let banner = new Banner({
+        target: banner_el,
+    });
+
+    function hide_status(){
+        status_display.classList.remove('error', 'success', 'saving');
+        status_display.classList.add('hidden');
+        status_display.innerHTML='';
+    }
+    function show_error(){
+        hide_status();
+        status_display.textContent='Could not save. Retrying...';
+        status_display.classList.add('error');
+        status_display.classList.remove('hidden');
+    }
+    function show_success(){
+        hide_status();
+        status_display.textContent='Saved!';
+        status_display.classList.add('success');
+        status_display.classList.remove('hidden');
+    }
+    function show_still_saving(){
+        status_display.textContent='Still saving...';
+    }
+    function show_saving(){
+        hide_status();
+        status_display.textContent='Saving...';
+        status_display.classList.add('saving');
+        status_display.classList.remove('hidden');
+        status_timeout = setTimeout(show_still_saving, 5000);
+    }
+
+    function save(){
+        hide_status();
+        clearTimeout(status_timeout);
+        status_timeout = setTimeout(show_saving, 70);
+
+        let promise = send_settings(get_all_inputs())
+            .then(() => {
+                show_success();
+                clearTimeout(status_timeout);
+                status_timeout = setTimeout(hide_status, 3000);
+                backoff_level = 0;
+            });
+        promise.catch(() => {
+            show_error();
+            clearTimeout(status_timeout);
+            status_timeout = setTimeout(save, Math.pow(2, backoff_level)*1000);
+            backoff_level += 1;
+            backoff_level = Math.min(backoff_level, 5);
+        });
+        promise.then(fetch_viewer).then(update_viewer);
+
+        // remove server-rendered banner
+        let banner = settings_section.querySelector('.banner');
+        if(banner){
+            settings_section.removeChild(banner);
+        }
+    }
+
+    function get_all_inputs(){
+        let o = Object();
+        for(let input of inputs){
+            if(input.type != 'radio' || input.checked){
+                o[input.name] = input.value;
+            }
+        }
+        return o;
+    }
+
+    function send_settings(body){
+        return fetch('/api/settings', {
+            method:'PUT',
+            credentials:'same-origin',
+            headers: {
+                'content-type': 'application/json',
+            },
+            body: JSON.stringify(body)
+        })
+            .then(resp => {
+                if(!resp.ok){ return Promise.reject(resp); }
+                return resp; })
+            .then(resp => resp.json())
+            .then(data => {
+                if(data.status == 'error'){ return Promise.reject(data); }
+                return data;
+            });
+    }
+
+    for(let input of inputs){
+        input.addEventListener('change', save);
+    }
+
+    // remove submit button since we're doing live updates
+    let submit = form.querySelector('input[type=submit]');
+    form.removeChild(submit);
+    inputs.splice(inputs.indexOf(submit), 1);
+
+    let status_display = document.createElement('span');
+    status_display.classList.add('status-display', 'hidden');
+    settings_section.insertBefore(status_display, settings_section.childNodes[0]);
+
+    // silently send_settings in case the user changed settings while the page was loading
+    send_settings(get_all_inputs());
+
+    let viewer_update_interval = 1500;
+
+    function fetch_viewer(){
+        viewer_update_interval *= 2;
+        viewer_update_interval = Math.min(30000, viewer_update_interval);
+        return fetch('/api/viewer', {
+            credentials: 'same-origin',
+        })
+            .then(resp => {
+                if(!resp.ok){
+                    if(resp.status == 403){
+                        // user was logged out in another client
+                        window.location = '/';
+                    }
+                    return Promise.reject(resp);
+                }
+                return resp; })
+            .then(resp => resp.json());
+    }
+
+    let last_viewer = {};
+    function update_viewer(viewer){
+        let dumped = JSON.stringify(viewer);
+        if(last_viewer == dumped){
+            return;
+        }
+        last_viewer = dumped;
+
+        document.querySelector('#post-count').textContent = viewer.post_count;
+        document.querySelector('#eligible-estimate').textContent = viewer.eligible_for_delete_estimate;
+        document.querySelector('#display-name').textContent = viewer.display_name || viewer.screen_name;
+        document.querySelector('#display-name').title = '@' + viewer.screen_name;
+        document.querySelector('#avatar').src = viewer.avatar_url;
+        viewer_update_interval = 1500;
+
+        if(viewer.next_delete){
+            viewer.next_delete = new Date(viewer.next_delete);
+        }
+        if(viewer.last_delete){
+            viewer.last_delete = new Date(viewer.last_delete);
+        }
+        banner.$set(viewer);
+    }
+
+    let viewer_from_dom = JSON.parse(document.querySelector('script[data-viewer]').textContent)
+
+    update_viewer(viewer_from_dom)
+
+    function set_viewer_timeout(){
+        setTimeout(() => fetch_viewer().then(update_viewer).then(set_viewer_timeout, set_viewer_timeout),
+            viewer_update_interval);
+    }
+    set_viewer_timeout();
+
+    banner.$on('toggle', event => {
+        let enabled = event.detail;
+        send_settings({policy_enabled: enabled}).then(fetch_viewer).then(update_viewer);
+        // TODO show error or spinner if it takes over a second
+    })
+
+    let reason_banner = document.querySelector('.banner[data-reason]');
+    if(reason_banner){
+        let dismiss = reason_banner.querySelector('input[type=submit]');
+        dismiss.addEventListener('click', e => {
+            e.preventDefault();
+
+            // we don't care if this succeeds or fails. worst
+            // case scenario the banner appears again on a future page load
+            fetch('/api/reason', {method: 'DELETE', credentials:'same-origin'});
+
+            reason_banner.parentElement.removeChild(reason_banner);
+        })
+    }
+
+    function bump_instance(service, instance_name){
+        let known_instances = known_load();
+        let found = false;
+        for(let instance of known_instances[service]){
+            if(instance['instance'] == instance_name){
+                instance.hits ++;
+                found = true;
+                break;
+            }
+        }
+        if(!found){
+            let instance = {"instance": instance_name, "hits": 1};
+            known_instances[service].push(instance);
+        }
+
+        known_save(known_instances);
+
+    }
+
+    if(location.hash == '#bump_instance' && (
+        viewer_from_dom['service'] == 'mastodon' || viewer_from_dom['service'] == 'misskey'
+    )){
+        bump_instance(viewer_from_dom['service'], viewer_from_dom['id'].split('@')[1])
+        let url = new URL(location.href)
+        url.hash = '';
+        history.replaceState('', '', url);
+    }
+})();

assets/settings_form.js

@@ -1,111 +0,0 @@
-(function(){
-    if(!('fetch' in window)){
-        return;
-    }
-
-    let status_timeout = null;
-
-    let form = document.forms.settings;
-    let backoff_level = 0
-
-    function hide_status(){
-        status_display.classList.remove('error', 'success', 'saving');
-        status_display.classList.add('hidden');
-        status_display.innerHTML='';
-    }
-    function show_error(e){
-        hide_status();
-        status_display.textContent='Could not save. Retrying...';
-        status_display.classList.add('error');
-        status_display.classList.remove('hidden');
-    }
-    function show_success(){
-        hide_status();
-        status_display.textContent='Saved!';
-        status_display.classList.add('success');
-        status_display.classList.remove('hidden');
-    }
-    function show_saving(){
-        hide_status();
-        status_display.textContent='Saving...';
-        status_display.classList.add('saving');
-        status_display.classList.remove('hidden');
-        status_timeout = setTimeout(show_still_saving, 5000);
-    }
-    function show_still_saving(){
-        status_display.textContent='Still saving...';
-    }
-
-    function on_change(e){
-        hide_status();
-        clearTimeout(status_timeout);
-        status_timeout = setTimeout(show_saving, 70);
-
-        send_settings(get_all_inputs())
-            .then(data => {
-                show_success();
-                clearTimeout(status_timeout);
-                status_timeout = setTimeout(hide_status, 3000);
-                backoff_level = 0;
-            })
-            .catch(e => {
-                console.error('Fetch rejected:', e);
-                show_error();
-                clearTimeout(status_timeout);
-                status_timeout = setTimeout(save, Math.pow(2, backoff_level)*1000);
-                backoff_level += 1;
-                backoff_level = Math.min(backoff_level, 5);
-            });
-
-        // remove server-rendered banner
-        let settings_section = document.querySelector('#settings-section');
-        let banner = settings_section.querySelector('.banner');
-        if(banner){
-            settings_section.removeChild(banner);
-        }
-    }
-
-    function get_all_inputs(){
-        let o = Object()
-        for(input of form.elements){
-            if(input.type != 'radio' || input.checked){
-                o[input.name] = input.value;
-            }
-        }
-        return o
-    }
-
-    function send_settings(body){
-        return fetch('/api/settings', {
-            method:'PUT',
-            credentials:'same-origin',
-            headers: {
-                'content-type': 'application/json',
-            },
-            body: JSON.stringify(body)
-        })
-            .then(resp => { if(!resp.ok){ return Promise.reject(resp) }; return resp; })
-            .then(resp => resp.json())
-            .then(data => {
-                if(data.status == 'error'){ return Promise.reject(data) }
-                return data
-            });
-    }
-
-    for(input of form.elements){
-        input.addEventListener('change', on_change);
-        input.addEventListener('change', e=>console.log(e.target));
-    }
-
-    // remove submit button since we're doing live updates
-    let submit = form.querySelector('input[type=submit]');
-    form.removeChild(submit);
-
-    let status_display = document.createElement('span');
-    status_display.classList.add('status-display', 'hidden');
-    let settings_title = document.querySelector('#settings-title');
-    settings_title.appendChild(status_display);
-
-    // silently send_settings in case the user changed settings while the page was loading
-    send_settings(get_all_inputs());
-})();

assets/styles.css

@@ -1,15 +1,15 @@
 body {
     margin: 0;
     font-family: sans-serif;
-    line-height: 1.5em;
 }
 
 *, *::before, *::after {
     box-sizing: border-box;
+    line-height: 1.5em;
 }
 
 body > section, body > header, body > footer {
-    max-width: 40rem;
+    max-width: 45rem;
     margin-left: auto;
     margin-right: auto;
 }
@@ -32,6 +32,10 @@ section > * {
     padding-right: 5rem;
 }
 
+section > .container {
+    padding: 0;
+}
+
 section > ul {
     padding-left: 7rem;
 }
@@ -40,19 +44,21 @@ h2 {
     font-size: 1.4em;
     font-weight: normal;
     padding-left: 2rem;
+    padding-right: 2rem;
 }
 
 h3 {
     font-size: inherit;
     font-weight: bold;
     padding-left: 5rem;
+    padding-right: 5rem;
 }
 
 input[type=number]{
     max-width: 8ch;
 }
 
-.viewer img.avatar {
+img.avatar {
     height: 1.5em;
     width: 1.5em;
     background-color: #ccc;
@@ -60,21 +66,37 @@ input[type=number]{
     transform:translateY(25%);
 }
 
+img.avatar.mastodon {
+    border-radius: 20%;
+}
+
 .banner {
     border-left-width: .7rem;
     border-left-style: solid;
     padding-top: .6rem;
     padding-bottom: .6rem;
     margin-bottom: 1rem;
+    padding-left: 1rem;
+    padding-right: 1rem;
 }
 
-body > section > .banner {
+.banner p {
+    margin: .3rem 0;
+}
+
+.banner p ~ p {
+    margin-top: 1.3rem;
+}
+
+body > section > .banner,
+body > section > .container > .banner {
     padding-left: 4.3rem;
+    padding-right: 4.3rem;
 }
 
 .banner.enabled {
     border-left-color: transparent;
-    background: #bbfbff;
+    background: #cde;
 }
 
 .banner.disabled {
@@ -113,13 +135,7 @@ footer {
     font-size: 0.9rem;
     margin-top: 5rem;
     margin-bottom: 3rem;
-    display: flex;
-    flex-direction: row;
-    justify-content: center;
-}
-
-footer p {
-    margin: 0 0.7rem;
+    text-align: center;
 }
 
 footer a {
@@ -132,6 +148,9 @@ footer a {
     margin-left: 1ch;
     padding: 0 0.4em;
     vertical-align: top;
+    position: sticky;
+    top: 0;
+    float: right;
 }
 
 .status-display.hidden {
@@ -143,7 +162,6 @@ footer a {
 }
 
 .status-display.success {
-    display: inline-block;
     background: #dec;
     animation: fade-background 2s forwards;
 }
@@ -178,5 +196,146 @@ footer a {
     }
 }
 
+form aside {
+    font-style: italic;
+    font-size: 0.8em;
+}
+
+button {
+    font-size: inherit;
+}
+
+.btn {
+    text-decoration: none;
+    border-radius: .2rem;
+    overflow: hidden;
+
+    display: inline-block;
+    padding: 0.4em 0.8em;
+    border: none;
+
+    cursor: pointer;
+}
+
+.btn-small {
+    font-size: 0.8em;
+    padding: 0.2em 0.5em;
+}
+
+.btn.primary {
+    color: white;
+    background-color: #37d;
+}
+
+.btn.primary.twitter-colored {
+    background-color: #1da1f2;
+}
+
+.btn.primary.mastodon-colored {
+    background-color: #282c37;
+}
+
+.btn.primary.misskey-colored {
+    background-color: #66b300;
+}
+
+.btn.secondary {
+    background-color: rgba(255,255,255,0.5);
+    box-shadow: inset 0 0 0 1px rgba(0,0,0,0.3);
+    color: inherit;
+}
+
+.btn:hover {
+    opacity: 0.9;
+}
 
 
+.btn img {
+    height: 1.1em;
+    transform: translatey(.2em);
+    margin: 0 .3em;
+}
+
+.btn img:first-child,
+.btn picture:first-child img {
+    margin-left: 0;
+}
+
+.btn-group.inline {
+    display: inline-block;
+    transform: translatey(20%);
+}
+
+.btn-group.right {
+    float: right;
+    margin-left: 1em;
+}
+
+form.btn-group {
+    margin-top: 0; margin-bottom: 0;
+}
+
+.clearfix {
+    clear: both;
+}
+
+section > pre.error-log {
+    overflow: auto;
+    padding-top: 2em;
+    padding-bottom: 2em;
+    opacity: 0.7;
+}
+
+.radiostrip {
+    display: inline-flex;
+    flex-direction: row;
+    flex-wrap: wrap;
+}
+
+.radiostrip .choice {
+    position: relative;
+}
+
+.radiostrip input[type=radio] {
+    position: absolute;
+    top: 0; left: 0;
+    width: 100%; height: 100%;
+    margin: 0;
+    opacity: 0;
+    cursor: pointer;
+}
+
+
+.radiostrip label {
+    padding: .3em .5em;
+    user-select: none;
+    background-color: transparent;
+    border: 1px solid rgba(0,0,0,0.3);
+}
+
+
+.radiostrip span.choice:first-child label {
+    border-radius: .2rem 0 0 .2rem;
+}
+
+.radiostrip span.choice:last-child label {
+    border-radius: 0 .2rem .2rem 0;
+}
+
+.radiostrip span.choice:not(:first-child) label {
+    border-left-color: transparent;
+}
+
+.radiostrip span.choice:not(:last-child) label {
+    border-right-color: transparent;
+}
+
+.radiostrip input[type=radio]:focus + label {
+    box-shadow: 0 0 5px #38f;
+}
+
+.radiostrip input[type=radio]:checked + label {
+    background: #37d;
+    color: white;
+    border-color: transparent;
+}

assets/twitter.svg

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Generator: Adobe Illustrator 20.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 400 400"
+   style="enable-background:new 0 0 400 400;"
+   xml:space="preserve"
+   id="svg12"
+   sodipodi:docname="twitter.svg"
+   inkscape:version="0.92.1 r"
+   inkscape:export-filename="/home/codl/dev/forget/assets/twitter.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96"><metadata
+     id="metadata18"><rdf:RDF><cc:Work
+         rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title /></cc:Work></rdf:RDF></metadata><defs
+     id="defs16" /><sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1020"
+     id="namedview14"
+     showgrid="false"
+     inkscape:zoom="0.59"
+     inkscape:cx="-95.762712"
+     inkscape:cy="193.22034"
+     inkscape:window-x="0"
+     inkscape:window-y="31"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg12"
+     inkscape:pagecheckerboard="true" /><style
+     type="text/css"
+     id="style2">
+	.st0{opacity:0.15;fill:#292F33;}
+	.st1{fill:#FFFFFF;}
+</style><g
+     id="_x31_0_x2013_20_x25__Black_Tint" /><g
+     id="Logo__x2014__FIXED"
+     transform="matrix(1.5838783,0,0,1.5838783,-116.80485,-116.77566)"
+     style="stroke-width:0.63136166"><g
+       id="g9"
+       style="stroke-width:0.63136166"><path
+         class="st1"
+         d="m 153.6,301.6 c 94.3,0 145.9,-78.2 145.9,-145.9 0,-2.2 0,-4.4 -0.1,-6.6 10,-7.2 18.7,-16.3 25.6,-26.6 -9.2,4.1 -19.1,6.8 -29.5,8.1 10.6,-6.3 18.7,-16.4 22.6,-28.4 -9.9,5.9 -20.9,10.1 -32.6,12.4 -9.4,-10 -22.7,-16.2 -37.4,-16.2 -28.3,0 -51.3,23 -51.3,51.3 0,4 0.5,7.9 1.3,11.7 -42.6,-2.1 -80.4,-22.6 -105.7,-53.6 -4.4,7.6 -6.9,16.4 -6.9,25.8 0,17.8 9.1,33.5 22.8,42.7 -8.4,-0.3 -16.3,-2.6 -23.2,-6.4 0,0.2 0,0.4 0,0.7 0,24.8 17.7,45.6 41.1,50.3 -4.3,1.2 -8.8,1.8 -13.5,1.8 -3.3,0 -6.5,-0.3 -9.6,-0.9 6.5,20.4 25.5,35.2 47.9,35.6 -17.6,13.8 -39.7,22 -63.7,22 -4.1,0 -8.2,-0.2 -12.2,-0.7 22.6,14.4 49.6,22.9 78.5,22.9"
+         id="path7"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff;stroke-width:0.63136166" /></g></g></svg>

assets/version.js

@@ -1,14 +0,0 @@
-(function(){
-
-if(!('fetch' in window)){
-    return
-}
-fetch('/api/about').then(r => r.json()).then(j =>  {
-    let ident = document.querySelector('#ident');
-    ident.textContent = j.service
-    if(j.version){
-        ident.textContent += ' ' + j.version
-    }
-})
-
-})();

components/Banner.html

@@ -0,0 +1,133 @@
+<div class='banner {policy_enabled?'enabled':'disabled'}'>
+    <div class='btn-group right'>
+        <button class='btn {policy_enabled?"secondary":"primary"}' on:click={toggle}>{policy_enabled?'Disable':'Enable'}</button>
+    </div>
+    <div>
+        Forget is currently
+        {#if policy_enabled }
+        <b>enabled</b>
+        {:else}
+        disabled
+        {/if}
+        on your account.
+    </div>
+    <div class='timers'>
+        <span class='last-delete' class:hidden={!last_delete} title={last_delete}>
+        {#if last_delete }
+            Last delete {rel_past(now - last_delete)}.
+        {/if }
+        </span>
+        <span class='next-delete'
+              class:hidden={!policy_enabled || !next_delete || !eligible_for_delete_estimate} title={next_delete}>
+            Next delete {rel_future(next_delete - now)}.
+        </span>
+    </div>
+</div>
+
+<style>
+.timers {
+    font-size: 0.8em;
+}
+
+.timers > * {
+    transition-property: opacity, transform;
+    transition-duration: 0.4s;
+    display: inline-block;
+}
+
+.timers > .hidden {
+    opacity: 0;
+    transform: translateY(-0.3em);
+    pointer-events: none;
+}
+
+.banner {
+    transition: background-color 0.6s;
+}
+</style>
+
+<script>
+function absmod(n, x){
+    // it's like modulo but never negative
+    n = n % x;
+    if(n < 0){
+        n += x
+    }
+    return n
+}
+function s(n){
+    // utility for plurals
+    if(n > 1){
+        return 's';
+    }
+    return '';
+}
+function rel(millis){
+    // returns human-readable duration from duration in millis
+    let secs = Math.round(millis/1000)
+    if(secs <= 120){
+        return `${secs} seconds`;
+    }
+    let mins = Math.round(secs/60);
+    if(mins <= 60){
+        return `${mins} minute${s(mins)}`;
+    }
+    let hours = Math.floor(mins/60);
+    mins = mins % 60;
+    if(hours < 6){
+        return `${hours}h ${mins}m`;
+    }
+    if(hours <= 48){
+        return `${hours} hour${s(hours)}`;
+    }
+    let days = Math.round(hours/24);
+    return `${days} days`;
+}
+
+function rel_future(millis){
+    // returns relative time from timestamp, assuming time is in the future
+    if(millis < 2000){
+        let secs = Math.floor(millis/1000)
+        let ndots = absmod(-secs, 3);
+        let out = 'anytime now';
+        for(; ndots > 0; ndots--){
+            out += '.';
+        }
+        return out;
+    }
+    return `in ${rel(millis)}`;
+}
+
+function rel_past(millis){
+    // returns relative time from timestamp, assuming time is in the past
+    if(millis < 2000){
+        return 'just now';
+    }
+    return `${rel(millis)} ago`;
+}
+
+import { onDestroy, createEventDispatcher } from 'svelte';
+
+const dispatch = createEventDispatcher();
+
+function toggle(){
+    console.log(policy_enabled);
+    policy_enabled = !policy_enabled;
+    if(policy_enabled){
+        next_delete = null;
+    }
+    dispatch('toggle', policy_enabled);
+}
+
+
+export let next_delete, last_delete, eligible_for_delete_estimate;
+export let policy_enabled = false;
+
+let now = +(new Date());
+
+let interval = setInterval(() =>
+    now = +(new Date())
+, 1000 );
+
+onDestroy(()=> clearInterval(interval));
+</script>

config.docker.py

@@ -0,0 +1,76 @@
+"""
+this is an example config file for Forget
+
+copy this file to config.py before editing
+
+lines starting with # demonstrate default or example values
+the # should be removed before editing
+"""
+
+"""
+DATABASE URI
+
+determines where to connect to the database
+see <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls> for syntax
+only postgresql with psycopg2 driver is officially supported
+"""
+SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://postgres:postgres@db/forget'
+
+"""
+REDIS URI
+
+see <https://redis-py.readthedocs.io/en/latest/#redis.ConnectionPool.from_url>
+for syntax reference
+"""
+REDIS_URI='redis://redis'
+
+"""
+SERVER ADDRESS
+
+This is the address at which forget will be reached.
+External services will redirect to this address when logging in.
+"""
+# SERVER_NAME="0.0.0.0:5000"
+# HTTPS=True
+
+"""
+TWITTER CREDENTIALS
+
+Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
+When prompted for it, your callback URL is {SERVER_NAME}/login/twitter/callback
+"""
+# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
+# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
+
+"""
+SENTRY
+
+If you want to send exceptions to sentry, enter your sentry DSN here
+"""
+# SENTRY_DSN=''
+
+"""
+HIDDEN INSTANCES
+
+The front page shows one-click login buttons for the mastodon and
+misskey instances that see the most heavy use. Instances configured in this
+list will be prevented from appearing in these buttons.
+
+They will still appear if a user has previously logged into them and their
+browser remembers it. A user will still be able to log into them by manually
+typing the address into the log in form.
+
+This is a space-delimited list. Example syntax:
+HIDDEN_INSTANCES='social.example.com pleroma.example.net mk.example.org'
+"""
+# HIDDEN_INSTANCES=''
+
+"""
+ADVANCED FLASK CONFIG
+
+you can also use any config variable that flask expects here
+A list of these config variables is available here:
+<http://flask.pocoo.org/docs/1.0/config/#builtin-configuration-values>
+"""
+# SESSION_COOKIE_SECURE=True
+# DEBUG=True

config.example.py

@@ -2,59 +2,75 @@
 this is an example config file for Forget
 
 copy this file to config.py before editing
+
+lines starting with # demonstrate default or example values
+the # should be removed before editing
 """
 
 """
 DATABASE URI
 
 determines where to connect to the database
-see http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls for syntax
+see <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls> for syntax
 only postgresql with psycopg2 driver is officially supported
 """
 # SQLALCHEMY_DATABASE_URI='postgresql+psycopg2:///forget'
 
 """
-TWITTER CREDENTIALS
+REDIS URI
 
-get these at apps.twitter.com
-blah
+see <https://redis-py.readthedocs.io/en/latest/#redis.ConnectionPool.from_url>
+for syntax reference
 """
-# TWITTER_CONSUMER_KEY='vdsvdsvds'
-# TWITTER_CONSUMER_SECRET='hjklhjklhjkl'
+# REDIS_URI='redis://'
 
 """
-this will be necessary so we can tell twitter where to redirect
+SERVER ADDRESS
+
+This is the address at which forget will be reached.
+External services will redirect to this address when logging in.
 """
 # SERVER_NAME="localhost:5000"
-
-# CELERY_BROKER='redis://'
-
 # HTTPS=True
 
-# SENTRY_DSN='https://foo:bar@sentry.io/69420'
+"""
+TWITTER CREDENTIALS
 
-'''
-you can set this to memory:// if you only have one web process
-or if you don't care about people exhausting your twitter api
-key and your celery workers by making hundreds of login
-requests and uploading hundreds of bogus tweet archives
-
-docs here <https://flask-limiter.readthedocs.io/en/stable/#configuration>
-'''
-# RATELIMIT_STORAGE_URL='redis://'
-
-# REDIS=dict(
-#        db=0
-#
-#        host='localhost'
-#        port=6379
-#        # or...
-#        unix_socket_path='/var/run/redis/redis.sock'
-#        # see `pydoc redis.StrictRedis.__init__` for full list of arguments
-#     )
+Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
+When prompted for it, your callback URL is {SERVER_NAME}/login/twitter/callback
+"""
+# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
+# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
 
 """
-you can also use any config variable that flask expects here, such as
+SENTRY
+
+If you want to send exceptions to sentry, enter your sentry DSN here
+"""
+# SENTRY_DSN=''
+
+"""
+HIDDEN INSTANCES
+
+The front page shows one-click login buttons for the mastodon and
+misskey instances that see the most heavy use. Instances configured in this
+list will be prevented from appearing in these buttons.
+
+They will still appear if a user has previously logged into them and their
+browser remembers it. A user will still be able to log into them by manually
+typing the address into the log in form.
+
+This is a space-delimited list. Example syntax:
+HIDDEN_INSTANCES='social.example.com pleroma.example.net mk.example.org'
+"""
+# HIDDEN_INSTANCES=''
+
+"""
+ADVANCED FLASK CONFIG
+
+you can also use any config variable that flask expects here
+A list of these config variables is available here:
+<http://flask.pocoo.org/docs/1.0/config/#builtin-configuration-values>
 """
 # SESSION_COOKIE_SECURE=True
 # DEBUG=True

docker-compose.override.example.yml

@@ -0,0 +1,33 @@
+## uncomment below and change network name to use a reverse proxy on an
+## existing network
+#networks:
+#  mycoolreverseproxynetwork:
+#    external: true
+
+services:
+  www:
+## uncomment below to stop listening on 127.0.0.1
+#    ports: []
+
+    networks:
+      - forget
+## uncomment below and change network name to use a reverse proxy on an
+## existing network
+#      - mycoolreverseproxynetwork
+
+
+## if you wish to change where postgres, redis, and celery persistent data is
+## stored, uncomment below and change the first part of each volume (before the `:`)
+
+#  redis:
+#    volumes:
+#      - /my/cool/redis/path:/data
+#  db:
+#    volumes:
+#      - /my/cool/postgres/path:/var/lib/postgresql/data
+#  worker:
+#    volumes:
+#      - /my/cool/celery/path:/var/run/celery
+#  beat:
+#    volumes:
+#      - /my/cool/celery/path:/var/run/celery

docker-compose.yml

@@ -0,0 +1,103 @@
+services:
+  www:
+    build:
+      context: ./
+    image: ghcr.io/codl/forget
+    pull_policy: missing
+    restart: always
+    volumes:
+      - type: bind
+        source: ./config.py
+        target: /usr/src/app/config.py
+        read_only: true
+    depends_on:
+      - redis
+      - db
+      - worker
+      - beat
+    command: bash -c "
+        flask db upgrade &&
+        gunicorn -w 9 -t 3600 -b 0.0.0.0:42157 forget:app
+      "
+    networks:
+      - forget
+    expose:
+      - 42157
+    ports:
+      - "127.0.0.1:42157:42157"
+
+  worker:
+    build:
+      context: ./
+    image: ghcr.io/codl/forget
+    pull_policy: missing
+    restart: always
+    volumes:
+      - type: bind
+        source: ./config.py
+        target: /usr/src/app/config.py
+        read_only: true
+      - ./data/celery/run:/var/run/celery
+    depends_on:
+      - redis
+      - db
+    networks:
+      - forget
+    command: bash -c "
+        mkdir -p /var/run/celery &&
+        chown -R nobody:nogroup /var/run/celery &&
+        exec celery --app=tasks worker
+                    --loglevel=INFO
+                    --statedb=/var/run/celery/worker.state
+                    --hostname=worker
+                    --uid=nobody --gid=nogroup
+      "
+
+  beat:
+    build:
+      context: ./
+    image: ghcr.io/codl/forget
+    pull_policy: missing
+    restart: always
+    volumes:
+      - type: bind
+        source: ./config.py
+        target: /usr/src/app/config.py
+        read_only: true
+      - ./data/celery/run:/var/run/celery
+    depends_on:
+      - redis
+      - db
+    networks:
+      - forget
+    command: bash -c "
+        mkdir -p /var/run/celery &&
+        chown -R nobody:nogroup /var/run/celery &&
+        exec celery --app=tasks beat
+                    --loglevel=INFO
+                    --schedule=/var/run/celery/schedule
+                    --uid=nobody --gid=nogroup
+      "
+
+  redis:
+    image: redis:4.0-alpine
+    restart: always
+    volumes:
+      - ./data/redis:/data
+    networks:
+      - forget
+
+  db:
+    image: postgres:14-alpine
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=forget
+    volumes:
+      - ./data/postgres:/var/lib/postgresql/data
+    networks:
+      - forget
+
+networks:
+  forget:

dodo.py

@@ -1,40 +1,101 @@
-def task_gen_logo():
-    """generate versions of the logo in various sizes"""
+from doit import create_after
+from glob import glob
+from itertools import chain
 
+
+def reltouch(source_filename, dest_filename):
+    from os import stat, utime
+    stat_res = stat(source_filename)
+    utime(dest_filename, ns=(stat_res.st_atime_ns, stat_res.st_mtime_ns))
+
+
+def resize_image(basename, width, image_format):
     from PIL import Image
+    with Image.open('assets/{}.png'.format(basename)) as im:
+        if 'A' in im.getbands() and image_format != 'jpeg':
+            im = im.convert('RGBA')
+        else:
+            im = im.convert('RGB')
+        height = im.height * width // im.width
+        new = im.resize((width, height), resample=Image.LANCZOS)
+        if image_format == 'jpeg':
+            kwargs = dict(
+                    optimize=True,
+                    progressive=True,
+                    quality=80,
+                    )
+        elif image_format == 'webp':
+            kwargs = dict(
+                    quality=79,
+                    )
+        elif image_format == 'png':
+            kwargs = dict(
+                    optimize=True,
+                    )
+        new.save('static/{}-{}.{}'.format(basename, width, image_format),
+                 **kwargs)
+        reltouch('assets/{}.png'.format(basename),
+                 'static/{}-{}.{}'.format(basename, width, image_format))
 
-    def resize_logo(width):
-        with Image.open('assets/logotype.png') as im:
-            im = im.convert('L')
-            height = im.height * width // im.width
-            new = im.resize((width,height), resample=Image.LANCZOS)
-            new.save('static/logotype-{}.png'.format(width), optimize=True)
 
+def task_logotype():
+    """resize and convert logotype"""
     widths = (200, 400, 600, 800)
+    image_formats = ('jpeg', 'webp')
     for width in widths:
-        yield dict(
-                name=str(width),
-                actions=[(resize_logo, (width,))],
-                targets=[f'static/logotype-{width}.png'],
-                file_dep=['assets/logotype.png'],
-                clean=True,
-            )
+        for image_format in image_formats:
+            yield dict(
+                    name='{}.{}'.format(width, image_format),
+                    actions=[(resize_image,
+                              ('logotype', width, image_format))],
+                    targets=[f'static/logotype-{width}.{image_format}'],
+                    file_dep=['assets/logotype.png'],
+                    clean=True,
+                )
 
 
-def task_copy_asset():
-    import shutil
-    assets = ('icon.png', 'logotype.png', 'version.js', 'settings_form.js')
+def task_service_icon():
+    """resize and convert service icons"""
+    widths = (20, 40, 80)
+    formats = ('webp', 'png')
+    for width in widths:
+        for image_format in formats:
+            for basename in ('twitter', 'mastodon', 'misskey'):
+                yield dict(
+                    name='{}-{}.{}'.format(basename, width, image_format),
+                    actions=[(resize_image, (basename, width, image_format))],
+                    targets=[
+                        'static/{}-{}.{}'.format(basename, width,
+                                                 image_format)],
+                    file_dep=['assets/{}.png'.format(basename)],
+                    clean=True,
+                )
+
+
+def task_copy():
+    "copy assets verbatim"
+
+    assets = ('icon.png', 'logotype.png')
+
+    def do_the_thing(src, dst):
+        from shutil import copy
+        copy(src, dst)
+        reltouch(src, dst)
+
     for asset in assets:
+        src = 'assets/{}'.format(asset)
+        dst = 'static/{}'.format(asset)
         yield dict(
                 name=asset,
-                actions=[(lambda asset: shutil.copy(f'assets/{asset}', f'static/{asset}'), (asset,))],
-                targets=[f'static/{asset}'],
-                file_dep=[f'assets/{asset}'],
+                actions=[(do_the_thing, (src, dst))],
+                targets=[dst],
+                file_dep=[src],
                 clean=True,
             )
 
+
 def task_minify_css():
-    """minify css"""
+    """minify css file with csscompressor"""
 
     from csscompressor import compress
 
@@ -42,6 +103,7 @@ def task_minify_css():
         with open('assets/styles.css') as in_:
             with open('static/styles.css', 'w') as out:
                 out.write(compress(in_.read()))
+        reltouch('assets/styles.css', 'static/styles.css')
 
     return dict(
             actions=[minify],
@@ -50,39 +112,80 @@ def task_minify_css():
             clean=True,
         )
 
-def task_compress_static():
-    import brotli
-    import gzip
 
-    files = ('static/styles.css', 'static/icon.png', 'static/logotype.png') + tuple((f'static/logotype-{width}.png' for width in (200, 400, 600, 800)))
+def task_rollup():
+    """rollup javascript bundle"""
 
-    def compress_brotli(dependencies):
-        for filename in dependencies:
-            with open(filename, 'rb') as in_:
-                with open(filename + '.br', 'wb') as out:
-                    out.write(brotli.compress(in_.read()))
-    def compress_gzip(dependencies):
-        for filename in dependencies:
-            with open(filename, 'rb') as in_:
-                with gzip.open(filename + '.gz', 'wb') as out:
-                    out.write(in_.read())
+    filenames = ['settings.js', 'instance_buttons.js']
+    for filename in filenames:
+        src = 'assets/{}'.format(filename)
+        dst = 'static/{}'.format(filename)
+        name = filename.split('.')[0]
+        yield dict(
+                name=filename,
+                file_dep=list(chain(
+                    # fuck it
+                    glob('assets/*.js'),
+                    glob('components/*.html'))) + ['rollup.config.js'],
+                targets=[dst],
+                clean=True,
+                actions=[
+                    ['node_modules/.bin/rollup', '-c',
+                     '-i', src, '-o', dst, '-n', name, '-f', 'iife'],
+                ],
+            )
+
+
+@create_after('logotype')
+@create_after('service_icon')
+@create_after('copy')
+@create_after('minify_css')
+@create_after('rollup')
+def task_compress():
+    """
+    make gzip and brotli compressed versions of each
+    static file for the server to lazily serve
+    """
+
+    files = chain(
+            glob('static/*.css'),
+            glob('static/*.js'),
+            glob('static/*.jpeg'),
+            glob('static/*.png'),
+            glob('static/*.webp'),
+            )
+
+    def compress_brotli(filename):
+        import brotli
+        with open(filename, 'rb') as in_:
+            with open(filename + '.br', 'wb') as out:
+                out.write(brotli.compress(in_.read()))
+        reltouch(filename, filename+'.br')
+
+    def compress_gzip(filename):
+        import gzip
+        with open(filename, 'rb') as in_:
+            with gzip.open(filename + '.gz', 'wb') as out:
+                out.write(in_.read())
+        reltouch(filename, filename+'.gz')
 
     for filename in files:
         yield dict(
                 file_dep=(filename,),
                 targets=(filename+'.br',),
                 name=filename+'.br',
-                actions=[compress_brotli],
+                actions=[(compress_brotli, (filename,))],
                 clean=True,
             )
         yield dict(
                 file_dep=(filename,),
                 targets=(filename+'.gz',),
                 name=filename+'.gz',
-                actions=[compress_gzip],
+                actions=[(compress_gzip, (filename,))],
                 clean=True,
             )
 
+
 if __name__ == '__main__':
     import doit
     doit.run(globals())

forget.py

@@ -1,2 +1,9 @@
 from app import app
 import routes
+import routes.misc
+import routes.api
+
+assert app
+assert routes
+assert routes.misc
+assert routes.api

lib/__init__.py

@@ -1,7 +0,0 @@
-from . import auth
-from .interval import decompose_interval
-from .interval import SCALES as interval_scales
-from .cachebust import cachebust
-from .session import set_session_cookie, get_viewer_session, get_viewer
-from . import brotli
-from . import settings

lib/auth.py

@@ -1,20 +0,0 @@
-from flask import g, redirect, jsonify, make_response
-from functools import wraps
-
-def require_auth(fun):
-    @wraps(fun)
-    def wrapper(*args, **kwargs):
-        if not g.viewer:
-            return redirect('/')
-        return fun(*args, **kwargs)
-    return wrapper
-
-def require_auth_api(fun):
-    @wraps(fun)
-    def wrapper(*args, **kwargs):
-        if not g.viewer:
-            return make_response((jsonify(status='error', error='not logged in'), 403))
-        return fun(*args, **kwargs)
-    return wrapper
-
-

lib/brotli.py

@@ -1,66 +0,0 @@
-import brotli as brotli_
-from flask import request, make_response
-from threading import Thread
-from hashlib import sha256
-import redis
-import os.path
-import mimetypes
-
-class BrotliCache(object):
-    def __init__(self, redis_kwargs={}, max_wait=0.3, expire=60*60*12):
-        self.redis = redis.StrictRedis(**redis_kwargs)
-        self.max_wait = max_wait
-        self.expire = expire
-
-    def compress(self, cache_key, lock_key, body, mode=brotli_.MODE_GENERIC):
-        encbody = brotli_.compress(body, mode=mode)
-        self.redis.set(cache_key, encbody, ex=self.expire)
-        self.redis.delete(lock_key)
-
-    def wrap_response(self, response):
-        if 'br' not in request.accept_encodings or response.is_streamed:
-            return response
-
-        body = response.get_data()
-        digest = sha256(body).hexdigest()
-        cache_key = 'brotlicache:{}'.format(digest)
-
-        encbody = self.redis.get(cache_key)
-        if not encbody:
-            lock_key = 'brotlicache:lock:{}'.format(digest)
-            if self.redis.set(lock_key, 1, nx=True, ex=10):
-                mode = brotli_.MODE_TEXT if response.content_type.startswith('text/') else brotli_.MODE_GENERIC
-                t = Thread(target=self.compress, args=(cache_key, lock_key, body, mode))
-                t.start()
-                if self.max_wait > 0:
-                    t.join(self.max_wait)
-                    encbody = self.redis.get(cache_key)
-        if encbody:
-            response.headers.set('content-encoding', 'br')
-            response.headers.set('vary', 'accept-encoding')
-            response.set_data(encbody)
-            return response
-
-        return response
-
-def brotli(app, static = True, dynamic = True):
-    original_static = app.view_functions['static']
-    def static_maybe_gzip_brotli(filename=None):
-        path = os.path.join(app.static_folder, filename)
-        for encoding, extension in (('br', '.br'), ('gzip', '.gz')):
-            if encoding not in request.accept_encodings:
-                continue
-            encpath = path + extension
-            if os.path.isfile(encpath):
-                resp = make_response(original_static(filename=filename + extension))
-                resp.headers.set('content-encoding', encoding)
-                resp.headers.set('vary', 'accept-encoding')
-                mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
-                resp.headers.set('content-type', mimetype)
-                return resp
-        return original_static(filename=filename)
-    if static:
-        app.view_functions['static'] = static_maybe_gzip_brotli
-    if dynamic:
-        cache = BrotliCache()
-        app.after_request(cache.wrap_response)

lib/session.py

@@ -1,18 +0,0 @@
-from flask import request
-
-def set_session_cookie(session, response, secure=True):
-    response.set_cookie('forget_sid', session.id,
-        max_age=60*60*48,
-        httponly=True,
-        secure=secure)
-
-def get_viewer_session():
-    from model import Session
-    sid = request.cookies.get('forget_sid', None)
-    if sid:
-        return Session.query.get(sid)
-
-def get_viewer():
-    session = get_viewer_session()
-    if session:
-        return session.account

lib/twitter.py

@@ -1,161 +0,0 @@
-from twitter import Twitter, OAuth, TwitterHTTPError
-from werkzeug.urls import url_decode
-from model import OAuthToken, Account, Post
-from app import db, app
-from math import inf
-from datetime import datetime
-import locale
-
-def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
-    twitter = Twitter(
-            auth=OAuth('', '', consumer_key, consumer_secret),
-            format='', api_version=None)
-    resp = url_decode(twitter.oauth.request_token(oauth_callback=callback))
-    oauth_token = resp['oauth_token']
-    oauth_token_secret = resp['oauth_token_secret']
-
-    token = OAuthToken(token = oauth_token, token_secret = oauth_token_secret)
-    db.session.merge(token)
-    db.session.commit()
-
-    return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,)
-
-def account_from_api_user_object(obj):
-    return Account(
-            twitter_id = obj['id_str'],
-            display_name = obj['name'],
-            screen_name = obj['screen_name'],
-            avatar_url = obj['profile_image_url_https'],
-            reported_post_count = obj['statuses_count'])
-
-def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None):
-    temp_token = OAuthToken.query.get(oauth_token)
-    if not temp_token:
-        raise Exception("OAuth token has expired")
-    twitter = Twitter(
-            auth=OAuth(temp_token.token, temp_token.token_secret, consumer_key, consumer_secret),
-            format='', api_version=None)
-    resp = url_decode(twitter.oauth.access_token(oauth_verifier = oauth_verifier))
-    db.session.delete(temp_token)
-    new_token = OAuthToken(token = resp['oauth_token'], token_secret = resp['oauth_token_secret'])
-    new_token = db.session.merge(new_token)
-    new_twitter = Twitter(
-            auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret))
-    remote_acct = new_twitter.account.verify_credentials()
-    acct = account_from_api_user_object(remote_acct)
-    acct = db.session.merge(acct)
-
-    new_token.account = acct
-    db.session.commit()
-
-    return new_token
-
-def get_twitter_for_acc(account):
-
-    consumer_key = app.config['TWITTER_CONSUMER_KEY']
-    consumer_secret = app.config['TWITTER_CONSUMER_SECRET']
-
-    tokens = OAuthToken.query.with_parent(account).order_by(db.desc(OAuthToken.created_at)).all()
-    for token in tokens:
-        t = Twitter(
-                auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret))
-        try:
-            t.account.verify_credentials()
-            return t
-        except TwitterHTTPError as e:
-            if e.e.code == 401:
-                # token revoked
-                db.session.delete(token)
-                db.session.commit()
-            else:
-                # temporary error, re-raise
-                raise e
-
-    # if no tokens are valid, we log out the user so we'll get a fresh
-    # token when they log in again
-    account.force_log_out()
-    return None
-
-locale.setlocale(locale.LC_TIME, 'C')
-
-def post_from_api_tweet_object(tweet, post=None):
-    if not post:
-        post = Post()
-    post.twitter_id = tweet['id_str']
-    try:
-        post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
-    except ValueError:
-        post.created_at = datetime.strptime(tweet['created_at'], '%Y-%m-%d %H:%M:%S %z')
-        #whyyy
-    if 'full_text' in tweet:
-        post.body = tweet['full_text']
-    else:
-        post.body = tweet['text']
-    post.author_id = 'twitter:{}'.format(tweet['user']['id_str'])
-    if 'favorited' in tweet:
-        post.favourite = tweet['favorited']
-    if 'entities' in tweet:
-        post.has_media = bool('media' in tweet['entities'] and tweet['entities']['media'])
-    return post
-
-def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
-    t = get_twitter_for_acc(account)
-    if not t:
-        print("no twitter access, aborting")
-        return
-
-    user = t.account.verify_credentials()
-    db.session.merge(account_from_api_user_object(user))
-
-    kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' }
-    if cursor:
-        kwargs.update(cursor)
-
-    if 'max_id' not in kwargs:
-        most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first()
-        if most_recent_post:
-            kwargs['since_id'] = most_recent_post.twitter_id
-
-    tweets = t.statuses.user_timeline(**kwargs)
-
-    print("processing {} tweets for {acc}".format(len(tweets), acc=account))
-
-    if len(tweets) > 0:
-
-        kwargs['max_id'] = +inf
-
-        for tweet in tweets:
-            db.session.merge(post_from_api_tweet_object(tweet))
-            kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id'])
-
-    else:
-        kwargs = None
-
-    db.session.commit()
-
-    return kwargs
-
-
-def refresh_posts(posts):
-    if not posts:
-        return posts
-
-    t = get_twitter_for_acc(posts[0].author)
-    tweets = t.statuses.lookup(_id=",".join((post.twitter_id for post in posts)),
-            trim_user = True, tweet_mode = 'extended')
-    refreshed_posts = list()
-    for post in posts:
-        tweet = next((tweet for tweet in tweets if tweet['id_str'] == post.twitter_id), None)
-        if not tweet:
-            db.session.delete(post)
-        else:
-            post = db.session.merge(post_from_api_tweet_object(tweet))
-            refreshed_posts.append(post)
-
-    return refreshed_posts
-
-
-def delete(post):
-    t = get_twitter_for_acc(post.author)
-    t.statuses.destroy(id=post.twitter_id)
-    db.session.delete(post)

libforget/auth.py

@@ -0,0 +1,52 @@
+from flask import g, redirect, jsonify, make_response, abort, request
+from functools import wraps
+
+
+def require_auth(fun):
+    @wraps(fun)
+    def wrapper(*args, **kwargs):
+        if not g.viewer:
+            return redirect('/')
+        return fun(*args, **kwargs)
+    return wrapper
+
+
+def require_auth_api(fun):
+    @wraps(fun)
+    def wrapper(*args, **kwargs):
+        if not g.viewer:
+            return make_response((
+                jsonify(status='error', error='not logged in'),
+                403))
+        return fun(*args, **kwargs)
+    return wrapper
+
+
+def csrf(fun):
+    @wraps(fun)
+    def wrapper(*args, **kwargs):
+        if request.form.get('csrf-token') != g.viewer.csrf_token:
+            return abort(403)
+        return fun(*args, **kwargs)
+    return wrapper
+
+
+def set_session_cookie(session, response, secure=True):
+    response.set_cookie(
+        'forget_sid', session.id,
+        max_age=60*60*48,
+        httponly=True,
+        secure=secure)
+
+
+def get_viewer_session():
+    from model import Session
+    sid = request.cookies.get('forget_sid', None)
+    if sid:
+        return Session.query.get(sid)
+
+
+def get_viewer():
+    session = get_viewer_session()
+    if session:
+        return session.account

libforget/brotli.py

@@ -0,0 +1,94 @@
+import brotli as brotli_
+from flask import request, make_response
+from threading import Thread
+from hashlib import sha256
+import redis as libredis
+import os.path
+import mimetypes
+from redis.exceptions import RedisError
+
+
+class BrotliCache(object):
+    def __init__(self, redis_uri='redis://', timeout=0.100, expire=60*60*6):
+        self._redis = None
+        self._redis_uri = redis_uri
+        self.timeout = timeout
+        self.expire = expire
+
+    @property
+    def redis(self):
+        if not self._redis:
+            self._redis = libredis.StrictRedis.from_url(self._redis_uri)
+            self._redis.client_setname('brotlicache')
+        return self._redis
+
+    def compress_and_cache(self, cache_key, lock_key, body, mode=brotli_.MODE_GENERIC):
+        encbody = brotli_.compress(body, mode=mode)
+        self.redis.set(cache_key, encbody, px=int(self.expire*1000))
+        self.redis.delete(lock_key)
+
+    def wrap_response(self, response):
+        if 'br' not in request.accept_encodings or response.is_streamed:
+            return response
+
+        body = response.get_data()
+        digest = sha256(body).hexdigest()
+        cache_key = 'brotlicache:{}'.format(digest)
+
+        try:
+            encbody = self.redis.get(cache_key)
+            response.headers.set('brotli-cache', 'HIT')
+            if not encbody:
+                response.headers.set('brotli-cache', 'MISS')
+                lock_key = 'brotlicache:lock:{}'.format(digest)
+                if self.redis.set(lock_key, 1, nx=True, ex=10):
+                    mode = (
+                        brotli_.MODE_TEXT
+                        if response.content_type.startswith('text/')
+                        else brotli_.MODE_GENERIC)
+                    t = Thread(
+                        target=self.compress_and_cache,
+                        args=(cache_key, lock_key, body, mode))
+                    t.start()
+                    if self.timeout > 0:
+                        t.join(self.timeout)
+                        encbody = self.redis.get(cache_key)
+                    if not encbody:
+                        response.headers.set('brotli-cache', 'TIMEOUT')
+                else:
+                    response.headers.set('brotli-cache', 'LOCKED')
+            if encbody:
+                response.headers.set('content-encoding', 'br')
+                response.headers.set('vary', 'accept-encoding')
+                response.set_data(encbody)
+                return response
+        except RedisError:
+            response.headers.set('brotli-cache', 'ERROR')
+
+        return response
+
+
+def brotli(app, static=True, dynamic=True, **kwargs):
+    original_static = app.view_functions['static']
+
+    def static_maybe_gzip_brotli(filename=None):
+        path = os.path.join(app.static_folder, filename)
+        for encoding, extension in (('br', '.br'), ('gzip', '.gz')):
+            if encoding not in request.accept_encodings:
+                continue
+            encpath = path + extension
+            if os.path.isfile(encpath):
+                resp = make_response(
+                    original_static(filename=filename + extension))
+                resp.headers.set('content-encoding', encoding)
+                resp.headers.set('vary', 'accept-encoding')
+                mimetype = (mimetypes.guess_type(filename)[0]
+                            or 'application/octet-stream')
+                resp.headers.set('content-type', mimetype)
+                return resp
+        return original_static(filename=filename)
+    if static:
+        app.view_functions['static'] = static_maybe_gzip_brotli
+    if dynamic:
+        cache = BrotliCache(redis_uri=app.config.get('REDIS_URI'), **kwargs)
+        app.after_request(cache.wrap_response)

libforget/cachebust.py

@@ -1,22 +1,31 @@
 from flask import url_for, abort
 import os
+
+
 def cachebust(app):
-    @app.route('/static-<int:timestamp>/<path:filename>')
+    # pylint: disable=unused-variable
+
+    @app.route('/static-cb/<int:timestamp>/<path:filename>')
     def static_cachebust(timestamp, filename):
         path = os.path.join(app.static_folder, filename)
-        mtime = os.stat(path).st_mtime
+        try:
+            mtime = os.stat(path).st_mtime
+        except Exception:
+            return abort(404)
         if abs(mtime - timestamp) > 1:
             abort(404)
         else:
             resp = app.view_functions['static'](filename=filename)
-            resp.headers.set('cache-control', 'public, immutable, max-age=%s' % (60*60*24*365,))
+            resp.headers.set(
+                'cache-control',
+                'public, immutable, max-age={}'.format(60*60*24*365))
             if 'expires' in resp.headers:
                 resp.headers.remove('expires')
             return resp
 
     @app.context_processor
     def replace_url_for():
-        return dict(url_for = cachebust_url_for)
+        return dict(url_for=cachebust_url_for)
 
     def cachebust_url_for(endpoint, **kwargs):
         if endpoint == 'static':

libforget/exceptions.py

@@ -0,0 +1,6 @@
+class PermanentError(Exception):
+    pass
+
+
+class TemporaryError(Exception):
+    pass

libforget/img_proxy.py

@@ -0,0 +1,125 @@
+import requests
+import threading
+import redis as libredis
+from flask import make_response, abort
+import secrets
+import hmac
+import base64
+import pickle # nosec
+import re
+
+
+class ImgProxyCache(object):
+    def __init__(self, redis_uri='redis://', timeout=10, expire=60*60,
+                 prefix='img_proxy', hmac_hash='sha1'):
+        self._redis = None
+        self._redis_uri = redis_uri
+        self.timeout = timeout
+        self.expire = expire
+        self.prefix = prefix
+        self.hash = hmac_hash
+        self.hmac_key = None
+
+    @property
+    def redis(self):
+        if not self._redis:
+            self._redis = libredis.StrictRedis.from_url(self._redis_uri)
+            self._redis.client_setname('img_proxy')
+        return self._redis
+
+    def key(self, *args):
+        return '{prefix}:1:{args}'.format(
+                prefix=self.prefix, args=":".join(args))
+
+    def token(self):
+        if not self.hmac_key:
+            t = self.redis.get(self.key('hmac_key'))
+            if not t:
+                t = secrets.token_urlsafe().encode('ascii')
+                self.redis.set(self.key('hmac_key'), t)
+            self.hmac_key = t
+        return self.hmac_key
+
+    def identifier_for(self, url):
+        url_hmac = hmac.new(self.token(), url.encode('UTF-8'), self.hash)
+        return base64.urlsafe_b64encode(
+                '{}:{}'.format(url_hmac.hexdigest(), url)
+                .encode('UTF-8')
+                ).strip(b'=').decode('UTF-8')
+
+    def url_for(self, identifier):
+        try:
+            padding = (4 - len(identifier)) % 4
+            identifier += padding * '='
+            identifier = base64.urlsafe_b64decode(identifier).decode('UTF-8')
+            received_hmac, url = identifier.split(':', 1)
+            url_hmac = hmac.new(self.token(), url.encode('UTF-8'), self.hash)
+            if not hmac.compare_digest(url_hmac.hexdigest(), received_hmac):
+                return None
+        except Exception:
+            return None
+        return url
+
+    def fetch_and_cache(self, url):
+        resp = requests.get(url)
+        if(resp.status_code != 200):
+            return
+
+        allowed_headers = [
+                'content-type',
+                'cache-control',
+                'etag',
+                'date',
+                'last-modified',
+                ]
+        headers = {}
+
+        expire = self.expire
+        if 'cache-control' in resp.headers:
+            for value in resp.headers['cache-control'].split(','):
+                match = re.match(' *max-age *= *([0-9]+) *', value)
+                if match:
+                    expire = max(self.expire, int(match.group(1)))
+
+        for key in allowed_headers:
+            if key in resp.headers:
+                headers[key] = resp.headers[key]
+        self.redis.set(self.key('headers', url), pickle.dumps(headers, -1),
+                       px=expire*1000)
+        self.redis.set(self.key('body', url),
+                       resp.content, px=expire*1000)
+
+    def respond(self, identifier):
+        url = self.url_for(identifier)
+        if not url:
+            return abort(403)
+
+        x_imgproxy_cache = 'HIT'
+        headers = self.redis.get(self.key('headers', url))
+        body = self.redis.get(self.key('body', url))
+
+        if not body or not headers:
+            x_imgproxy_cache = 'MISS'
+            if self.redis.set(
+                    self.key('lock', url), 1, nx=True, ex=10*self.timeout):
+                t = threading.Thread(target=self.fetch_and_cache, args=(url,))
+                t.start()
+                t.join(self.timeout)
+                headers = self.redis.get(self.key('headers', url))
+                body = self.redis.get(self.key('body', url))
+
+        try:
+            headers = pickle.loads(headers) # nosec
+        except Exception:
+            self.redis.delete(self.key('headers', url))
+            headers = None
+
+        if not body or not headers:
+            return abort(404)
+
+        resp = make_response(body, 200)
+        resp.headers.set('imgproxy-cache', x_imgproxy_cache)
+        resp.headers.set('cache-control', 'max-age={}'.format(self.expire))
+        for key, value in headers.items():
+            resp.headers.set(key, value)
+        return resp

libforget/interval.py

@@ -1,30 +1,6 @@
-from datetime import timedelta
-from statistics import mean
+from datetime import timedelta, datetime, timezone
+from .timescales import SCALES
 
-SCALES = [
-    ('minutes', timedelta(minutes=1)),
-    ('hours', timedelta(hours=1)),
-    ('days', timedelta(days=1)),
-    ('weeks', timedelta(days=7)),
-    ('months', timedelta(days=
-    # you, a fool: a month is 30 days
-    # me, wise:
-        mean((31,
-            mean((29 if year % 400 == 0
-                        or (year % 100 != 0 and year % 4 == 0)
-                    else 28
-                    for year in range(400)))
-            ,31,30,31,30,31,31,30,31,30,31))
-        )),
-    ('years', timedelta(days=
-    # you, a fool: ok. a year is 365.25 days. happy?
-    # me, wise: absolutely not
-        mean((366 if year % 400 == 0
-                or (year % 100 != 0 and year % 4 == 0)
-            else 365
-            for year in range(400)))
-        )),
-]
 
 def decompose_interval(attrname):
     scales = [scale[1] for scale in SCALES]
@@ -48,7 +24,7 @@ def decompose_interval(attrname):
 
         @scale.setter
         def scale(self, value):
-            if(type(value) != timedelta):
+            if not isinstance(value, timedelta):
                 value = timedelta(seconds=float(value))
             setattr(self, attrname, max(1, getattr(self, sig_name)) * value)
 
@@ -58,20 +34,47 @@ def decompose_interval(attrname):
 
         @significand.setter
         def significand(self, value):
-            if type(value) == str and value.strip() == '':
+            if isinstance(value, str) and value.strip() == '':
                 value = 0
 
             try:
                 value = int(value)
-                assert value >= 0
-            except (ValueError, AssertionError) as e:
+                if not value >= 0:
+                    raise ValueError(value)
+            except ValueError as e:
                 raise ValueError("Incorrect time interval", e)
             setattr(self, attrname, value * getattr(self, scl_name))
 
-
         setattr(cls, scl_name, scale)
         setattr(cls, sig_name, significand)
 
         return cls
 
     return decorator
+
+
+def relative(interval):
+    # special cases
+    if interval > timedelta(seconds=-15) and interval < timedelta(0):
+        return "just now"
+    elif interval > timedelta(0) and interval < timedelta(seconds=15):
+        return "in a few seconds"
+    else:
+        output = None
+        for name, scale in reversed(SCALES):
+            if abs(interval) > scale:
+                value = abs(interval) // scale
+                output = '{} {}'.format(value, name)
+                if value == 1:
+                    output = output[:-1]
+                break
+        if not output:
+            output = '{} seconds'.format(abs(interval).seconds)
+        if interval > timedelta(0):
+            return 'in {}'.format(output)
+        else:
+            return '{} ago'.format(output)
+
+
+def relnow(time):
+    return relative(time - datetime.now(timezone.utc))

libforget/json.py

@@ -0,0 +1,23 @@
+from json import dumps
+
+
+def account(acc):
+    last_delete = None
+    next_delete = None
+    if acc.last_delete:
+        last_delete = acc.last_delete.isoformat()
+    if acc.next_delete:
+        next_delete = acc.next_delete.isoformat()
+    return dumps(dict(
+            post_count=acc.post_count(),
+            eligible_for_delete_estimate=acc.estimate_eligible_for_delete(),
+            display_name=acc.display_name,
+            screen_name=acc.screen_name,
+            avatar_url=acc.get_avatar(),
+            avatar_url_orig=acc.avatar_url,
+            id=acc.id,
+            service=acc.service,
+            policy_enabled=acc.policy_enabled,
+            next_delete=next_delete,
+            last_delete=last_delete,
+        ))

libforget/mastodon.py

@@ -0,0 +1,205 @@
+from mastodon import Mastodon
+from mastodon.Mastodon import MastodonAPIError,\
+                              MastodonNetworkError,\
+                              MastodonNotFoundError,\
+                              MastodonRatelimitError,\
+                              MastodonUnauthorizedError
+from model import MastodonApp, Account, OAuthToken, Post, MastodonInstance
+from requests import head
+import requests
+from app import db, sentry
+from libforget.exceptions import TemporaryError
+from functools import lru_cache
+from libforget.session import make_session
+
+
+def get_or_create_app(instance_url, callback, website):
+    instance_url = instance_url
+    app = MastodonApp.query.get(instance_url)
+    try:
+        head('https://{}'.format(instance_url)).raise_for_status()
+        proto = 'https'
+    except Exception:
+        head('http://{}'.format(instance_url)).raise_for_status()
+        proto = 'http'
+
+    if not app:
+        client_id, client_secret = Mastodon.create_app(
+                'forget',
+                scopes=('read', 'write'),
+                api_base_url='{}://{}'.format(proto, instance_url),
+                redirect_uris=callback,
+                website=website,
+            )
+        app = MastodonApp()
+        app.instance = instance_url
+        app.client_id = client_id
+        app.client_secret = client_secret
+        app.protocol = proto
+    return app
+
+
+
+def anonymous_api(app):
+    return Mastodon(
+            app.client_id,
+            client_secret=app.client_secret,
+            api_base_url='{}://{}'.format(app.protocol, app.instance),
+            session=make_session(),
+            )
+
+
+def login_url(app, callback):
+    return anonymous_api(app).auth_request_url(
+            redirect_uris=callback,
+            scopes=('read', 'write',)
+            )
+
+
+def receive_code(code, app, callback):
+    api = anonymous_api(app)
+    access_token = api.log_in(
+            code=code,
+            scopes=('read', 'write'),
+            redirect_uri=callback,
+            )
+
+    remote_acc = api.account_verify_credentials()
+    acc = account_from_api_object(remote_acc, app.instance)
+    acc = db.session.merge(acc)
+    token = OAuthToken(token=access_token)
+    token = db.session.merge(token)
+    token.account = acc
+
+    return token
+
+
+@lru_cache()
+def get_api_for_acc(account):
+    app = MastodonApp.query.get(account.mastodon_instance)
+    for token in account.tokens:
+        api = Mastodon(
+                app.client_id,
+                client_secret=app.client_secret,
+                api_base_url='{}://{}'.format(app.protocol, app.instance),
+                access_token=token.token,
+                ratelimit_method='throw',
+                session=make_session(),
+            )
+        try:
+            # api.verify_credentials()
+            # doesnt error even if the token is revoked lol
+            # https://github.com/tootsuite/mastodon/issues/4637
+            # so we have to do this:
+            api.timeline()
+            if api.ratelimit_remaining / api.ratelimit_limit < 1/4:
+                raise TemporaryError("Rate limit too low")
+            return api
+        except MastodonUnauthorizedError as e:
+            if sentry:
+                sentry.captureMessage(
+                        'Mastodon auth revoked or incorrect',
+                        extra=locals())
+            db.session.delete(token)
+            db.session.commit()
+            continue
+        except MastodonAPIError as e:
+            raise TemporaryError(e)
+        except (MastodonNetworkError,
+                MastodonRatelimitError) as e:
+            raise TemporaryError(e)
+    raise TemporaryError('No access to account {}'.format(account))
+
+
+def fetch_posts(acc, max_id, since_id):
+    api = get_api_for_acc(acc)
+
+    try:
+        newacc = account_from_api_object(
+                api.account_verify_credentials(), acc.mastodon_instance)
+        acc = db.session.merge(newacc)
+
+        kwargs = dict(limit=40)
+        if max_id:
+            kwargs['max_id'] = max_id
+        if since_id:
+            kwargs['since_id'] = since_id
+
+        statuses = api.account_statuses(acc.mastodon_id, **kwargs)
+
+        return [post_from_api_object(status, acc.mastodon_instance) for status in statuses]
+
+    except (MastodonAPIError,
+            MastodonNetworkError,
+            MastodonRatelimitError) as e:
+        raise TemporaryError(e)
+
+
+def post_from_api_object(obj, instance):
+    return Post(
+            mastodon_instance=instance,
+            mastodon_id=obj['id'],
+            favourite=obj['favourited'],
+            has_media=('media_attachments' in obj
+                       and bool(obj['media_attachments'])),
+            created_at=obj['created_at'],
+            author_id=account_from_api_object(obj['account'], instance).id,
+            direct=obj['visibility'] == 'direct',
+            is_reblog=obj['reblog'] is not None,
+        )
+
+
+def account_from_api_object(obj, instance):
+    return Account(
+            mastodon_instance=instance,
+            mastodon_id=obj['id'],
+            screen_name='{}@{}'.format(obj['username'], instance),
+            display_name=obj['display_name'],
+            avatar_url=obj['avatar'],
+            reported_post_count=obj['statuses_count'],
+        )
+
+
+def refresh_posts(posts):
+    acc = posts[0].author
+    api = get_api_for_acc(acc)
+
+    new_posts = list()
+    with db.session.no_autoflush:
+        for post in posts:
+            print('Refreshing {}'.format(post))
+            try:
+                status = api.status(post.mastodon_id)
+                new_post = db.session.merge(
+                        post_from_api_object(status, post.mastodon_instance))
+                new_post.touch()
+                new_posts.append(new_post)
+            except MastodonNotFoundError:
+                db.session.delete(post)
+            except (MastodonAPIError,
+                    MastodonNetworkError,
+                    MastodonRatelimitError) as e:
+                raise TemporaryError(e)
+
+    return new_posts
+
+
+def delete(post):
+    api = get_api_for_acc(post.author)
+    try:
+        api.status_delete(post.mastodon_id)
+        db.session.delete(post)
+    except (MastodonAPIError,
+            MastodonNetworkError,
+            MastodonRatelimitError) as e:
+        raise TemporaryError(e)
+
+
+def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
+    return tuple((ins.instance for ins in (
+            MastodonInstance.query
+            .filter(MastodonInstance.popularity > min_popularity)
+            .filter(~MastodonInstance.instance.in_(blocklist))
+            .order_by(db.desc(MastodonInstance.popularity),
+                      MastodonInstance.instance)
+            .limit(limit).all())))

libforget/misskey.py

@@ -0,0 +1,191 @@
+from app import db, sentry
+from model import MisskeyApp, MisskeyInstance, Account, OAuthToken, Post
+from uuid import uuid4
+from hashlib import sha256
+from libforget.exceptions import TemporaryError, PermanentError
+from libforget.session import make_session
+
+def get_or_create_app(instance_url, callback, website, session):
+    instance_url = instance_url
+    app = MisskeyApp.query.get(instance_url)
+    
+    if not app:
+        # check if the instance uses https while getting instance infos
+        try:
+            r = session.post('https://{}/api/meta'.format(instance_url))
+            r.raise_for_status()
+            proto = 'https'
+        except Exception:
+            r = session.post('http://{}/api/meta'.format(instance_url))
+            r.raise_for_status()
+            proto = 'http'
+        
+        # This is using the legacy authentication method, because the newer
+        # Miauth method breaks the ability to log out and log back into forget.
+        
+        app = MisskeyApp()
+        app.instance = instance_url
+        app.protocol = proto
+        
+        # register the app
+        r = session.post('{}://{}/api/app/create'.format(app.protocol, app.instance), json = {
+            'name': 'forget',
+            'description': website,
+            'permission': ['write:notes', 'read:reactions'],
+            'callbackUrl': callback
+        })
+        r.raise_for_status()
+        app.client_secret = r.json()['secret']
+
+    return app
+
+def login_url(app, callback, session):
+    # will use the callback we gave the server in `get_or_create_app`
+    r = session.post('{}://{}/api/auth/session/generate'.format(app.protocol, app.instance), json = {
+        'appSecret': app.client_secret
+    })
+    r.raise_for_status()
+    # we already get the retrieval token here, but we get it again later so
+    # we do not have to store it
+    return r.json()['url']
+
+def receive_token(token, app):
+    session = make_session()
+
+    r = session.post('{}://{}/api/auth/session/userkey'.format(app.protocol, app.instance), json = {
+        'appSecret': app.client_secret,
+        'token': token
+    })
+    r.raise_for_status()
+
+    token = sha256(r.json()['accessToken'].encode('utf-8') + app.client_secret.encode('utf-8')).hexdigest()
+
+    acc = account_from_user(r.json()['user'], app.instance)
+    acc = db.session.merge(acc)
+    token = OAuthToken(token = token)
+    token = db.session.merge(token)
+    token.account = acc
+
+    return token
+
+def check_auth(account, app, session):
+    # there is no explicit check, we can only try getting user info
+    r = session.post('{}://{}/api/i'.format(app.protocol, app.instance), json = {'i': account.tokens[0].token})
+
+    if r.status_code != 200:
+        raise TemporaryError("{} {}".format(r.status_code, r.text))
+
+    if r.json()['isSuspended']:
+        # this is technically a temporary error, but like for twitter
+        # its handled as permanent to not make useless API calls
+        raise PermanentError("Misskey account suspended")
+
+def account_from_user(user, host):
+    return Account(
+            # in objects that get returned from misskey, the local host is
+            # set to None
+            misskey_instance=host,
+            misskey_id=user['id'],
+            screen_name='{}@{}'.format(user['username'], host),
+            display_name=user['name'],
+            avatar_url=user['avatarUrl'],
+            # the notes count is not always included, especially not when
+            # fetching posts. in that case assume its not needed
+            reported_post_count=user.get('notesCount', None),
+        )
+
+def post_from_api_object(obj, host):
+    return Post(
+            # in objects that get returned from misskey, the local host is
+            # set to None
+            misskey_instance=host,
+            misskey_id=obj['id'],
+            favourite=('myReaction' in obj
+                       and bool(obj['myReaction'])),
+            has_media=('fileIds' in obj
+                       and bool(obj['fileIds'])),
+            created_at=obj['createdAt'],
+            author_id=account_from_user(obj['user'], host).id,
+            direct=obj['visibility'] == 'specified',
+            is_reblog=obj['renoteId'] is not None,
+        )
+
+def fetch_posts(acc, max_id, since_id):
+    app = MisskeyApp.query.get(acc.misskey_instance)
+    session = make_session()
+    check_auth(acc, app, session)
+
+    kwargs = dict(
+        limit=100,
+        userId=acc.misskey_id,
+        # for some reason the token is needed so misskey also sends `myReaction`
+        i=acc.tokens[0].token
+    )
+    if max_id:
+        kwargs['untilId'] = max_id
+    if since_id:
+        kwargs['sinceId'] = since_id
+
+    notes = session.post('{}://{}/api/users/notes'.format(app.protocol, app.instance), json=kwargs)
+
+    if notes.status_code != 200:
+        raise TemporaryError('{} {}'.format(notes.status_code, notes.text))
+
+    return [post_from_api_object(note, app.instance) for note in notes.json()]
+
+
+def refresh_posts(posts):
+    acc = posts[0].author
+    app = MisskeyApp.query.get(acc.misskey_instance)
+    session = make_session()
+    check_auth(acc, app, session)
+
+    new_posts = list()
+    with db.session.no_autoflush:
+        for post in posts:
+            print('Refreshing {}'.format(post))
+            r = session.post('{}://{}/api/notes/show'.format(app.protocol, app.instance), json={
+                'i': acc.tokens[0].token,
+                'noteId': post.misskey_id
+            })
+            if r.status_code != 200:
+                try:
+                    if r.json()['error']['code'] == 'NO_SUCH_NOTE':
+                        db.session.delete(post)
+                        continue
+                except Exception as e:
+                    raise TemporaryError(e)
+                raise TemporaryError('{} {}'.format(r.status_code, r.text))
+
+            new_post = db.session.merge(post_from_api_object(r.json(), app.instance))
+            new_post.touch()
+            new_posts.append(new_post)
+    return new_posts
+
+def delete(post):
+    acc = post.author
+    app = MisskeyApp.query.get(post.misskey_instance)
+    session = make_session()
+    if not app:
+        # how? if this happens, it doesnt make sense to repeat it,
+        # so use a permanent error
+        raise PermanentError("instance not registered for delete")
+
+    r = session.post('{}://{}/api/notes/delete'.format(app.protocol, app.instance), json = {
+        'i': acc.tokens[0].token,
+        'noteId': post.misskey_id
+    })
+    
+    if r.status_code != 204:
+        raise TemporaryError("{} {}".format(r.status_code, r.text))
+
+    db.session.delete(post)
+
+def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
+    return tuple((ins.instance for ins in (
+            MisskeyInstance.query
+            .filter(MisskeyInstance.popularity > min_popularity)
+            .filter(~MisskeyInstance.instance.in_(blocklist))
+            .order_by(db.desc(MisskeyInstance.popularity),
+                      MisskeyInstance.instance)
+            .limit(limit).all())))

libforget/session.py

@@ -0,0 +1,14 @@
+import requests
+import version
+
+
+def make_session():
+    s = requests.Session()
+    s.headers.update(
+        {
+            "user-agent": "Forget/{version} +https://forget.codl.fr".format(
+                version=version.get_versions()["version"]
+            )
+        }
+    )
+    return s

libforget/settings.py

@@ -6,4 +6,6 @@ attrs = (
     'policy_keep_younger_scale',
     'policy_keep_younger_significand',
     'policy_keep_media',
+    'policy_keep_direct',
+    'policy_enabled',
     )

libforget/timescales.py

@@ -0,0 +1,29 @@
+# flake8: noqa
+from datetime import timedelta
+from statistics import mean
+
+SCALES = [
+    ('minutes', timedelta(minutes=1)),
+    ('hours', timedelta(hours=1)),
+    ('days', timedelta(days=1)),
+    ('weeks', timedelta(days=7)),
+    ('months', timedelta(days=
+    # you, a fool: a month is 30 days
+    # me, wise:
+        mean((31,
+            mean((29 if year % 400 == 0
+                        or (year % 100 != 0 and year % 4 == 0)
+                    else 28
+                    for year in range(400)))
+            ,31,30,31,30,31,31,30,31,30,31))
+        )),
+    ('years', timedelta(days=
+    # you, a fool: ok. a year is 365.25 days. happy?
+    # me, wise: absolutely not
+        mean((366 if year % 400 == 0
+                or (year % 100 != 0 and year % 4 == 0)
+            else 365
+            for year in range(400)))
+        )),
+]
+

libforget/twitter.py

@@ -0,0 +1,201 @@
+from twitter import Twitter, OAuth, TwitterHTTPError, TwitterError
+from werkzeug.urls import url_decode
+from model import OAuthToken, Account, Post, TwitterArchive
+from app import db, app, sentry
+from math import inf
+from datetime import datetime
+import locale
+from zipfile import ZipFile
+from io import BytesIO
+from libforget.exceptions import PermanentError, TemporaryError
+from urllib.error import URLError
+
+
+def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
+    twitter = Twitter(
+            auth=OAuth('', '', consumer_key, consumer_secret),
+            format='', api_version=None)
+    resp = url_decode(twitter.oauth.request_token(oauth_callback=callback))
+    oauth_token = resp['oauth_token']
+    oauth_token_secret = resp['oauth_token_secret']
+
+    token = OAuthToken(token=oauth_token, token_secret=oauth_token_secret)
+    db.session.merge(token)
+    db.session.commit()
+
+    return (
+        "https://api.twitter.com/oauth/authenticate?oauth_token=%s"
+        % (oauth_token,))
+
+
+def account_from_api_user_object(obj):
+    return Account(
+            twitter_id=obj['id_str'],
+            display_name=obj['name'],
+            screen_name=obj['screen_name'],
+            avatar_url=obj['profile_image_url_https'],
+            reported_post_count=obj['statuses_count'])
+
+
+def receive_verifier(oauth_token, oauth_verifier,
+                     consumer_key=None, consumer_secret=None):
+    temp_token = OAuthToken.query.get(oauth_token)
+    if not temp_token:
+        raise Exception("OAuth token has expired")
+    twitter = Twitter(
+            auth=OAuth(temp_token.token, temp_token.token_secret,
+                       consumer_key, consumer_secret),
+            format='', api_version=None)
+    resp = url_decode(
+            twitter.oauth.access_token(oauth_verifier=oauth_verifier))
+    db.session.delete(temp_token)
+    new_token = OAuthToken(token=resp['oauth_token'],
+                           token_secret=resp['oauth_token_secret'])
+    new_token = db.session.merge(new_token)
+    new_twitter = Twitter(
+            auth=OAuth(new_token.token, new_token.token_secret,
+                       consumer_key, consumer_secret))
+    remote_acct = new_twitter.account.verify_credentials()
+    acct = account_from_api_user_object(remote_acct)
+    acct = db.session.merge(acct)
+
+    new_token.account = acct
+    db.session.commit()
+
+    return new_token
+
+
+def get_twitter_for_acc(account):
+    consumer_key = app.config['TWITTER_CONSUMER_KEY']
+    consumer_secret = app.config['TWITTER_CONSUMER_SECRET']
+
+    tokens = (OAuthToken.query.with_parent(account)
+              .order_by(db.desc(OAuthToken.created_at)).all())
+    for token in tokens:
+        t = Twitter(
+                auth=OAuth(token.token, token.token_secret,
+                           consumer_key, consumer_secret))
+        try:
+            t.account.verify_credentials()
+            return t
+        except TwitterHTTPError as e:
+            if e.e.code == 401:
+                # token revoked
+
+                if sentry:
+                    sentry.captureMessage(
+                            'Twitter auth revoked', extra=locals())
+                db.session.delete(token)
+                db.session.commit()
+            else:
+                raise TemporaryError(e)
+        except URLError as e:
+            raise TemporaryError(e)
+
+    raise TemporaryError("No access to account {}".format(account))
+
+
+locale.setlocale(locale.LC_TIME, 'C')
+
+
+def post_from_api_tweet_object(tweet, post=None):
+    if not post:
+        post = Post()
+    post.twitter_id = tweet['id_str']
+    try:
+        post.created_at = datetime.strptime(
+                tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
+    except ValueError:
+        post.created_at = datetime.strptime(
+                tweet['created_at'], '%Y-%m-%d %H:%M:%S %z')
+        # whyyy
+    post.author_id = 'twitter:{}'.format(tweet['user']['id_str'])
+    if 'favorited' in tweet:
+        post.favourite = tweet['favorited']
+    if 'entities' in tweet:
+        post.has_media = bool(
+                'media' in tweet['entities'] and tweet['entities']['media'])
+    post.is_reblog = 'retweeted_status' in tweet
+    return post
+
+
+def fetch_posts(account, max_id, since_id):
+    t = get_twitter_for_acc(account)
+
+    try:
+        user = t.account.verify_credentials()
+        db.session.merge(account_from_api_user_object(user))
+
+        kwargs = {
+                'user_id': account.twitter_id,
+                'count': 200,
+                'trim_user': True,
+                'tweet_mode': 'extended',
+                }
+        if max_id:
+            kwargs['max_id'] = max_id
+        if since_id:
+            kwargs['since_id'] = since_id
+
+        tweets = t.statuses.user_timeline(**kwargs)
+    except (TwitterError, URLError) as e:
+        handle_error(e)
+
+    return [post_from_api_tweet_object(tweet) for tweet in tweets]
+
+def refresh_posts(posts):
+    if not posts:
+        return posts
+
+    t = get_twitter_for_acc(posts[0].author)
+    try:
+        tweets = t.statuses.lookup(
+                    _id=",".join((post.twitter_id for post in posts)),
+                    trim_user=True, tweet_mode='extended')
+    except (URLError, TwitterError) as e:
+        handle_error(e)
+    refreshed_posts = list()
+    for post in posts:
+        tweet = next(
+            (tweet for tweet in tweets if tweet['id_str'] == post.twitter_id),
+            None)
+        if not tweet:
+            db.session.delete(post)
+        else:
+            post = db.session.merge(post_from_api_tweet_object(tweet))
+            post.touch()
+            refreshed_posts.append(post)
+
+    return refreshed_posts
+
+
+def delete(post):
+    t = get_twitter_for_acc(post.author)
+    t.statuses.destroy(id=post.twitter_id)
+    db.session.delete(post)
+
+
+def chunk_twitter_archive(archive_id):
+    ta = TwitterArchive.query.get(archive_id)
+
+    with ZipFile(BytesIO(ta.body), 'r') as zipfile:
+        files = [filename for filename in zipfile.namelist()
+                 if filename.startswith('data/js/tweets/')
+                 and filename.endswith('.js')]
+
+    files.sort()
+
+    return files
+
+
+def handle_error(e):
+    if isinstance(e, TwitterHTTPError):
+        data = e.response_data
+        if isinstance(data, dict) and 'errors' in data.keys():
+            for error in data['errors']:
+                if error.get('code',0) == 326:
+                    # account locked lol rip
+                    # although this is a temporary error in twitter terms
+                    # it's best not to waste api calls on locked accounts
+                    raise PermanentError(e)
+    raise TemporaryError(e)

libforget/version.py

@@ -0,0 +1,4 @@
+from app import app
+
+def url_for_version(ver):
+    return app.config['CHANGELOG_URL'].format(hash=ver['full-revisionid'])

migrations/versions/2bd33abe291c_add_three_way_favourite_policy.py

@@ -0,0 +1,53 @@
+"""add three-way favourite policy
+
+Revision ID: 2bd33abe291c
+Revises: 583cdac8eba1
+Create Date: 2018-01-03 17:31:03.718648
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2bd33abe291c'
+down_revision = '583cdac8eba1'
+branch_labels = None
+depends_on = None
+
+transitional = sa.table('accounts',
+                        sa.column('policy_keep_favourites'),
+                        sa.column('old_policy_keep_favourites'))
+
+
+
+def upgrade():
+    ThreeWayPolicyEnum = sa.Enum('keeponly', 'deleteonly', 'none',
+                                name='enum_3way_policy')
+    op.alter_column('accounts', 'policy_keep_favourites',
+                    new_column_name='old_policy_keep_favourites')
+    op.add_column(
+        'accounts',
+        sa.Column('policy_keep_favourites', ThreeWayPolicyEnum,
+                  nullable=False, server_default='none'))
+
+    op.execute(transitional.update()
+            .where(transitional.c.old_policy_keep_favourites)
+            .values(policy_keep_favourites=op.inline_literal('keeponly')))
+
+    op.drop_column('accounts', 'old_policy_keep_favourites')
+
+
+def downgrade():
+    op.alter_column('accounts', 'policy_keep_favourites',
+                    new_column_name='old_policy_keep_favourites')
+    op.add_column(
+        'accounts',
+        sa.Column('policy_keep_favourites', sa.Boolean(),
+                  nullable=False, server_default='f'))
+
+    op.execute(transitional.update()
+            .where(transitional.c.old_policy_keep_favourites == op.inline_literal('keeponly'))
+            .values(policy_keep_favourites=op.inline_literal('t')))
+
+    op.drop_column('accounts', 'old_policy_keep_favourites')

migrations/versions/3a0138499994_add_reason_to_account.py

@@ -0,0 +1,24 @@
+"""add reason to account
+
+Revision ID: 3a0138499994
+Revises: 41ef02e66382
+Create Date: 2017-09-02 19:46:14.035946
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3a0138499994'
+down_revision = '41ef02e66382'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('reason', sa.String(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('accounts', 'reason')

migrations/versions/41ef02e66382_default_next_delete_to_null.py

@@ -0,0 +1,26 @@
+"""default next_delete to null
+
+Revision ID: 41ef02e66382
+Revises: f95af1a8d89f
+Create Date: 2017-08-31 21:19:44.304952
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '41ef02e66382'
+down_revision = 'f95af1a8d89f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column('accounts', 'next_delete', server_default=None)
+    op.execute("UPDATE accounts SET next_delete = NULL where next_delete = 'epoch';")
+
+
+def downgrade():
+    op.alter_column('accounts', 'next_delete', server_default='epoch')
+    op.execute("UPDATE accounts SET next_delete = 'epoch' where next_delete IS NULL;")

migrations/versions/4b56cde3ebd7_new_fetching_flags.py

@@ -0,0 +1,28 @@
+"""new fetching flags
+
+Revision ID: 4b56cde3ebd7
+Revises: c136aa1157f9
+Create Date: 2019-02-24 11:53:29.128983
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4b56cde3ebd7'
+down_revision = 'c136aa1157f9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('fetch_current_batch_end_id', sa.String(), nullable=True))
+    op.add_column('accounts', sa.Column('fetch_history_complete', sa.Boolean(), server_default='FALSE', nullable=False))
+    op.create_foreign_key(op.f('fk_accounts_fetch_current_batch_end_id_posts'), 'accounts', 'posts', ['fetch_current_batch_end_id'], ['id'], ondelete='SET NULL')
+
+
+def downgrade():
+    op.drop_constraint(op.f('fk_accounts_fetch_current_batch_end_id_posts'), 'accounts', type_='foreignkey')
+    op.drop_column('accounts', 'fetch_history_complete')
+    op.drop_column('accounts', 'fetch_current_batch_end_id')

migrations/versions/583cdac8eba1_add_three_way_media_policy.py

@@ -0,0 +1,59 @@
+"""add three-way media policy
+
+Revision ID: 583cdac8eba1
+Revises: 7e255d4ea34d
+Create Date: 2017-12-28 00:46:56.023649
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '583cdac8eba1'
+down_revision = '7e255d4ea34d'
+branch_labels = None
+depends_on = None
+
+transitional = sa.table('accounts',
+                        sa.column('policy_keep_media'),
+                        sa.column('old_policy_keep_media'))
+
+
+
+def upgrade():
+    ThreeWayPolicyEnum = sa.Enum('keeponly', 'deleteonly', 'none',
+                                name='enum_3way_policy')
+    op.execute("""
+        CREATE TYPE enum_3way_policy AS ENUM ('keeponly', 'deleteonly', 'none')
+    """)
+    op.alter_column('accounts', 'policy_keep_media',
+                    new_column_name='old_policy_keep_media')
+    op.add_column(
+        'accounts',
+        sa.Column('policy_keep_media', ThreeWayPolicyEnum,
+                  nullable=False, server_default='none'))
+
+    op.execute(transitional.update()
+            .where(transitional.c.old_policy_keep_media)
+            .values(policy_keep_media=op.inline_literal('keeponly')))
+
+    op.drop_column('accounts', 'old_policy_keep_media')
+
+
+def downgrade():
+    op.alter_column('accounts', 'policy_keep_media',
+                    new_column_name='old_policy_keep_media')
+    op.add_column(
+        'accounts',
+        sa.Column('policy_keep_media', sa.Boolean(),
+                  nullable=False, server_default='f'))
+
+    op.execute(transitional.update()
+            .where(transitional.c.old_policy_keep_media == op.inline_literal('keeponly'))
+            .values(policy_keep_media=op.inline_literal('t')))
+
+    op.drop_column('accounts', 'old_policy_keep_media')
+    op.execute("""
+        DROP TYPE enum_3way_policy
+    """)

migrations/versions/5fec5f5e8a5e_add_post_direct_and_account_policy_keep_.py

@@ -0,0 +1,26 @@
+"""add post.direct and account.policy_keep_direct
+
+Revision ID: 5fec5f5e8a5e
+Revises: 8993e80e7aa3
+Create Date: 2017-08-20 18:16:26.682744
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '5fec5f5e8a5e'
+down_revision = '8993e80e7aa3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('policy_keep_direct', sa.Boolean(), server_default='TRUE', nullable=False))
+    op.add_column('posts', sa.Column('direct', sa.Boolean(), server_default='FALSE', nullable=False))
+
+
+def downgrade():
+    op.drop_column('posts', 'direct')
+    op.drop_column('accounts', 'policy_keep_direct')

migrations/versions/6d298e6406f2_add_some_probably_good_indexes.py

@@ -0,0 +1,34 @@
+"""add some probably good indexes (???)
+
+Revision ID: 6d298e6406f2
+Revises: 8fac6e10bdb3
+Create Date: 2017-08-14 20:27:49.103672
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6d298e6406f2'
+down_revision = '8fac6e10bdb3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index(op.f('ix_accounts_last_delete'), 'accounts', ['last_delete'], unique=False)
+    op.create_index(op.f('ix_accounts_last_fetch'), 'accounts', ['last_fetch'], unique=False)
+    op.create_index(op.f('ix_accounts_last_refresh'), 'accounts', ['last_refresh'], unique=False)
+    op.create_index(op.f('ix_oauth_tokens_account_id'), 'oauth_tokens', ['account_id'], unique=False)
+    op.create_index(op.f('ix_posts_author_id'), 'posts', ['author_id'], unique=False)
+    op.create_index(op.f('ix_sessions_account_id'), 'sessions', ['account_id'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_sessions_account_id'), table_name='sessions')
+    op.drop_index(op.f('ix_posts_author_id'), table_name='posts')
+    op.drop_index(op.f('ix_oauth_tokens_account_id'), table_name='oauth_tokens')
+    op.drop_index(op.f('ix_accounts_last_refresh'), table_name='accounts')
+    op.drop_index(op.f('ix_accounts_last_fetch'), table_name='accounts')
+    op.drop_index(op.f('ix_accounts_last_delete'), table_name='accounts')

migrations/versions/6fd1f5b43824_add_last_delete_back.py

@@ -0,0 +1,26 @@
+"""add last_delete back
+
+Revision ID: 6fd1f5b43824
+Revises: d97fa46b5560
+Create Date: 2017-08-29 17:22:00.747220
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6fd1f5b43824'
+down_revision = 'd97fa46b5560'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('last_delete', sa.DateTime(), nullable=True))
+    op.create_index(op.f('ix_accounts_last_delete'), 'accounts', ['last_delete'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_accounts_last_delete'), table_name='accounts')
+    op.drop_column('accounts', 'last_delete')

migrations/versions/740fe24a7712_add_misskey.py

@@ -0,0 +1,48 @@
+"""add misskey
+
+Revision ID: 740fe24a7712
+Revises: af763dccc0b4
+Create Date: 2021-11-10 00:13:37.344364
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '740fe24a7712'
+down_revision = 'af763dccc0b4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('misskey_instances',
+    sa.Column('instance', sa.String(), nullable=False),
+    sa.Column('popularity', sa.Float(), server_default='10', nullable=False),
+    sa.PrimaryKeyConstraint('instance', name=op.f('pk_misskey_instances'))
+    )
+    op.execute("""
+        INSERT INTO misskey_instances (instance, popularity) VALUES
+            ('misskey.io', 100),
+            ('cliq.social', 60),
+            ('misskey.dev', 50),
+            ('quietplace.xyz', 40),
+            ('mk.nixnet.social', 30),
+            ('jigglypuff.club', 20);
+    """)
+
+    op.create_table('misskey_apps',
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('instance', sa.String(), nullable=False),
+    sa.Column('client_secret', sa.String(), nullable=False),
+    sa.Column('protocol', sa.Enum('http', 'https', name='enum_protocol_misskey'), nullable=False),
+    sa.PrimaryKeyConstraint('instance', name=op.f('pk_misskey_apps'))
+    )
+
+
+def downgrade():
+    op.drop_table('misskey_instances')
+    op.drop_table('misskey_apps')
+    op.execute('DROP TYPE enum_protocol_misskey;')

migrations/versions/7afc7b343323_add_mastodon_apps.py

@@ -0,0 +1,32 @@
+"""add mastodon apps
+
+Revision ID: 7afc7b343323
+Revises: f63bf9e73bc9
+Create Date: 2017-08-18 20:36:00.104508
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7afc7b343323'
+down_revision = 'f63bf9e73bc9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('mastodon_app',
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+    sa.Column('instance', sa.String(), nullable=False),
+    sa.Column('client_id', sa.String(), nullable=False),
+    sa.Column('client_secret', sa.String(), nullable=False),
+    sa.Column('protocol', sa.Enum('http', 'https', name='enum_protocol'), nullable=False),
+    sa.PrimaryKeyConstraint('instance', name=op.f('pk_mastodon_app'))
+    )
+
+
+def downgrade():
+    op.drop_table('mastodon_app')

migrations/versions/7b0e9b8e0887_remove_fetch_batch_fk.py

@@ -0,0 +1,64 @@
+"""remove fetch batch fk
+
+things are real bad if the associated post is deleted and this is nulled
+keeping an opaque ID and associated date should work fine
+see GH-584
+
+Revision ID: 7b0e9b8e0887
+Revises: 740fe24a7712
+Create Date: 2022-02-27 11:48:55.107299
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7b0e9b8e0887'
+down_revision = '740fe24a7712'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', type_='foreignkey')
+    op.add_column('accounts', sa.Column('fetch_current_batch_end_date', sa.DateTime(timezone=True), nullable=True))
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_date = posts.created_at
+            FROM posts WHERE accounts.fetch_current_batch_end_id = posts.id;
+            ''')
+
+    # update ids from "mastodon:69420@chitter.xyz" format to just "69420"
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_id =
+            split_part(
+                split_part(fetch_current_batch_end_id, ':', 2),
+                    '@', 1);
+        ''')
+
+
+
+def downgrade():
+    # converts ids like "69420" back to "mastodon:69420@chitter.xyz"
+    # i sure hope there isn't a mastodon-compatible out there that can have
+    # : or @ in its post ids
+    op.execute('''
+        WITH accounts_exploded_ids AS (
+            SELECT
+                id,
+                split_part(id, ':', 1) || ':' AS service,
+                CASE WHEN position('@' IN id) != 0
+                    THEN '@' || split_part(id, @, 2)
+                    ELSE ''
+                END as instance
+            FROM accounts
+        )
+        UPDATE accounts SET fetch_current_batch_end_id = e.service || fetch_current_batch_end_id || e.instance
+        FROM accounts_exploded_ids AS e WHERE e.id = accounts.id AND fetch_current_batch_end_id IS NOT NULL;
+    ''')
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_id = NULL
+            WHERE NOT EXISTS (SELECT 1 FROM posts WHERE fetch_current_batch_end_id = posts.id);
+            ''')
+    op.create_foreign_key('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', 'posts', ['fetch_current_batch_end_id'], ['id'], ondelete='SET NULL')
+    op.drop_column('accounts', 'fetch_current_batch_end_date')

migrations/versions/7e255d4ea34d_add_is_reblog_to_post.py

@@ -0,0 +1,24 @@
+"""add is_reblog to Post
+
+Revision ID: 7e255d4ea34d
+Revises: 83510ef8c1a5
+Create Date: 2017-12-27 21:18:48.988601
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7e255d4ea34d'
+down_revision = '83510ef8c1a5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('posts', sa.Column('is_reblog', sa.Boolean(), server_default='FALSE', nullable=False))
+
+
+def downgrade():
+    op.drop_column('posts', 'is_reblog')

migrations/versions/83510ef8c1a5_add_favourite_reblog_count_to_posts.py

@@ -0,0 +1,26 @@
+"""add favourite, reblog count to posts
+
+Revision ID: 83510ef8c1a5
+Revises: c1f7444d0f75
+Create Date: 2017-12-27 20:40:31.576201
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '83510ef8c1a5'
+down_revision = 'c1f7444d0f75'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('posts', sa.Column('favourites', sa.Integer(), nullable=True))
+    op.add_column('posts', sa.Column('reblogs', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('posts', 'reblogs')
+    op.drop_column('posts', 'favourites')

migrations/versions/8993e80e7aa3_remove_post_body.py

@@ -0,0 +1,24 @@
+"""remove post body
+
+Revision ID: 8993e80e7aa3
+Revises: c80af843eed3
+Create Date: 2017-08-20 18:04:28.516129
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8993e80e7aa3'
+down_revision = 'c80af843eed3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column('posts', 'body')
+
+
+def downgrade():
+    op.add_column('posts', sa.Column('body', sa.VARCHAR(), autoincrement=False, nullable=True))

migrations/versions/8fac6e10bdb3_add_last_refresh_to_match_last_fetch_.py

@@ -0,0 +1,24 @@
+"""add last_refresh to match last_fetch and last_delete
+
+Revision ID: 8fac6e10bdb3
+Revises: 04da9abf37e2
+Create Date: 2017-08-12 22:55:33.004791
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8fac6e10bdb3'
+down_revision = '04da9abf37e2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('last_refresh', sa.DateTime(), server_default='epoch', nullable=True))
+
+
+def downgrade():
+    op.drop_column('accounts', 'last_refresh')

migrations/versions/90b5b84abc6a_change_timestamps_to_timestamptzs.py

@@ -0,0 +1,30 @@
+"""change timestamps to timestamptzs
+
+Revision ID: 90b5b84abc6a
+Revises: 6fd1f5b43824
+Create Date: 2017-08-31 16:46:16.785021
+
+"""
+from alembic import op
+from sqlalchemy.types import DateTime
+
+
+# revision identifiers, used by Alembic.
+revision = '90b5b84abc6a'
+down_revision = '6fd1f5b43824'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    for table in ('accounts', 'oauth_tokens', 'posts', 'sessions',
+                  'twitter_archives', 'mastodon_apps'):
+        for column in ('created_at', 'updated_at'):
+            op.alter_column(table, column, type_=DateTime(timezone=True))
+
+
+def downgrade():
+    for table in ('account', 'oauth_tokens', 'posts', 'sessions',
+                  'twitter_archives', 'mastodon_apps'):
+        for column in ('created_at', 'updated_at'):
+            op.alter_column(table, column, type_=DateTime(timezone=False))

migrations/versions/af763dccc0b4_remove_favourite_reblog_count_from_posts.py

@@ -0,0 +1,26 @@
+"""remove favourite, reblog count from posts
+
+Revision ID: af763dccc0b4
+Revises: 4b56cde3ebd7
+Create Date: 2021-05-14 19:45:37.429645
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'af763dccc0b4'
+down_revision = '4b56cde3ebd7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column('posts', 'reblogs')
+    op.drop_column('posts', 'favourites')
+
+
+def downgrade():
+    op.add_column('posts', sa.Column('favourites', sa.Integer(), nullable=True))
+    op.add_column('posts', sa.Column('reblogs', sa.Integer(), nullable=True))

migrations/versions/c136aa1157f9_add_backoff.py

@@ -0,0 +1,26 @@
+"""add backoff
+
+Revision ID: c136aa1157f9
+Revises: 2bd33abe291c
+Create Date: 2018-07-06 00:13:29.726250
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c136aa1157f9'
+down_revision = '2bd33abe291c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('backoff_level', sa.Integer(), server_default='0', nullable=False))
+    op.add_column('accounts', sa.Column('backoff_until', sa.DateTime(timezone=True), server_default='now', nullable=False))
+
+
+def downgrade():
+    op.drop_column('accounts', 'backoff_until')
+    op.drop_column('accounts', 'backoff_level')

migrations/versions/c1f7444d0f75_add_dormant_to_account.py

@@ -0,0 +1,24 @@
+"""add dormant to account
+
+Revision ID: c1f7444d0f75
+Revises: 3a0138499994
+Create Date: 2017-09-04 21:57:23.648580
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c1f7444d0f75'
+down_revision = '3a0138499994'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('dormant', sa.Boolean(), server_default='FALSE', nullable=False))
+
+
+def downgrade():
+    op.drop_column('accounts', 'dormant')

migrations/versions/c80af843eed3_make_token_secret_nullable.py

@@ -0,0 +1,28 @@
+"""make token secret nullable
+
+Revision ID: c80af843eed3
+Revises: fbdc10b29df9
+Create Date: 2017-08-18 21:25:17.933702
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c80af843eed3'
+down_revision = 'fbdc10b29df9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column('oauth_tokens', 'token_secret',
+               existing_type=sa.VARCHAR(),
+               nullable=True)
+
+
+def downgrade():
+    op.alter_column('oauth_tokens', 'token_secret',
+               existing_type=sa.VARCHAR(),
+               nullable=False)

migrations/versions/d97fa46b5560_add_csrf_tokens.py

@@ -0,0 +1,26 @@
+"""add csrf tokens
+
+Revision ID: d97fa46b5560
+Revises: f8a153bc809b
+Create Date: 2017-08-25 10:10:18.148120
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd97fa46b5560'
+down_revision = 'f8a153bc809b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('sessions', sa.Column('csrf_token', sa.String(), nullable=True))
+    op.execute('DELETE FROM sessions')
+    op.alter_column('sessions', 'csrf_token', nullable=False)
+
+
+def downgrade():
+    op.drop_column('sessions', 'csrf_token')

migrations/versions/e769c033e5c9_replace_last_delete_with_next_delete.py

@@ -0,0 +1,32 @@
+"""replace last_delete with next_delete
+
+Revision ID: e769c033e5c9
+Revises: 6d298e6406f2
+Create Date: 2017-08-14 20:51:02.248343
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e769c033e5c9'
+down_revision = '6d298e6406f2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('next_delete', sa.DateTime(), server_default='epoch', nullable=True))
+    op.execute('UPDATE accounts SET next_delete = last_delete + policy_delete_every;')
+    op.create_index(op.f('ix_accounts_next_delete'), 'accounts', ['next_delete'], unique=False)
+    op.drop_index('ix_accounts_last_delete', table_name='accounts')
+    op.drop_column('accounts', 'last_delete')
+
+
+def downgrade():
+    op.add_column('accounts', sa.Column('last_delete', postgresql.TIMESTAMP(), server_default=sa.text("'1970-01-01 00:00:00'::timestamp without time zone"), autoincrement=False, nullable=True))
+    op.execute('UPDATE accounts SET last__delete = next_delete - policy_delete_every;')
+    op.create_index('ix_accounts_last_delete', 'accounts', ['last_delete'], unique=False)
+    op.drop_index(op.f('ix_accounts_next_delete'), table_name='accounts')
+    op.drop_column('accounts', 'next_delete')

migrations/versions/f63bf9e73bc9_replace_index_on_posts_author_id_with_.py

@@ -0,0 +1,26 @@
+"""replace index on posts.author_id with composite index on author_id and created_at
+
+Revision ID: f63bf9e73bc9
+Revises: e769c033e5c9
+Create Date: 2017-08-15 23:55:46.945437
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f63bf9e73bc9'
+down_revision = 'e769c033e5c9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index('ix_posts_author_id_created_at', 'posts', ['author_id', 'created_at'], unique=False)
+    op.drop_index('ix_posts_author_id', table_name='posts')
+
+
+def downgrade():
+    op.create_index('ix_posts_author_id', 'posts', ['author_id'], unique=False)
+    op.drop_index('ix_posts_author_id_created_at', table_name='posts')

migrations/versions/f8a153bc809b_add_mastodon_instances.py

@@ -0,0 +1,40 @@
+"""add mastodon_instances
+
+Revision ID: f8a153bc809b
+Revises: 5fec5f5e8a5e
+Create Date: 2017-08-23 11:27:19.223721
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f8a153bc809b'
+down_revision = '5fec5f5e8a5e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('mastodon_instances',
+    sa.Column('instance', sa.String(), nullable=False),
+    sa.Column('popularity', sa.Float(), server_default='10', nullable=False),
+    sa.PrimaryKeyConstraint('instance', name=op.f('pk_mastodon_instances'))
+    )
+    op.execute("""
+        INSERT INTO mastodon_instances (instance, popularity) VALUES
+            ('mastodon.social', 100),
+            ('mastodon.cloud', 90),
+            ('social.tchncs.de', 80),
+            ('mastodon.xyz', 70),
+            ('mstdn.io', 60),
+            ('awoo.space', 50),
+            ('cybre.space', 40),
+            ('mastodon.art', 30)
+            ;
+    """)
+
+
+def downgrade():
+    op.drop_table('mastodon_instances')

migrations/versions/f95af1a8d89f_change_more_timestamps_to_timestamptzs.py

@@ -0,0 +1,26 @@
+"""change more timestamps to timestamptzs
+
+Revision ID: f95af1a8d89f
+Revises: 90b5b84abc6a
+Create Date: 2017-08-31 17:00:20.538070
+
+"""
+from alembic import op
+from sqlalchemy.types import DateTime
+
+
+# revision identifiers, used by Alembic.
+revision = 'f95af1a8d89f'
+down_revision = '90b5b84abc6a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    for column in ('last_fetch', 'last_refresh', 'last_delete', 'next_delete'):
+        op.alter_column('accounts', column, type_=DateTime(timezone=True))
+
+
+def downgrade():
+    for column in ('last_fetch', 'last_refresh', 'last_delete', 'next_delete'):
+        op.alter_column('accounts', column, type_=DateTime(timezone=False))

migrations/versions/fbdc10b29df9_it_s_supposed_to_be_plural_dummy.py

@@ -0,0 +1,24 @@
+"""it's supposed to be plural, dummy
+
+Revision ID: fbdc10b29df9
+Revises: 7afc7b343323
+Create Date: 2017-08-18 20:39:39.119165
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'fbdc10b29df9'
+down_revision = '7afc7b343323'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute('ALTER TABLE mastodon_app RENAME TO mastodon_apps')
+
+
+def downgrade():
+    op.execute('ALTER TABLE mastodon_apps RENAME TO mastodon_app')

model.py

@@ -1,18 +1,23 @@
-from datetime import datetime
+from datetime import timedelta, datetime, timezone
 
 from app import db
-
-from twitter import Twitter, OAuth
 import secrets
-from lib import decompose_interval
-from datetime import timedelta
+from libforget.interval import decompose_interval
+import random
+from sqlalchemy.ext.declarative import declared_attr
+
 
 class TimestampMixin(object):
-    created_at = db.Column(db.DateTime, server_default=db.func.now(), nullable=False)
-    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now(), nullable=False)
+    created_at = db.Column(
+            db.DateTime(timezone=True), server_default=db.func.now(),
+            nullable=False)
+    updated_at = db.Column(
+            db.DateTime(timezone=True), server_default=db.func.now(),
+            onupdate=db.func.now(), nullable=False)
 
     def touch(self):
-        self.updated_at=db.func.now()
+        self.updated_at = db.func.now()
+
 
 class RemoteIDMixin(object):
     @property
@@ -26,12 +31,83 @@ class RemoteIDMixin(object):
         if not self.id:
             return None
         if self.service != "twitter":
-            raise Exception("tried to get twitter id for a {} {}".format(self.service, type(self)))
+            raise Exception(
+                    "tried to get twitter id for a {} {}"
+                    .format(self.service, type(self)))
         return self.id.split(":")[1]
 
     @twitter_id.setter
-    def twitter_id(self, id):
-        self.id = "twitter:{}".format(id)
+    def twitter_id(self, id_):
+        self.id = "twitter:{}".format(id_)
+
+    @property
+    def mastodon_instance(self):
+        if not self.id:
+            return None
+        if self.service != "mastodon":
+            raise Exception(
+                    "tried to get mastodon instance for a {} {}"
+                    .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[1]
+
+    @mastodon_instance.setter
+    def mastodon_instance(self, instance):
+        self.id = "mastodon:{}@{}".format(self.mastodon_id, instance)
+
+    @property
+    def mastodon_id(self):
+        if not self.id:
+            return None
+        if self.service != "mastodon":
+            raise Exception(
+                    "tried to get mastodon id for a {} {}"
+                    .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[0]
+
+    @mastodon_id.setter
+    def mastodon_id(self, id_):
+        self.id = "mastodon:{}@{}".format(id_, self.mastodon_instance)
+        
+    @property
+    def misskey_instance(self):
+        if not self.id:
+            return None
+        if self.service != "misskey":
+            raise Exception(
+                "tried to get misskey instance for a {} {}"
+                .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[1]
+        
+    @misskey_instance.setter
+    def misskey_instance(self, instance):
+        self.id = "misskey:{}@{}".format(self.misskey_id, instance)
+        
+    @property
+    def misskey_id(self):
+        if not self.id:
+            return None
+        if self.service != "misskey":
+            raise Exception(
+                    "tried to get misskey id for a {} {}"
+                    .format(self.service, type(self)))
+        return self.id.split(":", 1)[1].split('@')[0]
+
+    @misskey_id.setter
+    def misskey_id(self, id_):
+        self.id = "misskey:{}@{}".format(id_, self.misskey_instance)
+
+    @property
+    def remote_id(self):
+        if self.service == 'twitter':
+            return self.twitter_id
+        elif self.service == 'mastodon':
+            return self.mastodon_id
+        elif self.service == 'misskey':
+            return self.misskey_id
+
+
+ThreeWayPolicyEnum = db.Enum('keeponly', 'deleteonly', 'none',
+        name='enum_3way_policy')
 
 
 @decompose_interval('policy_delete_every')
@@ -40,128 +116,241 @@ class Account(TimestampMixin, RemoteIDMixin):
     __tablename__ = 'accounts'
     id = db.Column(db.String, primary_key=True)
 
-    policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
-    policy_keep_latest = db.Column(db.Integer, server_default='100', nullable=False)
-    policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False)
-    policy_keep_media = db.Column(db.Boolean, server_default='FALSE', nullable=False)
-    policy_delete_every = db.Column(db.Interval, server_default='30 minutes', nullable=False)
-    policy_keep_younger = db.Column(db.Interval, server_default='365 days', nullable=False)
+    policy_enabled = db.Column(db.Boolean, server_default='FALSE',
+                               nullable=False)
+    policy_keep_latest = db.Column(db.Integer, server_default='100',
+                                   nullable=False)
+    policy_keep_favourites = db.Column(ThreeWayPolicyEnum,
+                                       server_default='none', nullable=False)
+    policy_keep_media = db.Column(ThreeWayPolicyEnum, server_default='none',
+                                  nullable=False)
+    policy_delete_every = db.Column(db.Interval, server_default='30 minutes',
+                                    nullable=False)
+    policy_keep_younger = db.Column(db.Interval, server_default='365 days',
+                                    nullable=False)
+    policy_keep_direct = db.Column(db.Boolean, server_default='TRUE',
+                                   nullable=False)
 
     display_name = db.Column(db.String)
     screen_name = db.Column(db.String)
     avatar_url = db.Column(db.String)
     reported_post_count = db.Column(db.Integer)
 
-    last_fetch = db.Column(db.DateTime, server_default='epoch')
-    last_delete = db.Column(db.DateTime, server_default='epoch')
+    last_fetch = db.Column(db.DateTime(timezone=True),
+                           server_default='epoch', index=True)
+    last_refresh = db.Column(db.DateTime(timezone=True),
+                             server_default='epoch', index=True)
+    last_delete = db.Column(db.DateTime(timezone=True), index=True)
+    next_delete = db.Column(db.DateTime(timezone=True), index=True)
+
+    fetch_history_complete = db.Column(db.Boolean, server_default='FALSE',
+                                       nullable=False)
+
+    fetch_current_batch_end_id = db.Column(db.String)
+    fetch_current_batch_end_date = db.Column(db.DateTime(timezone=True))
+
+    reason = db.Column(db.String)
+    dormant = db.Column(db.Boolean, server_default='FALSE', nullable=False)
+
+    backoff_level = db.Column(db.Integer, server_default='0', nullable=False)
+    backoff_until = db.Column(db.DateTime(timezone=True), server_default='now', nullable=False)
+    BACKOFF_MAX = 14
+    # backoff is 10 seconds * 2^backoff_level
+    # this gives us roughly 1.8 days at level 14
 
     def touch_fetch(self):
         self.last_fetch = db.func.now()
 
     def touch_delete(self):
         self.last_delete = db.func.now()
+        # if it's been more than 1 delete cycle ago that we've deleted a post,
+        # reset next_delete to be 1 cycle away
+        if (datetime.now(timezone.utc) - self.next_delete
+           > self.policy_delete_every):
+            self.next_delete = db.func.now() + self.policy_delete_every
+        else:
+            self.next_delete += self.policy_delete_every
+
+    def touch_refresh(self):
+        self.last_refresh = db.func.now()
+
+    def get_avatar(self):
+        from app import imgproxy
+        from flask import url_for
+        return url_for('avatar', identifier=imgproxy.identifier_for(self.avatar_url))
 
     @db.validates('policy_keep_younger', 'policy_delete_every')
     def validate_intervals(self, key, value):
         if not (value == timedelta(0) or value >= timedelta(minutes=1)):
             value = timedelta(minutes=1)
+        if key == 'policy_delete_every' and \
+           self.next_delete and\
+           datetime.now(timezone.utc) + value < self.next_delete:
+            # make sure that next delete is not in the far future
+            self.next_delete = datetime.now(timezone.utc) + value
         return value
 
+    # pylint: disable=R0201
     @db.validates('policy_keep_latest')
     def validate_empty_string_is_zero(self, key, value):
-        if type(value) == str and value.strip() == '':
+        if isinstance(value, str) and value.strip() == '':
             return 0
         return value
 
+    @db.validates('policy_enabled')
+    def on_policy_enable(self, key, enable):
+        if not self.policy_enabled and enable:
+            self.next_delete = (
+                datetime.now(timezone.utc) + self.policy_delete_every)
+            self.reason = None
+            self.dormant = False
+        return enable
+
+    @db.validates('policy_keep_direct')
+    def validate_bool_accept_string(self, key, value):
+        if isinstance(value, str):
+            return value.lower() == 'true'
+        return value
+
 
     # backref: tokens
     # backref: twitter_archives
     # backref: posts
     # backref: sessions
 
-    def __repr__(self):
-        return f"<Account({self.id}, {self.screen_name}, {self.display_name})>"
-
     def post_count(self):
-        return Post.query.with_parent(self).count()
+        return Post.query.with_parent(self, 'posts').count()
 
     def estimate_eligible_for_delete(self):
         """
-        this is an estimation because we do not know if favourite status has changed since last time a post was refreshed
-        and it is unfeasible to refresh every single post every time we need to know how many posts are eligible to delete
+        this is an estimation because we do not know if favourite status has
+        changed since last time a post was refreshed and it is unfeasible to
+        refresh every single post every time we need to know how many posts are
+        eligible to delete
         """
-        latest_n_posts = Post.query.with_parent(self).order_by(db.desc(Post.created_at)).limit(self.policy_keep_latest)
-        query = Post.query.with_parent(self).\
-            filter(Post.created_at + self.policy_keep_younger <= db.func.now()).\
-            except_(latest_n_posts)
-        if(self.policy_keep_favourites):
-            query = query.filter_by(favourite = False)
-        if(self.policy_keep_media):
-            query = query.filter_by(has_media = False)
+        latest_n_posts = (Post.query.with_parent(self, 'posts')
+                          .order_by(db.desc(Post.created_at))
+                          .limit(self.policy_keep_latest))
+        query = (Post.query.with_parent(self, 'posts')
+                 .filter(Post.created_at <=
+                         db.func.now() - self.policy_keep_younger)
+                 .except_(latest_n_posts))
+        if(self.policy_keep_favourites != 'none'):
+            query = query.filter(db.or_(
+                Post.favourite == (self.policy_keep_favourites == 'deleteonly'),
+                Post.is_reblog))
+        if(self.policy_keep_media != 'none'):
+            query = query.filter(db.or_(
+                Post.has_media == (self.policy_keep_media == 'deleteonly'),
+                Post.is_reblog))
+        if(self.policy_keep_direct):
+            query = query.filter(~Post.direct)
         return query.count()
 
-
     def force_log_out(self):
         Session.query.with_parent(self).delete()
         db.session.commit()
 
+    def backoff(self):
+        self.backoff_level = min(self.backoff_level + 1, self.BACKOFF_MAX)
+        backoff_for = 10 * 2 ** self.backoff_level
+        backoff_for *= random.uniform(1, 1.3)
+        self.backoff_until = datetime.utcnow() + timedelta(seconds=backoff_for)
+
+    def reset_backoff(self):
+        self.backoff_until = datetime.utcnow()
+        self.backoff_level = 0
+
 
 class Account(Account, db.Model):
-    pass
+    def __str__(self):
+        return f"<Account({self.id}, {self.screen_name}, {self.display_name})>"
 
 
 class OAuthToken(db.Model, TimestampMixin):
     __tablename__ = 'oauth_tokens'
 
     token = db.Column(db.String, primary_key=True)
-    token_secret = db.Column(db.String, nullable=False)
+    token_secret = db.Column(db.String, nullable=True)
 
-    account_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=True)
-    account = db.relationship(Account, backref=db.backref('tokens', order_by=lambda: db.desc(OAuthToken.created_at)))
+    account_id = db.Column(db.String,
+                           db.ForeignKey('accounts.id', ondelete='CASCADE',
+                                         onupdate='CASCADE'),
+                           nullable=True, index=True)
+    account = db.relationship(
+            Account,
+            backref=db.backref('tokens',
+                               order_by=lambda: db.desc(OAuthToken.created_at))
+            )
 
-    # note: account_id is nullable here because we don't know what account a token is for
-    # until we call /account/verify_credentials with it
+    # note: account_id is nullable here because we don't know what account a
+    # token is for until we call /account/verify_credentials with it
 
 
 class Session(db.Model, TimestampMixin):
     __tablename__ = 'sessions'
 
-    id = db.Column(db.String, primary_key=True, default=lambda: secrets.token_urlsafe())
+    id = db.Column(db.String, primary_key=True,
+                   default=secrets.token_urlsafe)
 
-    account_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+    account_id = db.Column(
+            db.String,
+            db.ForeignKey('accounts.id',
+                          ondelete='CASCADE', onupdate='CASCADE'),
+            nullable=False, index=True)
     account = db.relationship(Account, lazy='joined', backref='sessions')
 
+    csrf_token = db.Column(db.String,
+                           default=secrets.token_urlsafe,
+                           nullable=False)
+
 
 class Post(db.Model, TimestampMixin, RemoteIDMixin):
     __tablename__ = 'posts'
 
     id = db.Column(db.String, primary_key=True)
-    body = db.Column(db.String)
 
-    author_id = db.Column(db.String, db.ForeignKey('accounts.id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
-    author = db.relationship(Account,
-            backref=db.backref('posts', order_by=lambda: db.desc(Post.created_at)))
+    author_id = db.Column(
+            db.String,
+            db.ForeignKey('accounts.id',
+                          ondelete='CASCADE', onupdate='CASCADE'),
+            nullable=False)
+    author = db.relationship(
+            Account,
+            foreign_keys = (author_id,),
+            backref=db.backref('posts',
+                               order_by=lambda: db.desc(Post.created_at)))
 
     favourite = db.Column(db.Boolean, server_default='FALSE', nullable=False)
     has_media = db.Column(db.Boolean, server_default='FALSE', nullable=False)
+    direct = db.Column(db.Boolean, server_default='FALSE', nullable=False)
 
-    def snippet(self):
-        if len(self.body) > 20:
-            return self.body[:19] + "✂"
-        return self.body
+    is_reblog = db.Column(db.Boolean, server_default='FALSE', nullable=False)
+
+    def __str__(self):
+        return '<Post ({}, Author: {})>'.format(self.id, self.author_id)
+
+
+db.Index('ix_posts_author_id_created_at', Post.author_id, Post.created_at)
 
-    def __repr__(self):
-        return '<Post ({}, "{}", Author: {})>'.format(self.id, self.snippet(), self.author_id)
 
 class TwitterArchive(db.Model, TimestampMixin):
     __tablename__ = 'twitter_archives'
 
     id = db.Column(db.Integer, primary_key=True)
-    account_id = db.Column(db.String, db.ForeignKey('accounts.id', onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
-    account = db.relationship(Account, backref=db.backref('twitter_archives', order_by=lambda: db.desc(TwitterArchive.id)))
+    account_id = db.Column(
+            db.String,
+            db.ForeignKey('accounts.id',
+                          onupdate='CASCADE', ondelete='CASCADE'),
+            nullable=False)
+    account = db.relationship(
+            Account,
+            backref=db.backref('twitter_archives',
+                               order_by=lambda: db.desc(TwitterArchive.id)))
     body = db.deferred(db.Column(db.LargeBinary, nullable=False))
     chunks = db.Column(db.Integer)
-    chunks_successful = db.Column(db.Integer, server_default='0', nullable=False)
+    chunks_successful = db.Column(db.Integer,
+                                  server_default='0', nullable=False)
     chunks_failed = db.Column(db.Integer, server_default='0', nullable=False)
 
     def status(self):
@@ -170,3 +359,52 @@ class TwitterArchive(db.Model, TimestampMixin):
         if self.chunks_successful == self.chunks:
             return 'successful'
         return 'pending'
+
+
+ProtoEnum = db.Enum('http', 'https', name='enum_protocol')
+
+
+class MastodonApp(db.Model, TimestampMixin):
+    __tablename__ = 'mastodon_apps'
+
+    instance = db.Column(db.String, primary_key=True)
+    client_id = db.Column(db.String, nullable=False)
+    client_secret = db.Column(db.String, nullable=False)
+    protocol = db.Column(ProtoEnum, nullable=False)
+
+
+class MastodonInstance(db.Model):
+    """
+    this is for the autocomplete in the mastodon login form
+
+    it isn't coupled with anything else so that we can seed it with
+    some popular instances ahead of time
+    """
+    __tablename__ = 'mastodon_instances'
+
+    instance = db.Column(db.String, primary_key=True)
+    popularity = db.Column(db.Float, server_default='10', nullable=False)
+
+    def bump(self, value=1):
+        self.popularity = (self.popularity or 10) + value
+
+class MisskeyApp(db.Model, TimestampMixin):
+    __tablename__ = 'misskey_apps'
+
+    instance = db.Column(db.String, primary_key=True)
+    protocol = db.Column(db.String, nullable=False)
+    client_secret = db.Column(db.String, nullable=False)
+
+class MisskeyInstance(db.Model):
+    """
+    this is for the autocomplete in the misskey login form
+    it isn't coupled with anything else so that we can seed it with
+    some popular instances ahead of time
+    """
+    __tablename__ = 'misskey_instances'
+
+    instance = db.Column(db.String, primary_key=True)
+    popularity = db.Column(db.Float, server_default='10', nullable=False)
+
+    def bump(self, value=1):
+        self.popularity = (self.popularity or 10) + value

package-lock.json

@@ -0,0 +1,263 @@
+{
+  "name": "forget",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "devDependencies": {
+        "rollup": "^2.42.4",
+        "rollup-plugin-node-resolve": "^5.2.0",
+        "rollup-plugin-svelte": "^7.1.0",
+        "svelte": "^3.35.0"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "12.0.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
+      "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==",
+      "dev": true
+    },
+    "node_modules/@types/resolve": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-0.0.8.tgz",
+      "integrity": "sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/builtin-modules": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz",
+      "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/estree-walker": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz",
+      "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==",
+      "dev": true
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/is-module": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz",
+      "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=",
+      "dev": true
+    },
+    "node_modules/path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "dev": true
+    },
+    "node_modules/require-relative": {
+      "version": "0.8.7",
+      "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz",
+      "integrity": "sha1-eZlTn8ngR6N5KPoZb44VY9q9Nt4=",
+      "dev": true
+    },
+    "node_modules/resolve": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
+      "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==",
+      "dev": true,
+      "dependencies": {
+        "path-parse": "^1.0.6"
+      }
+    },
+    "node_modules/rollup": {
+      "version": "2.68.0",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.68.0.tgz",
+      "integrity": "sha512-XrMKOYK7oQcTio4wyTz466mucnd8LzkiZLozZ4Rz0zQD+HeX4nUK4B8GrTX/2EvN2/vBF/i2WnaXboPxo0JylA==",
+      "dev": true,
+      "bin": {
+        "rollup": "dist/bin/rollup"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
+      }
+    },
+    "node_modules/rollup-plugin-node-resolve": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-5.2.0.tgz",
+      "integrity": "sha512-jUlyaDXts7TW2CqQ4GaO5VJ4PwwaV8VUGA7+km3n6k6xtOEacf61u0VXwN80phY/evMcaS+9eIeJ9MOyDxt5Zw==",
+      "deprecated": "This package has been deprecated and is no longer maintained. Please use @rollup/plugin-node-resolve.",
+      "dev": true,
+      "dependencies": {
+        "@types/resolve": "0.0.8",
+        "builtin-modules": "^3.1.0",
+        "is-module": "^1.0.0",
+        "resolve": "^1.11.1",
+        "rollup-pluginutils": "^2.8.1"
+      },
+      "peerDependencies": {
+        "rollup": ">=1.11.0"
+      }
+    },
+    "node_modules/rollup-plugin-svelte": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-svelte/-/rollup-plugin-svelte-7.1.0.tgz",
+      "integrity": "sha512-vopCUq3G+25sKjwF5VilIbiY6KCuMNHP1PFvx2Vr3REBNMDllKHFZN2B9jwwC+MqNc3UPKkjXnceLPEjTjXGXg==",
+      "dev": true,
+      "dependencies": {
+        "require-relative": "^0.8.7",
+        "rollup-pluginutils": "^2.8.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "rollup": ">=2.0.0",
+        "svelte": ">=3.5.0"
+      }
+    },
+    "node_modules/rollup-pluginutils": {
+      "version": "2.8.2",
+      "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz",
+      "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==",
+      "dev": true,
+      "dependencies": {
+        "estree-walker": "^0.6.1"
+      }
+    },
+    "node_modules/svelte": {
+      "version": "3.46.4",
+      "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.46.4.tgz",
+      "integrity": "sha512-qKJzw6DpA33CIa+C/rGp4AUdSfii0DOTCzj/2YpSKKayw5WGSS624Et9L1nU1k2OVRS9vaENQXp2CVZNU+xvIg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    }
+  },
+  "dependencies": {
+    "@types/node": {
+      "version": "12.0.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
+      "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==",
+      "dev": true
+    },
+    "@types/resolve": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-0.0.8.tgz",
+      "integrity": "sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "builtin-modules": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz",
+      "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==",
+      "dev": true
+    },
+    "estree-walker": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz",
+      "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==",
+      "dev": true
+    },
+    "fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "optional": true
+    },
+    "is-module": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz",
+      "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=",
+      "dev": true
+    },
+    "path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "dev": true
+    },
+    "require-relative": {
+      "version": "0.8.7",
+      "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz",
+      "integrity": "sha1-eZlTn8ngR6N5KPoZb44VY9q9Nt4=",
+      "dev": true
+    },
+    "resolve": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
+      "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==",
+      "dev": true,
+      "requires": {
+        "path-parse": "^1.0.6"
+      }
+    },
+    "rollup": {
+      "version": "2.68.0",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.68.0.tgz",
+      "integrity": "sha512-XrMKOYK7oQcTio4wyTz466mucnd8LzkiZLozZ4Rz0zQD+HeX4nUK4B8GrTX/2EvN2/vBF/i2WnaXboPxo0JylA==",
+      "dev": true,
+      "requires": {
+        "fsevents": "~2.3.2"
+      }
+    },
+    "rollup-plugin-node-resolve": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-5.2.0.tgz",
+      "integrity": "sha512-jUlyaDXts7TW2CqQ4GaO5VJ4PwwaV8VUGA7+km3n6k6xtOEacf61u0VXwN80phY/evMcaS+9eIeJ9MOyDxt5Zw==",
+      "dev": true,
+      "requires": {
+        "@types/resolve": "0.0.8",
+        "builtin-modules": "^3.1.0",
+        "is-module": "^1.0.0",
+        "resolve": "^1.11.1",
+        "rollup-pluginutils": "^2.8.1"
+      }
+    },
+    "rollup-plugin-svelte": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/rollup-plugin-svelte/-/rollup-plugin-svelte-7.1.0.tgz",
+      "integrity": "sha512-vopCUq3G+25sKjwF5VilIbiY6KCuMNHP1PFvx2Vr3REBNMDllKHFZN2B9jwwC+MqNc3UPKkjXnceLPEjTjXGXg==",
+      "dev": true,
+      "requires": {
+        "require-relative": "^0.8.7",
+        "rollup-pluginutils": "^2.8.2"
+      }
+    },
+    "rollup-pluginutils": {
+      "version": "2.8.2",
+      "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz",
+      "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==",
+      "dev": true,
+      "requires": {
+        "estree-walker": "^0.6.1"
+      }
+    },
+    "svelte": {
+      "version": "3.46.4",
+      "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.46.4.tgz",
+      "integrity": "sha512-qKJzw6DpA33CIa+C/rGp4AUdSfii0DOTCzj/2YpSKKayw5WGSS624Et9L1nU1k2OVRS9vaENQXp2CVZNU+xvIg==",
+      "dev": true
+    }
+  }
+}

package.json

@@ -0,0 +1,8 @@
+{
+  "devDependencies": {
+    "rollup": "^2.42.4",
+    "rollup-plugin-node-resolve": "^5.2.0",
+    "rollup-plugin-svelte": "^7.1.0",
+    "svelte": "^3.35.0"
+  }
+}

requirements-dev.txt

@@ -0,0 +1,73 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements --dev
+#
+
+# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
+# requirements. To emit only development requirements, pass "--dev-only".
+
+-i https://pypi.python.org/simple
+alembic==1.7.6
+amqp==5.0.9; python_version >= '3.6'
+attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+billiard==3.6.4.0
+blinker==1.4
+blurhash==1.1.4
+brotli==1.0.9
+celery==5.2.3
+certifi==2021.10.8
+charset-normalizer==2.0.12; python_version >= '3'
+click-didyoumean==0.3.0; python_version < '4' and python_full_version >= '3.6.2'
+click-plugins==1.1.1
+click-repl==0.2.0
+click==8.0.4; python_version >= '3.6'
+cloudpickle==2.0.0; python_version >= '3.6'
+codecov==2.1.12
+coverage==6.3.2
+csscompressor==0.9.5
+decorator==5.1.1; python_version >= '3.5'
+deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+doit==0.34.2
+flask-migrate==3.1.0
+flask-sqlalchemy==2.5.1
+flask==2.0.3
+greenlet==1.1.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+gunicorn==20.1.0
+honcho==1.1.0
+idna==3.3; python_version >= '3'
+iniconfig==1.1.1
+itsdangerous==2.1.0; python_version >= '3.7'
+jinja2==3.0.3; python_version >= '3.6'
+kombu==5.2.3; python_version >= '3.7'
+mako==1.1.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+markupsafe==2.1.0; python_version >= '3.7'
+mastodon.py==1.5.1
+packaging==21.3; python_version >= '3.6'
+pillow==9.0.1
+pluggy==1.0.0; python_version >= '3.6'
+prompt-toolkit==3.0.28; python_full_version >= '3.6.2'
+psycopg2==2.9.3
+py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pyinotify==0.9.6; sys_platform == 'linux'
+pyparsing==3.0.7; python_version >= '3.6'
+pytest-cov==3.0.0
+pytest==7.0.1
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-magic==0.4.25; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pytz==2021.3
+raven==6.10.0
+redis==4.1.4
+requests==2.27.1
+setuptools==59.6.0; python_version >= '3.6'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sqlalchemy==1.4.31
+tomli==2.0.1; python_version >= '3.7'
+twitter==1.19.3
+urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+versioneer==0.21
+vine==5.0.0; python_version >= '3.6'
+wcwidth==0.2.5
+werkzeug==2.0.3; python_version >= '3.6'
+wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

requirements.txt

@@ -1,36 +1,60 @@
-alembic==0.9.5
-amqp==2.2.1
-billiard==3.5.0.3
-Brotli==0.6.0
-celery==4.1.0
-click==6.7
-cloudpickle==0.4.0
-contextlib2==0.5.5
-csscompressor==0.9.4
-doit==0.30.3
-Flask==0.12.2
-Flask-Limiter==0.9.5
-Flask-Migrate==2.1.0
-Flask-SQLAlchemy==2.2
-gunicorn==19.7.1
-honcho==1.0.1
-itsdangerous==0.24
-Jinja2==2.9.6
-kombu==4.1.0
-limits==1.2.1
-Mako==1.0.7
-MarkupSafe==1.0
-olefile==0.44
-Pillow==4.2.1
-psycopg2==2.7.3
-pyinotify==0.9.6
-python-dateutil==2.6.1
-python-editor==1.0.3
-pytz==2017.2
-raven==6.1.0
-redis==2.10.5
-six==1.10.0
-SQLAlchemy==1.1.13
-twitter==1.17.1
-vine==1.1.4
-Werkzeug==0.12.2
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+-i https://pypi.python.org/simple
+alembic==1.7.6
+amqp==5.0.9; python_version >= '3.6'
+billiard==3.6.4.0
+blinker==1.4
+blurhash==1.1.4
+brotli==1.0.9
+celery==5.2.3
+certifi==2021.10.8
+charset-normalizer==2.0.12; python_version >= '3'
+click-didyoumean==0.3.0; python_version < '4' and python_full_version >= '3.6.2'
+click-plugins==1.1.1
+click-repl==0.2.0
+click==8.0.4; python_version >= '3.6'
+cloudpickle==2.0.0; python_version >= '3.6'
+csscompressor==0.9.5
+decorator==5.1.1; python_version >= '3.5'
+deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+doit==0.34.2
+flask-migrate==3.1.0
+flask-sqlalchemy==2.5.1
+flask==2.0.3
+greenlet==1.1.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+gunicorn==20.1.0
+honcho==1.1.0
+idna==3.3; python_version >= '3'
+itsdangerous==2.1.0; python_version >= '3.7'
+jinja2==3.0.3; python_version >= '3.6'
+kombu==5.2.3; python_version >= '3.7'
+mako==1.1.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+markupsafe==2.1.0; python_version >= '3.7'
+mastodon.py==1.5.1
+packaging==21.3; python_version >= '3.6'
+pillow==9.0.1
+prompt-toolkit==3.0.28; python_full_version >= '3.6.2'
+psycopg2==2.9.3
+pyinotify==0.9.6; sys_platform == 'linux'
+pyparsing==3.0.7; python_version >= '3.6'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-magic==0.4.25; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pytz==2021.3
+raven==6.10.0
+redis==4.1.4
+requests==2.27.1
+setuptools==59.6.0; python_version >= '3.6'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sqlalchemy==1.4.31
+twitter==1.19.3
+urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+vine==5.0.0; python_version >= '3.6'
+wcwidth==0.2.5
+werkzeug==2.0.3; python_version >= '3.6'
+wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

rollup.config.js

@@ -0,0 +1,17 @@
+import svelte from 'rollup-plugin-svelte';
+import node_resolve from 'rollup-plugin-node-resolve';
+
+export default {
+    output: {
+        format: 'iife',
+    },
+    plugins: [
+        svelte({
+            extensions: ['.html'],
+            include: 'components/**/*.html',
+            compilerOptions: {hydratable: true},
+            emitCss: false,
+        }),
+        node_resolve(),
+    ]
+}

routes.py

@@ -1,182 +0,0 @@
-from flask import render_template, url_for, redirect, request, g, Response, jsonify
-from datetime import datetime, timedelta
-import lib.twitter
-import lib
-from lib.auth import require_auth, require_auth_api
-from lib import set_session_cookie
-from lib import get_viewer_session, get_viewer
-from model import Account, Session, Post, TwitterArchive
-from app import app, db, sentry, limiter
-import tasks
-from zipfile import BadZipFile
-from twitter import TwitterError
-from urllib.error import URLError
-import version
-import lib.brotli
-import lib.settings
-
-@app.before_request
-def load_viewer():
-    g.viewer = get_viewer_session()
-    if g.viewer and sentry:
-        sentry.user_context({
-                'id': g.viewer.account.id,
-                'username': g.viewer.account.screen_name,
-                'service': g.viewer.account.service
-            })
-
-@app.context_processor
-def inject_version():
-    return dict(version=version.version)
-
-@app.context_processor
-def inject_sentry():
-    if sentry:
-        client_dsn = app.config.get('SENTRY_DSN').split('@')
-        client_dsn[:1] = client_dsn[0].split(':')
-        client_dsn = ':'.join(client_dsn[0:2]) + '@' + client_dsn[3]
-        return dict(sentry_dsn=client_dsn)
-    return dict()
-
-@app.after_request
-def touch_viewer(resp):
-    if 'viewer' in g and g.viewer:
-        set_session_cookie(g.viewer, resp, app.config.get('HTTPS'))
-        g.viewer.touch()
-        db.session.commit()
-    return resp
-
-
-lib.brotli.brotli(app)
-
-@app.route('/')
-def index():
-    if g.viewer:
-        return render_template('logged_in.html', scales=lib.interval_scales,
-                tweet_archive_failed = 'tweet_archive_failed' in request.args,
-                settings_error = 'settings_error' in request.args
-                )
-    else:
-        return render_template('index.html',
-                twitter_login_error = 'twitter_login_error' in request.args)
-
-@app.route('/login/twitter')
-@limiter.limit('3/minute')
-def twitter_login_step1():
-    try:
-        return redirect(lib.twitter.get_login_url(
-            callback = url_for('twitter_login_step2', _external=True),
-            **app.config.get_namespace("TWITTER_")
-            ))
-    except (TwitterError, URLError):
-        return redirect(url_for('index', twitter_login_error='', _anchor='log_in'))
-
-@app.route('/login/twitter/callback')
-@limiter.limit('3/minute')
-def twitter_login_step2():
-    try:
-        oauth_token = request.args['oauth_token']
-        oauth_verifier = request.args['oauth_verifier']
-        token = lib.twitter.receive_verifier(oauth_token, oauth_verifier, **app.config.get_namespace("TWITTER_"))
-
-        session = Session(account_id = token.account_id)
-        db.session.add(session)
-        db.session.commit()
-
-        tasks.fetch_acc.s(token.account_id).apply_async(routing_key='high')
-
-        resp = Response(status=302, headers={"location": url_for('index')})
-        set_session_cookie(session, resp, app.config.get('HTTPS'))
-        return resp
-    except (TwitterError, URLError):
-        return redirect(url_for('index', twitter_login_error='', _anchor='log_in'))
-
-@app.route('/upload_tweet_archive', methods=('POST',))
-@limiter.limit('10/10 minutes')
-@require_auth
-def upload_tweet_archive():
-    ta = TwitterArchive(account = g.viewer.account,
-            body = request.files['file'].read())
-    db.session.add(ta)
-    db.session.commit()
-
-    try:
-        tasks.chunk_twitter_archive(ta.id)
-
-        assert ta.chunks > 0
-
-        return redirect(url_for('index', _anchor='recent_archives'))
-    except (BadZipFile, AssertionError):
-        return redirect(url_for('index', tweet_archive_failed='', _anchor='tweet_archive_import'))
-
-@app.route('/settings', methods=('POST',))
-@require_auth
-def settings():
-    for attr in lib.settings.attrs:
-        try:
-            if attr in request.form:
-                setattr(g.viewer.account, attr, request.form[attr])
-        except ValueError:
-            return redirect(url_for('index', settings_error=''))
-
-    db.session.commit()
-
-    return redirect(url_for('index', settings_saved=''))
-
-@app.route('/disable', methods=('POST',))
-@require_auth
-def disable():
-    g.viewer.account.policy_enabled = False
-    db.session.commit()
-
-    return redirect(url_for('index'))
-
-@app.route('/enable', methods=('POST',))
-@require_auth
-def enable():
-
-    risky = False
-    if not 'confirm' in request.form and not g.viewer.account.policy_enabled:
-        if g.viewer.account.policy_delete_every == timedelta(0):
-            approx = g.viewer.account.estimate_eligible_for_delete()
-            return render_template('warn.html', message=f"""You've set the time between deleting posts to 0. Every post that matches your expiration rules will be deleted within minutes.
-                    { ("That's about " + str(approx) + " posts.") if approx > 0 else "" }
-                    Go ahead?""")
-        if g.viewer.account.last_delete < datetime.now() - timedelta(days=365):
-            return render_template('warn.html', message="""Once you enable Forget, posts that match your expiration rules will be deleted <b>permanently</b>. We can't bring them back. Make sure that you won't miss them.""")
-
-
-    if not g.viewer.account.policy_enabled:
-        g.viewer.account.last_delete = db.func.now()
-
-    g.viewer.account.policy_enabled = True
-    db.session.commit()
-
-    return redirect(url_for('index'))
-
-
-@app.route('/logout')
-@require_auth
-def logout():
-    if(g.viewer):
-        db.session.delete(g.viewer)
-        db.session.commit()
-        g.viewer = None
-    return redirect(url_for('index'))
-
-@app.route('/api/about')
-def api_about():
-    return jsonify(service='Forget', version=version.version)
-
-@app.route('/api/settings', methods=('PUT',))
-@require_auth_api
-def api_settings_put():
-    viewer = get_viewer()
-    data = request.json
-    updated = dict()
-    for key in lib.settings.attrs:
-        if key in data:
-            setattr(viewer, key, data[key])
-            updated[key] = data[key]
-    db.session.commit()
-    return jsonify(status='success', updated=updated)

routes/__init__.py

@@ -0,0 +1,333 @@
+from flask import render_template, url_for, redirect, request, g,\
+                  make_response
+from datetime import datetime, timedelta, timezone
+import libforget.twitter
+import libforget.mastodon
+import libforget.misskey
+from libforget.auth import require_auth, csrf,\
+                     get_viewer
+from libforget.session import make_session
+from model import Session, TwitterArchive, MastodonApp, MisskeyApp
+from app import app, db, sentry, imgproxy
+import tasks
+from zipfile import BadZipFile
+from twitter import TwitterError
+from urllib.parse import urlparse
+from urllib.error import URLError
+import libforget.version
+import libforget.settings
+import libforget.json
+import re
+
+
+@app.route('/')
+def index():
+    viewer = get_viewer()
+    if viewer:
+        return render_template(
+                'logged_in.html',
+                scales=libforget.interval.SCALES,
+                tweet_archive_failed='tweet_archive_failed' in request.args,
+                settings_error='settings_error' in request.args,
+                viewer_json=libforget.json.account(viewer),
+                )
+    else:
+        return redirect(url_for('about'))
+
+
+@app.route('/about/')
+def about():
+    blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
+    mastodon_instances = libforget.mastodon.suggested_instances(blocklist=blocklist)
+    misskey_instances = libforget.misskey.suggested_instances(blocklist=blocklist)
+    return render_template(
+            'about.html',
+            mastodon_instances=mastodon_instances,
+            misskey_instances=misskey_instances,
+            twitter_login_error='twitter_login_error' in request.args)
+
+
+@app.route('/about/privacy')
+def privacy():
+    return render_template('privacy.html')
+
+
+@app.route('/login/twitter')
+def twitter_login_step1():
+    try:
+        return redirect(libforget.twitter.get_login_url(
+            callback=url_for('twitter_login_step2', _external=True),
+            **app.config.get_namespace("TWITTER_")
+            ))
+    except (TwitterError, URLError):
+        if sentry:
+            sentry.captureException()
+        return redirect(
+                url_for('about', twitter_login_error='', _anchor='log_in'))
+
+
+def login(account_id):
+    session = Session(account_id=account_id)
+    db.session.add(session)
+    db.session.commit()
+
+    session.account.dormant = False
+    db.session.commit()
+
+    tasks.fetch_acc.s(account_id).apply_async(routing_key='high')
+
+    return session
+
+
+@app.route('/login/twitter/callback')
+def twitter_login_step2():
+    try:
+        oauth_token = request.args.get('oauth_token', '')
+        oauth_verifier = request.args.get('oauth_verifier', '')
+        token = libforget.twitter.receive_verifier(
+                oauth_token, oauth_verifier,
+                **app.config.get_namespace("TWITTER_"))
+
+        session = login(token.account_id)
+
+        g.viewer = session
+        return redirect(url_for('index'))
+    except Exception:
+        if sentry:
+            sentry.captureException()
+        return redirect(
+                url_for('about', twitter_login_error='', _anchor='log_in'))
+
+
+@app.route('/upload_tweet_archive', methods=('POST',))
+def upload_tweet_archive():
+    return 403, 'Tweet archive support is temporarily disabled, see banner on the front page.'
+
+
+@app.route('/settings', methods=('POST',))
+@csrf
+@require_auth
+def settings():
+    viewer = get_viewer()
+    try:
+        for attr in libforget.settings.attrs:
+            if attr in request.form:
+                setattr(viewer, attr, request.form[attr])
+        db.session.commit()
+    except ValueError:
+        if sentry:
+            sentry.captureException()
+        return 400
+
+    return redirect(url_for('index', settings_saved=''))
+
+
+@app.route('/disable', methods=('POST',))
+@csrf
+@require_auth
+def disable():
+    g.viewer.account.policy_enabled = False
+    db.session.commit()
+
+    return redirect(url_for('index'))
+
+
+@app.route('/enable', methods=('POST',))
+@csrf
+@require_auth
+def enable():
+    if 'confirm' not in request.form and not g.viewer.account.policy_enabled:
+        if g.viewer.account.policy_delete_every == timedelta(0):
+            approx = g.viewer.account.estimate_eligible_for_delete()
+            return render_template(
+                'warn.html',
+                message=f"""
+                    You've set the time between deleting posts to 0. Every post
+                    that matches your expiration rules will be deleted within
+                    minutes.
+                    { ("That's about " + str(approx) + " posts.") if approx > 0
+                        else "" }
+                    Go ahead?
+                    """)
+        if (not g.viewer.account.last_delete or
+           g.viewer.account.last_delete <
+           datetime.now(timezone.utc) - timedelta(days=365)):
+            return render_template(
+                    'warn.html',
+                    message="""
+                        Once you enable Forget, posts that match your
+                        expiration rules will be deleted <b>permanently</b>.
+                        We can't bring them back. Make sure that you won't
+                        miss them.
+                        """)
+
+    g.viewer.account.policy_enabled = True
+    db.session.commit()
+
+    return redirect(url_for('index'))
+
+
+@app.route('/logout')
+@require_auth
+def logout():
+    if(g.viewer):
+        db.session.delete(g.viewer)
+        db.session.commit()
+        g.viewer = None
+    return redirect(url_for('about'))
+
+
+def domain_from_url(url):
+    return urlparse(url).netloc.lower() or urlparse("//"+url).netloc.lower()
+
+@app.route('/login/mastodon', methods=('GET', 'POST'))
+def mastodon_login_step1(instance=None):
+
+    instance_url = (request.args.get('instance_url', None)
+                    or request.form.get('instance_url', None))
+
+    if not instance_url:
+        blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
+        instances = libforget.mastodon.suggested_instances(
+                limit=30,
+                min_popularity=1,
+                blocklist=blocklist,
+        )
+        return render_template(
+                'mastodon_login.html', instances=instances,
+                address_error=request.method == 'POST',
+                generic_error='error' in request.args
+                )
+
+    instance_url = domain_from_url(instance_url)
+
+    callback = url_for('mastodon_login_step2',
+                       instance_url=instance_url, _external=True)
+
+    try:
+        mastoapp = libforget.mastodon.get_or_create_app(
+                instance_url,
+                callback,
+                url_for('index', _external=True))
+        db.session.merge(mastoapp)
+
+        db.session.commit()
+
+        return redirect(libforget.mastodon.login_url(mastoapp, callback))
+
+    except Exception:
+        if sentry:
+            sentry.captureException()
+        return redirect(url_for('mastodon_login_step1', error=True))
+
+
+@app.route('/login/mastodon/callback/<instance_url>')
+def mastodon_login_step2(instance_url):
+    code = request.args.get('code', None)
+    mastoapp = MastodonApp.query.get(instance_url)
+    if not code or not mastoapp:
+        return redirect(url_for('mastodon_login_step1', error=True))
+
+    callback = url_for('mastodon_login_step2',
+                       instance_url=instance_url, _external=True)
+
+    token = libforget.mastodon.receive_code(code, mastoapp, callback)
+    account = token.account
+
+    session = login(account.id)
+
+    db.session.commit()
+
+    g.viewer = session
+
+    resp = redirect(url_for('index', _anchor='bump_instance'))
+    return resp
+
+
+@app.route('/login/misskey', methods=('GET', 'POST'))
+def misskey_login(instance=None):
+    instance_url = (request.args.get('instance_url', None)
+                    or request.form.get('instance_url', None))
+
+    if not instance_url:
+        blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
+        instances = libforget.misskey.suggested_instances(
+            limit = 30,
+            min_popularity = 1,
+            blocklist=blocklist,
+        )
+        return render_template(
+                'misskey_login.html', instances=instances,
+                address_error=request.method == 'POST',
+                generic_error='error' in request.args
+                )
+
+    instance_url = domain_from_url(instance_url)
+
+    callback = url_for('misskey_callback',
+                       instance_url=instance_url, _external=True)
+
+    try:
+        session = make_session()
+        mkapp = libforget.misskey.get_or_create_app(
+                instance_url,
+                callback,
+                url_for('index', _external=True),
+                session)
+        db.session.merge(mkapp)
+
+        db.session.commit()
+
+        return redirect(libforget.misskey.login_url(mkapp, callback, session))
+
+    except Exception:
+        if sentry:
+            sentry.captureException()
+        return redirect(url_for('misskey_login', error=True))
+
+
+@app.route('/login/misskey/callback/<instance_url>')
+def misskey_callback(instance_url):
+    # legacy auth and miauth use different parameter names
+    token = request.args.get('token', None) or request.args.get('session', None)
+    mkapp = MisskeyApp.query.get(instance_url)
+    if not token or not mkapp:
+        return redirect(url_for('misskey_login', error=True))
+
+    token = libforget.misskey.receive_token(token, mkapp)
+    account = token.account
+
+    session = login(account.id)
+
+    db.session.commit()
+
+    g.viewer = session
+
+    resp = redirect(url_for('index', _anchor='bump_instance'))
+    return resp
+
+
+@app.route('/sentry/setup.js')
+def sentry_setup():
+    client_dsn = app.config.get('SENTRY_DSN').split('@')
+    client_dsn[:1] = client_dsn[0].split(':')
+    client_dsn = ':'.join(client_dsn[0:2]) + '@' + client_dsn[3]
+    resp = make_response(render_template(
+        'sentry.js', sentry_dsn=client_dsn))
+    resp.headers.set('content-type', 'text/javascript')
+    resp.headers.set('cache-control', 'public; max-age=3600')
+    return resp
+
+
+@app.route('/dismiss', methods={'POST'})
+@csrf
+@require_auth
+def dismiss():
+    get_viewer().reason = None
+    db.session.commit()
+    return redirect(url_for('index'))
+
+
+@app.route('/avatar/<identifier>')
+def avatar(identifier):
+    return imgproxy.respond(identifier)

routes/api.py

@@ -0,0 +1,69 @@
+from app import app, db, imgproxy
+from libforget.auth import require_auth_api, get_viewer
+from flask import jsonify, redirect, make_response, request, Response
+from model import Account
+import libforget.settings
+import libforget.json
+import random
+
+@app.route('/api/health_check') # deprecated 2021-03-12
+@app.route('/api/status_check')
+def api_status_check():
+    try:
+        db.session.execute('SELECT 1')
+    except Exception:
+        return ('PostgreSQL bad', 500)
+
+    try:
+        imgproxy.redis.set('forget-status-check', 'howdy', ex=5)
+    except Exception:
+        return ('Redis bad', 500)
+
+    return 'OK'
+
+
+@app.route('/api/settings', methods=('PUT',))
+@require_auth_api
+def api_settings_put():
+    viewer = get_viewer()
+    data = request.json
+    updated = dict()
+    for key in libforget.settings.attrs:
+        if key in data:
+            if (
+                    isinstance(getattr(viewer, key), bool) and
+                    isinstance(data[key], str)):
+                data[key] = data[key] == 'true'
+            setattr(viewer, key, data[key])
+            updated[key] = data[key]
+    db.session.commit()
+    return jsonify(status='success', updated=updated)
+
+
+@app.route('/api/viewer')
+@require_auth_api
+def api_viewer():
+    viewer = get_viewer()
+    resp = make_response(libforget.json.account(viewer))
+    resp.headers.set('content-type', 'application/json')
+    return resp
+
+
+@app.route('/api/reason', methods={'DELETE'})
+@require_auth_api
+def delete_reason():
+    get_viewer().reason = None
+    db.session.commit()
+    return jsonify(status='success')
+
+
+@app.route('/api/badge/users')
+def users_badge():
+    count = (
+        Account.query.filter(Account.policy_enabled)
+        .filter(~Account.dormant)
+        .count()
+        )
+    return redirect(
+            "https://img.shields.io/badge/active%20users-{}-blue.svg"
+            .format(count))

routes/misc.py

@@ -0,0 +1,65 @@
+from app import app, db, sentry
+from flask import g, render_template, make_response, redirect, request
+import version
+import libforget.version
+from libforget.auth import get_viewer_session, set_session_cookie
+
+
+@app.before_request
+def load_viewer():
+    g.viewer = get_viewer_session()
+    if g.viewer and sentry:
+        sentry.user_context({
+                'id': g.viewer.account.id,
+                'username': g.viewer.account.screen_name,
+                'service': g.viewer.account.service
+            })
+
+
+@app.context_processor
+def inject_version():
+    v = version.get_versions()
+    return dict(
+            version=v['version'],
+            repo_url=libforget.version.url_for_version(v),
+        )
+
+
+@app.context_processor
+def inject_sentry():
+    if sentry:
+        return dict(sentry=True)
+    return dict()
+
+
+@app.after_request
+def touch_viewer(resp):
+    if 'viewer' in g and g.viewer:
+        set_session_cookie(g.viewer, resp, app.config.get('HTTPS'))
+        g.viewer.touch()
+        db.session.commit()
+    return resp
+
+
+@app.errorhandler(404)
+def not_found(e):
+    return (render_template('404.html', e=e), 404)
+
+
+@app.errorhandler(500)
+def internal_server_error(e):
+    if request.endpoint and request.endpoint.startswith('api_'):
+        return e.get_response()
+    return (render_template('500.html', e=e), 500)
+
+
+@app.route('/robots.txt')
+def robotstxt():
+    resp = make_response('')
+    resp.headers.set('content-type', 'text/plain')
+    return resp
+
+
+@app.route('/humans.txt')
+def humanstxt():
+    return redirect('https://github.com/codl/forget/graph/contributors')

setup.cfg

@@ -0,0 +1,8 @@
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = version.py
+versionfile_build = version.py
+tag_prefix = v
+#parentdir_prefix =
+

setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup
+import versioneer
+
+setup(
+        name='forget',
+        description='A post-expiring service.',
+
+        url='https://forget.codl.fr/',
+        author='codl',
+        author_email='codl@codl.fr',
+
+        version=versioneer.get_version(),
+        cmdclass=versioneer.get_cmdclass(),
+)
+
+