Compare commits
177 Commits
Author | SHA1 | Date |
---|---|---|
codl | 50ba3dfaa0 | |
codl | 222dca4fb1 | |
codl | 1524554a40 | |
codl | 70b90366b6 | |
codl | 9ac6227ca5 | |
codl | 5afa982cf3 | |
codl | 0fdff9ff6e | |
codl | 43ba8e1362 | |
codl | 3c372a2afb | |
codl | c82c15e0da | |
codl | 3497a63cff | |
codl | 30b7b24e68 | |
codl | fddcbbe8a0 | |
shibao | 59095ae1ef | |
shibao | de2329d041 | |
shibao | 69cb8db391 | |
shibao | 4f2770e4e2 | |
shibao | d2bb9094f0 | |
shibao | b89122edb5 | |
shibao | 5872ce6da8 | |
shibao | 659bb1dfb8 | |
codl | f195ed3261 | |
codl | 5fdb99256f | |
codl | 916a47ef9d | |
codl | 9c035d5132 | |
codl | 38a1c543af | |
codl | 1354d415d3 | |
codl | 44632934a7 | |
codl | 299a99844f | |
codl | c621982424 | |
codl | 82a72bfd32 | |
codl | 917202de1d | |
codl | bbbb2470ed | |
codl | 77bb52cb9e | |
codl | 688b787c16 | |
Johann150 | 88760deb6f | |
Johann150 | aacc5100de | |
Johann150 | 4efe6ec316 | |
codl | d5e0a364a8 | |
codl | ce1990cd0d | |
codl | 61700f3dd9 | |
codl | a66ad7db9c | |
codl | 1eacfca8b4 | |
codl | a6a4416254 | |
codl | c7762e839b | |
codl | 6a14b70d88 | |
codl | 1c65cd2556 | |
codl | a85095cd00 | |
codl | 825313091c | |
codl | 9d2147e905 | |
codl | 20c0c93a5e | |
codl | 1cade39fb9 | |
codl | 21eff570a0 | |
codl | 00bf83388f | |
Johann150 | b54a26cf8f | |
codl | f7f2276cec | |
codl | e5971e3848 | |
Johann150 | e7744a1964 | |
Johann150 | 77a2687d9e | |
Johann150 | 37652e4053 | |
Johann150 | ab2c5c9aae | |
Johann150 | 1522d766fa | |
Johann150 | 623a7e4415 | |
Johann150 | fce0f88d2c | |
Johann150 | dbd7193636 | |
Johann150 | ba72b8acf9 | |
Johann150 | bb496bb5e6 | |
Johann150 | eee3bb82fe | |
Johann150 | 8214cda672 | |
Johann150 | 8b5f56bef2 | |
Johann150 | b20395cc8f | |
Johann150 | ed1c42d30d | |
Johann150 | ce35aa939b | |
Johann150 | 05db96236c | |
codl | 159c1826d3 | |
codl | f3e68e7fb4 | |
codl | 3c0b017141 | |
codl | 8c750d3207 | |
codl | 85b716c11c | |
codl | 98bee9b1cd | |
dependabot[bot] | 96f4c74d8f | |
dependabot[bot] | 1f15a87d97 | |
dependabot[bot] | f2a2976c08 | |
dependabot[bot] | 25aa3c3844 | |
dependabot[bot] | 1dec832666 | |
dependabot[bot] | 0ac2c95284 | |
dependabot[bot] | 6092398dcc | |
codl | 521cd7b1dc | |
codl | f10b9dac80 | |
dependabot[bot] | 785d0509af | |
dependabot[bot] | 2c13e2f167 | |
dependabot[bot] | 7c1b42c7d0 | |
dependabot[bot] | bdfae33102 | |
dependabot[bot] | 4b3ffe976b | |
dependabot[bot] | 4c82298998 | |
dependabot[bot] | f763184ead | |
codl | e274999c2c | |
codl | 242ec23e2d | |
dependabot[bot] | dc5eba6ed5 | |
dependabot[bot] | 7e9aeb7c59 | |
dependabot[bot] | 3dc188e4d2 | |
dependabot[bot] | f6d1e62b01 | |
codl | 3752fb168a | |
dependabot[bot] | a43cf77195 | |
dependabot[bot] | 2352ed84ac | |
dependabot[bot] | 22a2c3a70c | |
dependabot[bot] | cd613259c8 | |
dependabot[bot] | c760178ba9 | |
dependabot[bot] | 1ea6d13bf0 | |
dependabot[bot] | bf6068f5d9 | |
dependabot[bot] | a8350cbf6f | |
dependabot[bot] | 297d4c9d94 | |
codl | 5866592f50 | |
dependabot[bot] | 1a4e3c10e6 | |
codl | ab5e27447d | |
codl | 2f9db993dc | |
codl | 9b42bb4bf0 | |
dependabot[bot] | 095a612473 | |
dependabot[bot] | 47b3384778 | |
dependabot-preview[bot] | ccbe71e650 | |
dependabot[bot] | ece2b33a73 | |
dependabot[bot] | e7140727fa | |
dependabot[bot] | dbdfe05950 | |
dependabot[bot] | 4fe30a38a9 | |
dependabot[bot] | e7774659de | |
dependabot[bot] | 66b7cd7ab8 | |
codl | 3f7f0f7f2f | |
dependabot[bot] | 2672f17d73 | |
dependabot[bot] | 0234217c64 | |
codl | ad56e4720a | |
codl | 162f798df7 | |
codl | ed9146231e | |
codl | 57c0752994 | |
codl | d2c3d95b1e | |
codl | 48b9ec7796 | |
dependabot-preview[bot] | 8d256e1756 | |
codl | 92cce06ff1 | |
dependabot-preview[bot] | 542d0fb35b | |
dependabot-preview[bot] | 65ac5440c7 | |
dependabot-preview[bot] | 4d4a151ce6 | |
dependabot-preview[bot] | 80601ae42e | |
codl | 16c02bb198 | |
dependabot-preview[bot] | d17cf789e4 | |
dependabot-preview[bot] | 560283caa2 | |
codl | d1f02513d6 | |
codl | 5cf0efe05d | |
codl | eabd4740ad | |
codl | 4540d87038 | |
codl | 2e2e915b0a | |
codl | 73ce05d9b7 | |
codl | 25dcf17f54 | |
codl | cdd7d43a18 | |
dependabot-preview[bot] | 4f0bae9a74 | |
codl | 6c8d20b87e | |
codl | 9592ab8511 | |
codl | c5c4b72c6f | |
dependabot-preview[bot] | 741a44bed8 | |
dependabot-preview[bot] | 19dc13bc93 | |
dependabot-preview[bot] | e369b554b1 | |
dependabot-preview[bot] | ac4d143d7f | |
dependabot-preview[bot] | 1668d1cd2d | |
dependabot-preview[bot] | 09f7127e52 | |
codl | 249842ed9f | |
codl | 387b287990 | |
codl | 195371dc97 | |
codl | 289a1df83f | |
codl | fb1725d43a | |
codl | 38339defba | |
codl | ac76dd4ad1 | |
codl | f01b5b1511 | |
dependabot-preview[bot] | cd5ac7a52e | |
dependabot-preview[bot] | cf2ebe22a0 | |
dependabot-preview[bot] | da8fb2b4b5 | |
dependabot-preview[bot] | 0386f55f7b | |
dependabot-preview[bot] | fe765165da | |
dependabot-preview[bot] | 45f70c1132 | |
codl | 7fad5ea458 |
|
@ -1 +0,0 @@
|
|||
exclude_paths: ['version.py', 'versioneer.py']
|
|
@ -0,0 +1,22 @@
|
|||
.envrc.example
|
||||
.gitignore
|
||||
.tool-versions
|
||||
*.md
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
#.git
|
||||
.github
|
||||
.codecov.yml
|
||||
.coveragerc
|
||||
.env
|
||||
.eslintrc.yml
|
||||
.gitattributes
|
||||
LICENSE
|
||||
CHANGELOG.markdown
|
||||
README.markdown
|
||||
config.example.py
|
||||
config.docker.py
|
||||
forget.example.service
|
||||
requirements-dev.txt
|
||||
data
|
||||
config.py
|
|
@ -0,0 +1,12 @@
|
|||
# These are supported funding model platforms
|
||||
|
||||
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: # Replace with a single Open Collective username
|
||||
ko_fi: # Replace with a single Ko-fi username
|
||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
liberapay: codl
|
||||
issuehunt: # Replace with a single IssueHunt username
|
||||
otechie: # Replace with a single Otechie username
|
||||
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
|
@ -0,0 +1,102 @@
|
|||
name: Docker
|
||||
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '18 10 * * *'
|
||||
push:
|
||||
branches: [ "master" ]
|
||||
# Publish semver tags as releases.
|
||||
tags: [ 'v*.*.*' ]
|
||||
pull_request:
|
||||
branches: [ "master" ]
|
||||
|
||||
env:
|
||||
# Use docker.io for Docker Hub if empty
|
||||
REGISTRY: ghcr.io
|
||||
# github.repository as <account>/<repo>
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
# This is used to complete the identity challenge
|
||||
# with sigstore/fulcio when running outside of PRs.
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# Install the cosign tool except on PR
|
||||
# https://github.com/sigstore/cosign-installer
|
||||
- name: Install cosign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: sigstore/cosign-installer@7e0881f8fe90b25e305bbf0309761e9314607e25
|
||||
with:
|
||||
cosign-release: 'v1.9.0'
|
||||
|
||||
|
||||
# Workaround: https://github.com/docker/build-push-action/issues/461
|
||||
- name: Setup Docker buildx
|
||||
uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
|
||||
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into registry ${{ env.REGISTRY }}
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Extract metadata (tags, labels) for Docker
|
||||
# https://github.com/docker/metadata-action
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=schedule
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
|
||||
with:
|
||||
context: .
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
# Sign the resulting Docker image digest except on PRs.
|
||||
# This will only write to the public Rekor transparency log when the Docker
|
||||
# repository is public to avoid leaking data. If you would like to publish
|
||||
# transparency data even for private images, pass --force to cosign below.
|
||||
# https://github.com/sigstore/cosign
|
||||
- name: Sign the published Docker image
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
env:
|
||||
COSIGN_EXPERIMENTAL: "true"
|
||||
# This step uses the identity token to provision an ephemeral certificate
|
||||
# against the sigstore community Fulcio instance.
|
||||
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
|
|
@ -0,0 +1,39 @@
|
|||
name: Run tests on pushes and on PRs
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
run_tests:
|
||||
name: Run tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.6"
|
||||
cache: 'pipenv'
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: "10"
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install pipenv and python dependencies
|
||||
run: |
|
||||
pip install -U pip pipenv
|
||||
pipenv sync -d
|
||||
- name: Install node dependencies
|
||||
run: npm install
|
||||
|
||||
- name: Start Redis
|
||||
run: docker run --name redis --publish 6379:6379 --detach redis
|
||||
|
||||
|
||||
- name: Run tests with pytest
|
||||
run: pipenv run pytest --cov=.
|
||||
|
||||
- uses: codecov/codecov-action@v2
|
|
@ -8,3 +8,8 @@ static/*
|
|||
.cache/
|
||||
.coverage
|
||||
.pytest_cache
|
||||
|
||||
data/*
|
||||
!data/.keep
|
||||
|
||||
docker-compose.override.yml
|
||||
|
|
19
.travis.yml
19
.travis.yml
|
@ -1,19 +0,0 @@
|
|||
dist: xenial
|
||||
sudo: true
|
||||
language: python
|
||||
python:
|
||||
- 3.6
|
||||
- 3.7
|
||||
- 3.7-dev
|
||||
install:
|
||||
- pip install -r requirements.txt -r requirements-dev.txt
|
||||
- npm install
|
||||
script:
|
||||
- pytest --cov=.
|
||||
after_success:
|
||||
- codecov
|
||||
cache:
|
||||
pip: true
|
||||
directories:
|
||||
- node_modules
|
||||
|
|
@ -1,3 +1,46 @@
|
|||
## v2.2.0
|
||||
|
||||
Released 2022-08-10
|
||||
|
||||
* add: instance hidelist
|
||||
<https://github.com/codl/forget/pull/590>
|
||||
* add: docker deployment support (Thanks @shibaobun !)
|
||||
<https://github.com/codl/forget/pull/612>
|
||||
* removed: migration path for known instances list from cookie to localstorage
|
||||
<https://github.com/codl/forget/pull/545>
|
||||
|
||||
## v2.1.0
|
||||
|
||||
Released 2022-03-04
|
||||
|
||||
* add: Misskey support (Thanks @Johann150 !)
|
||||
<https://github.com/codl/forget/pull/544>
|
||||
* fix: lowered database impact of a background task
|
||||
<https://github.com/codl/forget/issue/166>
|
||||
* fix: wording on "favourited posts" is unclear
|
||||
<https://github.com/codl/forget/pull/366>
|
||||
* fix: failing to fetch some posts in some circumstances
|
||||
<https://github.com/codl/forget/issue/584>
|
||||
|
||||
## v2.0.0
|
||||
|
||||
Released 2019-09-13
|
||||
|
||||
* fix: newer twitter accounts not fetching new posts past the initial historical fetch
|
||||
<https://github.com/codl/forget/pull/254>
|
||||
* fix: fetching task not getting rescheduled properly when doing the initial historic fetch
|
||||
<https://github.com/codl/forget/pull/264>
|
||||
* BREAKING: disabled tweet archive upload since twitter has dropped support for them
|
||||
<https://github.com/codl/forget/pull/265>
|
||||
|
||||
## v1.6.1
|
||||
|
||||
Released 2019-07-23
|
||||
|
||||
* increased frequency of refresh jobs
|
||||
* version number in footer now links to changelog instead of commit log
|
||||
* updated about page. less negative, more succint, clarifies that forget is not a purging tool
|
||||
|
||||
## v1.5.3
|
||||
|
||||
Released 2019-07-11
|
||||
|
|
|
@ -0,0 +1,51 @@
|
|||
# syntax=docker/dockerfile:1.4.2
|
||||
|
||||
FROM python:3.10.6-bullseye AS pydeps
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip/http pip install --upgrade pip==22.2.2
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN --mount=type=cache,target=/root/.cache/pip/http pip install -r requirements.txt
|
||||
|
||||
|
||||
FROM pydeps AS pynodedeps
|
||||
|
||||
RUN rm -f /etc/apt/apt.conf.d/docker-clean &&\
|
||||
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' \
|
||||
> /etc/apt/apt.conf.d/keep-cache
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
--mount=type=cache,target=/var/lib/apt \
|
||||
apt-get update -qq && apt-get install -qq nodejs npm
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN --mount=type=cache,target=/root/.npm npm clean-install
|
||||
|
||||
|
||||
FROM scratch AS layer-cake
|
||||
WORKDIR /
|
||||
|
||||
COPY *.py setup.cfg rollup.config.js ./
|
||||
COPY assets assets
|
||||
COPY components components
|
||||
COPY libforget libforget
|
||||
COPY migrations migrations
|
||||
COPY routes routes
|
||||
COPY static static
|
||||
COPY templates templates
|
||||
|
||||
|
||||
FROM pynodedeps AS build
|
||||
|
||||
COPY --from=layer-cake / ./
|
||||
RUN doit
|
||||
|
||||
FROM pydeps
|
||||
|
||||
COPY --from=build /usr/src/app ./
|
||||
|
||||
COPY .git/ .git/
|
||||
|
||||
ENV FLASK_APP=forget.py
|
||||
|
||||
VOLUME ["/var/run/celery"]
|
6
Pipfile
6
Pipfile
|
@ -9,10 +9,11 @@ name = "pypi"
|
|||
|
||||
alembic = "*"
|
||||
brotli = ">=1.0.1"
|
||||
#celery = "~=4.4.2"
|
||||
celery = "*"
|
||||
csscompressor = "*"
|
||||
doit = "*"
|
||||
flask = "~=1.0"
|
||||
flask = ">=1.1"
|
||||
flask-migrate = "*"
|
||||
flask-sqlalchemy = "*"
|
||||
gunicorn = ">=19.8"
|
||||
|
@ -24,7 +25,7 @@ redis = "*"
|
|||
requests = "*"
|
||||
sqlalchemy = "*"
|
||||
twitter = "*"
|
||||
"mastodon.py" = "~=1.2"
|
||||
"mastodon.py" = ">=1.2"
|
||||
blinker = "*"
|
||||
|
||||
|
||||
|
@ -34,5 +35,4 @@ coverage = "*"
|
|||
codecov = "*"
|
||||
pytest = "*"
|
||||
pytest-cov = "*"
|
||||
pytest-redis = "*"
|
||||
versioneer = "*"
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -1,16 +1,26 @@
|
|||
[![Forget](assets/promo.gif)](https://forget.codl.fr)
|
||||
![Forget](assets/promo.gif)
|
||||
|
||||
![User count](https://forget.codl.fr/api/badge/users)
|
||||
![Maintenance status](https://img.shields.io/maintenance/yes/2019.svg)
|
||||
![Maintenance status](https://img.shields.io/maintenance/no/2022.svg)
|
||||
|
||||
[![Build status](https://img.shields.io/travis/codl/forget.svg)](https://travis-ci.org/codl/forget/)
|
||||
[![Test coverage](https://img.shields.io/codecov/c/github/codl/forget.svg)](https://codecov.io/gh/codl/forget)
|
||||
[![Code quality](https://img.shields.io/codacy/grade/1780ac6071c04cbd9ccf75de0891e798.svg)](https://www.codacy.com/app/codl/forget?utm_source=github.com&utm_medium=referral&utm_content=codl/forget&utm_campaign=badger)
|
||||
|
||||
Forget is a post deleting service for Twitter and Mastodon. It lives at
|
||||
<https://forget.codl.fr>.
|
||||
Forget is a post deleting service for Twitter, Mastodon, and Misskey.
|
||||
|
||||
|
||||
## Features
|
||||
|
||||
* Delete your posts when they cross an age threshold.
|
||||
* Or keep your post count in check, deleting old posts when you go over.
|
||||
* Preserve old posts that matter by giving them a favourite or a reaction.
|
||||
* Set it and <i>forget</i> it. Forget works continuously in the background.
|
||||
|
||||
## Non-features
|
||||
|
||||
Forget is not a one-time purging tool. It is designed to prune your account
|
||||
continuously, not quickly. If you need a lot of posts gone fast, you may want
|
||||
to look for another more-suited tool.
|
||||
|
||||
## Running your own
|
||||
|
||||
### Requirements
|
||||
|
@ -18,7 +28,7 @@ Forget is a post deleting service for Twitter and Mastodon. It lives at
|
|||
* Postgresql
|
||||
* Redis
|
||||
* Python 3.6+
|
||||
* Yarn or NPM
|
||||
* Node.js 10+
|
||||
|
||||
|
||||
### Set up venv
|
||||
|
@ -85,7 +95,6 @@ $ honcho start
|
|||
|
||||
The application server will listen on `http://127.0.0.1:42157`.
|
||||
You'll want to use your favourite web server to proxy traffic to it.
|
||||
<small>This author suggests Caddy.</small>
|
||||
|
||||
### Development
|
||||
|
||||
|
@ -103,9 +112,39 @@ You can run the (currently very incomplete) test suite by running `pytest`.
|
|||
You'll need redis installed on your development machine, a temporary redis
|
||||
server will be started and shut down automatically by the test suite.
|
||||
|
||||
---
|
||||
## Docker
|
||||
|
||||
This project is also able to be deployed through Docker.
|
||||
|
||||
1. Copy `config.docker.py` to `config.py` and add additional configurations to
|
||||
your liking.
|
||||
1. By default, the webapp container will be listening on `127.0.0.1:42157`,
|
||||
which you can point a reverse proxy at.
|
||||
* If your reverse proxy is in another docker network then you'll need a
|
||||
`docker-compose.override.yml` file to attach the `www` service to the
|
||||
right network and not publish any ports. An example override file is
|
||||
provided. The web app will be listening on `http://forget-www-1:42157`.
|
||||
1. By default, the `docker-compose.yml` creates relative mounts `./redis`,
|
||||
`./postgres`, and `./celery` relative to the `docker-compose.yml` location.
|
||||
An example `docker-compose.override.yml` file is provided that shows how to
|
||||
change this.
|
||||
1. Run `docker-compose build` to build the image.
|
||||
1. Run `docker-compose up` to start or `docker-compose up -d` to start in the
|
||||
background, and use `docker-compose down` to stop.
|
||||
|
||||
## Contact
|
||||
|
||||
If you're having trouble with Forget, or if you're not having trouble but you
|
||||
just want to tell me you like it, you can drop me a note at
|
||||
[@codl@chitter.xyz](https://chitter.xyz/@codl) or
|
||||
[codl@codl.fr](mailto:codl@codl.fr). Thanks for reading this readme.
|
||||
[codl@codl.fr](mailto:codl@codl.fr).
|
||||
|
||||
## Greetz
|
||||
|
||||
Thank you bea, for making ephemeral, inspiring me to make [limiter][], then this,
|
||||
in an attempt to bring ephemeral with me everywhere. ☕
|
||||
|
||||
[limiter]: https://github.com/codl/limiter
|
||||
|
||||
Thank you to the kind folks who have emailed me to tell me Forget has made their
|
||||
time on social media less stressful. 🌻
|
||||
|
|
5
app.py
5
app.py
|
@ -7,6 +7,7 @@ from libforget.cachebust import cachebust
|
|||
import mimetypes
|
||||
import libforget.brotli
|
||||
import libforget.img_proxy
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
@ -16,7 +17,7 @@ default_config = {
|
|||
"HTTPS": True,
|
||||
"SENTRY_CONFIG": {},
|
||||
"REPO_URL": "https://github.com/codl/forget",
|
||||
"COMMIT_URL": "https://github.com/codl/forget/commits/{hash}",
|
||||
"CHANGELOG_URL": "https://github.com/codl/forget/blob/{hash}/CHANGELOG.markdown",
|
||||
"REDIS_URI": "redis://",
|
||||
}
|
||||
|
||||
|
@ -94,3 +95,5 @@ libforget.brotli.brotli(app)
|
|||
|
||||
imgproxy = (
|
||||
libforget.img_proxy.ImgProxyCache(redis_uri=app.config.get('REDIS_URI')))
|
||||
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
|
||||
|
|
|
@ -2,38 +2,26 @@ import {SLOTS, normalize_known, known_load, known_save} from './known_instances.
|
|||
|
||||
(function instance_buttons(){
|
||||
|
||||
const container = document.querySelector('#mastodon_instance_buttons');
|
||||
const button_template = Function('first', 'instance',
|
||||
'return `' + document.querySelector('#instance_button_template').innerHTML + '`;');
|
||||
const another_button_template = Function(
|
||||
const mastodon_container = document.querySelector('#mastodon_instance_buttons');
|
||||
const mastodon_button_template = Function('first', 'instance',
|
||||
'return `' + document.querySelector('#mastodon_instance_button_template').innerHTML + '`;');
|
||||
const mastodon_another_button_template = Function(
|
||||
'return `' +
|
||||
document.querySelector('#another_instance_button_template').innerHTML + '`;');
|
||||
const top_instances =
|
||||
Function('return JSON.parse(`' + document.querySelector('#top_instances').innerHTML + '`);')();
|
||||
|
||||
async function get_known(){
|
||||
let known = known_load();
|
||||
if(!known){
|
||||
let resp = await fetch('/api/known_instances');
|
||||
if(resp.ok && resp.headers.get('content-type') == 'application/json'){
|
||||
known = await resp.json();
|
||||
}
|
||||
else {
|
||||
known = [{
|
||||
"instance": "mastodon.social",
|
||||
"hits": 0
|
||||
}];
|
||||
}
|
||||
known_save(known)
|
||||
fetch('/api/known_instances', {method: 'DELETE'})
|
||||
}
|
||||
|
||||
return known;
|
||||
}
|
||||
document.querySelector('#mastodon_another_instance_button_template').innerHTML + '`;');
|
||||
const mastodon_top_instances =
|
||||
Function('return JSON.parse(`' + document.querySelector('#mastodon_top_instances').innerHTML + '`);')();
|
||||
|
||||
const misskey_container = document.querySelector('#misskey_instance_buttons');
|
||||
const misskey_button_template = Function('first', 'instance',
|
||||
'return `' + document.querySelector('#misskey_instance_button_template').innerHTML + '`;');
|
||||
const misskey_another_button_template = Function(
|
||||
'return `' +
|
||||
document.querySelector('#misskey_another_instance_button_template').innerHTML + '`;');
|
||||
const misskey_top_instances =
|
||||
Function('return JSON.parse(`' + document.querySelector('#misskey_top_instances').innerHTML + '`);')();
|
||||
|
||||
async function replace_buttons(){
|
||||
let known = await get_known();
|
||||
let known = known_load();
|
||||
|
||||
known = normalize_known(known);
|
||||
known_save(known);
|
||||
|
@ -41,7 +29,7 @@ import {SLOTS, normalize_known, known_load, known_save} from './known_instances.
|
|||
let filtered_top_instances = []
|
||||
for(let instance of top_instances){
|
||||
let found = false;
|
||||
for(let k of known){
|
||||
for(let k of known_instances){
|
||||
if(k['instance'] == instance['instance']){
|
||||
found = true;
|
||||
break;
|
||||
|
@ -52,20 +40,35 @@ import {SLOTS, normalize_known, known_load, known_save} from './known_instances.
|
|||
}
|
||||
}
|
||||
|
||||
let instances = known.concat(filtered_top_instances).slice(0, SLOTS);
|
||||
let instances = known_instances.concat(filtered_top_instances).slice(0, SLOTS);
|
||||
|
||||
let html = '';
|
||||
|
||||
let first = true;
|
||||
for(let instance of instances){
|
||||
html += button_template(first, instance['instance'])
|
||||
html += template(first, instance['instance'])
|
||||
first = false;
|
||||
}
|
||||
|
||||
html += another_button_template();
|
||||
html += template_another_instance();
|
||||
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
replace_buttons();
|
||||
async function init_buttons(){
|
||||
let known = await get_known();
|
||||
|
||||
known.mastodon = normalize_known(known.mastodon);
|
||||
known.misskey = normalize_known(known.misskey);
|
||||
known_save(known);
|
||||
|
||||
replace_buttons(mastodon_top_instances, known.mastodon,
|
||||
mastodon_container, mastodon_button_template,
|
||||
mastodon_another_button_template);
|
||||
replace_buttons(misskey_top_instances, known.misskey,
|
||||
misskey_container, misskey_button_template,
|
||||
misskey_another_button_template);
|
||||
}
|
||||
|
||||
init_buttons();
|
||||
})();
|
||||
|
|
|
@ -1,16 +1,44 @@
|
|||
const STORAGE_KEY = 'forget_known_instances';
|
||||
const STORAGE_KEY = 'forget_known_instances@2021-12-09';
|
||||
export const SLOTS = 5;
|
||||
|
||||
function load_and_migrate_old(){
|
||||
const OLD_KEY = "forget_known_instances";
|
||||
let olddata = localStorage.getItem(OLD_KEY);
|
||||
if(olddata != null){
|
||||
olddata = JSON.parse(olddata)
|
||||
let newdata = {
|
||||
mastodon: olddata,
|
||||
misskey: [{
|
||||
"instance": "misskey.io",
|
||||
"hits": 0
|
||||
}]
|
||||
};
|
||||
known_save(newdata);
|
||||
localStorage.removeItem(OLD_KEY);
|
||||
return newdata;
|
||||
}
|
||||
}
|
||||
|
||||
export function known_save(known){
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(known));
|
||||
}
|
||||
|
||||
export function known_load(){
|
||||
const default_ = {
|
||||
mastodon:[{ "instance": "mastodon.social", "hits": 0 }],
|
||||
misskey:[{ "instance": "misskey.io", "hits": 0 }],
|
||||
};
|
||||
// this makes mastodon.social and misskey.io show up on respective first
|
||||
// buttons by default even if they are not the most popular instance
|
||||
// according to the server
|
||||
|
||||
let known = localStorage.getItem(STORAGE_KEY);
|
||||
if(known){
|
||||
known = JSON.parse(known);
|
||||
} else {
|
||||
known = load_and_migrate_old();
|
||||
}
|
||||
return known;
|
||||
return known || default_;
|
||||
}
|
||||
|
||||
export function normalize_known(known){
|
||||
|
|
Binary file not shown.
After Width: | Height: | Size: 18 KiB |
|
@ -1,5 +1,4 @@
|
|||
import Banner from '../components/Banner.html';
|
||||
import ArchiveForm from '../components/ArchiveForm.html';
|
||||
import {known_load, known_save} from './known_instances.js'
|
||||
|
||||
(function settings_init(){
|
||||
|
@ -195,23 +194,10 @@ import {known_load, known_save} from './known_instances.js'
|
|||
})
|
||||
}
|
||||
|
||||
let archive_form_el = document.querySelector('#archive-form');
|
||||
if(archive_form_el){
|
||||
let csrf_token = archive_form_el.querySelector('input[name=csrf-token]').value;
|
||||
let archive_form = new ArchiveForm({
|
||||
target: archive_form_el,
|
||||
hydrate: true,
|
||||
data: {
|
||||
action: archive_form_el.action,
|
||||
csrf_token: csrf_token,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
function bump_instance(instance_name){
|
||||
function bump_instance(service, instance_name){
|
||||
let known_instances = known_load();
|
||||
let found = false;
|
||||
for(let instance of known_instances){
|
||||
for(let instance of known_instances[service]){
|
||||
if(instance['instance'] == instance_name){
|
||||
instance.hits ++;
|
||||
found = true;
|
||||
|
@ -220,15 +206,17 @@ import {known_load, known_save} from './known_instances.js'
|
|||
}
|
||||
if(!found){
|
||||
let instance = {"instance": instance_name, "hits": 1};
|
||||
known_instances.push(instance);
|
||||
known_instances[service].push(instance);
|
||||
}
|
||||
|
||||
known_save(known_instances);
|
||||
|
||||
}
|
||||
|
||||
if(viewer_from_dom['service'] == 'mastodon' && location.hash == '#bump_instance'){
|
||||
bump_instance(viewer_from_dom['id'].split('@')[1])
|
||||
if(location.hash == '#bump_instance' && (
|
||||
viewer_from_dom['service'] == 'mastodon' || viewer_from_dom['service'] == 'misskey'
|
||||
)){
|
||||
bump_instance(viewer_from_dom['service'], viewer_from_dom['id'].split('@')[1])
|
||||
let url = new URL(location.href)
|
||||
url.hash = '';
|
||||
history.replaceState('', '', url);
|
||||
|
|
|
@ -9,7 +9,7 @@ body {
|
|||
}
|
||||
|
||||
body > section, body > header, body > footer {
|
||||
max-width: 40rem;
|
||||
max-width: 45rem;
|
||||
margin-left: auto;
|
||||
margin-right: auto;
|
||||
}
|
||||
|
@ -227,6 +227,18 @@ button {
|
|||
background-color: #37d;
|
||||
}
|
||||
|
||||
.btn.primary.twitter-colored {
|
||||
background-color: #1da1f2;
|
||||
}
|
||||
|
||||
.btn.primary.mastodon-colored {
|
||||
background-color: #282c37;
|
||||
}
|
||||
|
||||
.btn.primary.misskey-colored {
|
||||
background-color: #66b300;
|
||||
}
|
||||
|
||||
.btn.secondary {
|
||||
background-color: rgba(255,255,255,0.5);
|
||||
box-shadow: inset 0 0 0 1px rgba(0,0,0,0.3);
|
||||
|
|
|
@ -1,38 +0,0 @@
|
|||
<form action={action} method='post' enctype='multipart/form-data'>
|
||||
{#if file_too_big}
|
||||
<div class="banner warning">
|
||||
<p>The file you have selected is very large.</p>
|
||||
|
||||
<p>Twitter has two types of archives, <b>Forget does not support
|
||||
"Your Twitter data" archives, only "Tweet archives"</b>
|
||||
available from
|
||||
<a href="https://twitter.com/settings/account#export_request">your account settings</a>.
|
||||
Please make sure you have the right type of archive.</p>
|
||||
</div>
|
||||
{/if}
|
||||
<input type="file" name="file" accept="application/zip,.zip"
|
||||
on:change={take_file(this.files)}>
|
||||
<input type="submit" value="Upload">
|
||||
<input type='hidden' name='csrf-token' value={csrf_token}>
|
||||
</form>
|
||||
|
||||
<script>
|
||||
export let csrf_token, action;
|
||||
|
||||
function get_file_size(file_list){
|
||||
/* returns size of selected file given an <input type="file">
|
||||
or 0 if no file is selected */
|
||||
let size = 0;
|
||||
for(let i = 0; i < file_list.length; i++){
|
||||
size += file_list[i].size;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
function take_file(file_list){
|
||||
file_size: get_file_size(file_list);
|
||||
}
|
||||
|
||||
let file_size = 0;
|
||||
$: file_too_big = file_size > 10000000; // 10 MB
|
||||
</script>
|
|
@ -0,0 +1,76 @@
|
|||
"""
|
||||
this is an example config file for Forget
|
||||
|
||||
copy this file to config.py before editing
|
||||
|
||||
lines starting with # demonstrate default or example values
|
||||
the # should be removed before editing
|
||||
"""
|
||||
|
||||
"""
|
||||
DATABASE URI
|
||||
|
||||
determines where to connect to the database
|
||||
see <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls> for syntax
|
||||
only postgresql with psycopg2 driver is officially supported
|
||||
"""
|
||||
SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://postgres:postgres@db/forget'
|
||||
|
||||
"""
|
||||
REDIS URI
|
||||
|
||||
see <https://redis-py.readthedocs.io/en/latest/#redis.ConnectionPool.from_url>
|
||||
for syntax reference
|
||||
"""
|
||||
REDIS_URI='redis://redis'
|
||||
|
||||
"""
|
||||
SERVER ADDRESS
|
||||
|
||||
This is the address at which forget will be reached.
|
||||
External services will redirect to this address when logging in.
|
||||
"""
|
||||
# SERVER_NAME="0.0.0.0:5000"
|
||||
# HTTPS=True
|
||||
|
||||
"""
|
||||
TWITTER CREDENTIALS
|
||||
|
||||
Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
|
||||
When prompted for it, your callback URL is {SERVER_NAME}/login/twitter/callback
|
||||
"""
|
||||
# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
|
||||
# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
|
||||
|
||||
"""
|
||||
SENTRY
|
||||
|
||||
If you want to send exceptions to sentry, enter your sentry DSN here
|
||||
"""
|
||||
# SENTRY_DSN=''
|
||||
|
||||
"""
|
||||
HIDDEN INSTANCES
|
||||
|
||||
The front page shows one-click login buttons for the mastodon and
|
||||
misskey instances that see the most heavy use. Instances configured in this
|
||||
list will be prevented from appearing in these buttons.
|
||||
|
||||
They will still appear if a user has previously logged into them and their
|
||||
browser remembers it. A user will still be able to log into them by manually
|
||||
typing the address into the log in form.
|
||||
|
||||
This is a space-delimited list. Example syntax:
|
||||
HIDDEN_INSTANCES='social.example.com pleroma.example.net mk.example.org'
|
||||
"""
|
||||
# HIDDEN_INSTANCES=''
|
||||
|
||||
"""
|
||||
ADVANCED FLASK CONFIG
|
||||
|
||||
you can also use any config variable that flask expects here
|
||||
A list of these config variables is available here:
|
||||
<http://flask.pocoo.org/docs/1.0/config/#builtin-configuration-values>
|
||||
"""
|
||||
# SESSION_COOKIE_SECURE=True
|
||||
# DEBUG=True
|
|
@ -2,6 +2,9 @@
|
|||
this is an example config file for Forget
|
||||
|
||||
copy this file to config.py before editing
|
||||
|
||||
lines starting with # demonstrate default or example values
|
||||
the # should be removed before editing
|
||||
"""
|
||||
|
||||
"""
|
||||
|
@ -21,15 +24,6 @@ for syntax reference
|
|||
"""
|
||||
# REDIS_URI='redis://'
|
||||
|
||||
"""
|
||||
TWITTER CREDENTIALS
|
||||
|
||||
Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
|
||||
Twitter locked me out of it so I can't guide you more than that. Sorry.
|
||||
"""
|
||||
# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
|
||||
# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
|
||||
|
||||
"""
|
||||
SERVER ADDRESS
|
||||
|
||||
|
@ -39,13 +33,37 @@ External services will redirect to this address when logging in.
|
|||
# SERVER_NAME="localhost:5000"
|
||||
# HTTPS=True
|
||||
|
||||
"""
|
||||
TWITTER CREDENTIALS
|
||||
|
||||
Apply for api keys on the developer portal <https://developer.twitter.com/en/apps>
|
||||
When prompted for it, your callback URL is {SERVER_NAME}/login/twitter/callback
|
||||
"""
|
||||
# TWITTER_CONSUMER_KEY='yN3DUNVO0Me63IAQdhTfCA'
|
||||
# TWITTER_CONSUMER_SECRET='c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g'
|
||||
|
||||
"""
|
||||
SENTRY
|
||||
|
||||
If you want to send exceptions to sentry, enter your sentry DSN here
|
||||
"""
|
||||
# SENTRY_DSN=
|
||||
# SENTRY_DSN=''
|
||||
|
||||
"""
|
||||
HIDDEN INSTANCES
|
||||
|
||||
The front page shows one-click login buttons for the mastodon and
|
||||
misskey instances that see the most heavy use. Instances configured in this
|
||||
list will be prevented from appearing in these buttons.
|
||||
|
||||
They will still appear if a user has previously logged into them and their
|
||||
browser remembers it. A user will still be able to log into them by manually
|
||||
typing the address into the log in form.
|
||||
|
||||
This is a space-delimited list. Example syntax:
|
||||
HIDDEN_INSTANCES='social.example.com pleroma.example.net mk.example.org'
|
||||
"""
|
||||
# HIDDEN_INSTANCES=''
|
||||
|
||||
"""
|
||||
ADVANCED FLASK CONFIG
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
## uncomment below and change network name to use a reverse proxy on an
|
||||
## existing network
|
||||
#networks:
|
||||
# mycoolreverseproxynetwork:
|
||||
# external: true
|
||||
|
||||
services:
|
||||
www:
|
||||
## uncomment below to stop listening on 127.0.0.1
|
||||
# ports: []
|
||||
|
||||
networks:
|
||||
- forget
|
||||
## uncomment below and change network name to use a reverse proxy on an
|
||||
## existing network
|
||||
# - mycoolreverseproxynetwork
|
||||
|
||||
|
||||
## if you wish to change where postgres, redis, and celery persistent data is
|
||||
## stored, uncomment below and change the first part of each volume (before the `:`)
|
||||
|
||||
# redis:
|
||||
# volumes:
|
||||
# - /my/cool/redis/path:/data
|
||||
# db:
|
||||
# volumes:
|
||||
# - /my/cool/postgres/path:/var/lib/postgresql/data
|
||||
# worker:
|
||||
# volumes:
|
||||
# - /my/cool/celery/path:/var/run/celery
|
||||
# beat:
|
||||
# volumes:
|
||||
# - /my/cool/celery/path:/var/run/celery
|
|
@ -0,0 +1,103 @@
|
|||
services:
|
||||
www:
|
||||
build:
|
||||
context: ./
|
||||
image: ghcr.io/codl/forget
|
||||
pull_policy: missing
|
||||
restart: always
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ./config.py
|
||||
target: /usr/src/app/config.py
|
||||
read_only: true
|
||||
depends_on:
|
||||
- redis
|
||||
- db
|
||||
- worker
|
||||
- beat
|
||||
command: bash -c "
|
||||
flask db upgrade &&
|
||||
gunicorn -w 9 -t 3600 -b 0.0.0.0:42157 forget:app
|
||||
"
|
||||
networks:
|
||||
- forget
|
||||
expose:
|
||||
- 42157
|
||||
ports:
|
||||
- "127.0.0.1:42157:42157"
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: ./
|
||||
image: ghcr.io/codl/forget
|
||||
pull_policy: missing
|
||||
restart: always
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ./config.py
|
||||
target: /usr/src/app/config.py
|
||||
read_only: true
|
||||
- ./data/celery/run:/var/run/celery
|
||||
depends_on:
|
||||
- redis
|
||||
- db
|
||||
networks:
|
||||
- forget
|
||||
command: bash -c "
|
||||
mkdir -p /var/run/celery &&
|
||||
chown -R nobody:nogroup /var/run/celery &&
|
||||
exec celery --app=tasks worker
|
||||
--loglevel=INFO
|
||||
--statedb=/var/run/celery/worker.state
|
||||
--hostname=worker
|
||||
--uid=nobody --gid=nogroup
|
||||
"
|
||||
|
||||
beat:
|
||||
build:
|
||||
context: ./
|
||||
image: ghcr.io/codl/forget
|
||||
pull_policy: missing
|
||||
restart: always
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ./config.py
|
||||
target: /usr/src/app/config.py
|
||||
read_only: true
|
||||
- ./data/celery/run:/var/run/celery
|
||||
depends_on:
|
||||
- redis
|
||||
- db
|
||||
networks:
|
||||
- forget
|
||||
command: bash -c "
|
||||
mkdir -p /var/run/celery &&
|
||||
chown -R nobody:nogroup /var/run/celery &&
|
||||
exec celery --app=tasks beat
|
||||
--loglevel=INFO
|
||||
--schedule=/var/run/celery/schedule
|
||||
--uid=nobody --gid=nogroup
|
||||
"
|
||||
|
||||
redis:
|
||||
image: redis:4.0-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- ./data/redis:/data
|
||||
networks:
|
||||
- forget
|
||||
|
||||
db:
|
||||
image: postgres:14-alpine
|
||||
restart: always
|
||||
environment:
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=postgres
|
||||
- POSTGRES_DB=forget
|
||||
volumes:
|
||||
- ./data/postgres:/var/lib/postgresql/data
|
||||
networks:
|
||||
- forget
|
||||
|
||||
networks:
|
||||
forget:
|
2
dodo.py
2
dodo.py
|
@ -60,7 +60,7 @@ def task_service_icon():
|
|||
formats = ('webp', 'png')
|
||||
for width in widths:
|
||||
for image_format in formats:
|
||||
for basename in ('twitter', 'mastodon'):
|
||||
for basename in ('twitter', 'mastodon', 'misskey'):
|
||||
yield dict(
|
||||
name='{}-{}.{}'.format(basename, width, image_format),
|
||||
actions=[(resize_image, (basename, width, image_format))],
|
||||
|
|
|
@ -5,6 +5,7 @@ from hashlib import sha256
|
|||
import redis as libredis
|
||||
import os.path
|
||||
import mimetypes
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
|
||||
class BrotliCache(object):
|
||||
|
@ -34,32 +35,35 @@ class BrotliCache(object):
|
|||
digest = sha256(body).hexdigest()
|
||||
cache_key = 'brotlicache:{}'.format(digest)
|
||||
|
||||
encbody = self.redis.get(cache_key)
|
||||
response.headers.set('brotli-cache', 'HIT')
|
||||
if not encbody:
|
||||
response.headers.set('brotli-cache', 'MISS')
|
||||
lock_key = 'brotlicache:lock:{}'.format(digest)
|
||||
if self.redis.set(lock_key, 1, nx=True, ex=10):
|
||||
mode = (
|
||||
brotli_.MODE_TEXT
|
||||
if response.content_type.startswith('text/')
|
||||
else brotli_.MODE_GENERIC)
|
||||
t = Thread(
|
||||
target=self.compress_and_cache,
|
||||
args=(cache_key, lock_key, body, mode))
|
||||
t.start()
|
||||
if self.timeout > 0:
|
||||
t.join(self.timeout)
|
||||
encbody = self.redis.get(cache_key)
|
||||
if not encbody:
|
||||
response.headers.set('brotli-cache', 'TIMEOUT')
|
||||
else:
|
||||
response.headers.set('brotli-cache', 'LOCKED')
|
||||
if encbody:
|
||||
response.headers.set('content-encoding', 'br')
|
||||
response.headers.set('vary', 'accept-encoding')
|
||||
response.set_data(encbody)
|
||||
return response
|
||||
try:
|
||||
encbody = self.redis.get(cache_key)
|
||||
response.headers.set('brotli-cache', 'HIT')
|
||||
if not encbody:
|
||||
response.headers.set('brotli-cache', 'MISS')
|
||||
lock_key = 'brotlicache:lock:{}'.format(digest)
|
||||
if self.redis.set(lock_key, 1, nx=True, ex=10):
|
||||
mode = (
|
||||
brotli_.MODE_TEXT
|
||||
if response.content_type.startswith('text/')
|
||||
else brotli_.MODE_GENERIC)
|
||||
t = Thread(
|
||||
target=self.compress_and_cache,
|
||||
args=(cache_key, lock_key, body, mode))
|
||||
t.start()
|
||||
if self.timeout > 0:
|
||||
t.join(self.timeout)
|
||||
encbody = self.redis.get(cache_key)
|
||||
if not encbody:
|
||||
response.headers.set('brotli-cache', 'TIMEOUT')
|
||||
else:
|
||||
response.headers.set('brotli-cache', 'LOCKED')
|
||||
if encbody:
|
||||
response.headers.set('content-encoding', 'br')
|
||||
response.headers.set('vary', 'accept-encoding')
|
||||
response.set_data(encbody)
|
||||
return response
|
||||
except RedisError:
|
||||
response.headers.set('brotli-cache', 'ERROR')
|
||||
|
||||
return response
|
||||
|
||||
|
|
|
@ -65,7 +65,7 @@ class ImgProxyCache(object):
|
|||
if(resp.status_code != 200):
|
||||
return
|
||||
|
||||
header_whitelist = [
|
||||
allowed_headers = [
|
||||
'content-type',
|
||||
'cache-control',
|
||||
'etag',
|
||||
|
@ -81,7 +81,7 @@ class ImgProxyCache(object):
|
|||
if match:
|
||||
expire = max(self.expire, int(match.group(1)))
|
||||
|
||||
for key in header_whitelist:
|
||||
for key in allowed_headers:
|
||||
if key in resp.headers:
|
||||
headers[key] = resp.headers[key]
|
||||
self.redis.set(self.key('headers', url), pickle.dumps(headers, -1),
|
||||
|
|
|
@ -145,8 +145,6 @@ def post_from_api_object(obj, instance):
|
|||
created_at=obj['created_at'],
|
||||
author_id=account_from_api_object(obj['account'], instance).id,
|
||||
direct=obj['visibility'] == 'direct',
|
||||
favourites=obj['favourites_count'],
|
||||
reblogs=obj['reblogs_count'],
|
||||
is_reblog=obj['reblog'] is not None,
|
||||
)
|
||||
|
||||
|
@ -197,11 +195,11 @@ def delete(post):
|
|||
raise TemporaryError(e)
|
||||
|
||||
|
||||
def suggested_instances(limit=5, min_popularity=5, blacklist=tuple()):
|
||||
def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
|
||||
return tuple((ins.instance for ins in (
|
||||
MastodonInstance.query
|
||||
.filter(MastodonInstance.popularity > min_popularity)
|
||||
.filter(~MastodonInstance.instance.in_(blacklist))
|
||||
.filter(~MastodonInstance.instance.in_(blocklist))
|
||||
.order_by(db.desc(MastodonInstance.popularity),
|
||||
MastodonInstance.instance)
|
||||
.limit(limit).all())))
|
||||
|
|
|
@ -0,0 +1,191 @@
|
|||
from app import db, sentry
|
||||
from model import MisskeyApp, MisskeyInstance, Account, OAuthToken, Post
|
||||
from uuid import uuid4
|
||||
from hashlib import sha256
|
||||
from libforget.exceptions import TemporaryError, PermanentError
|
||||
from libforget.session import make_session
|
||||
|
||||
def get_or_create_app(instance_url, callback, website, session):
|
||||
instance_url = instance_url
|
||||
app = MisskeyApp.query.get(instance_url)
|
||||
|
||||
if not app:
|
||||
# check if the instance uses https while getting instance infos
|
||||
try:
|
||||
r = session.post('https://{}/api/meta'.format(instance_url))
|
||||
r.raise_for_status()
|
||||
proto = 'https'
|
||||
except Exception:
|
||||
r = session.post('http://{}/api/meta'.format(instance_url))
|
||||
r.raise_for_status()
|
||||
proto = 'http'
|
||||
|
||||
# This is using the legacy authentication method, because the newer
|
||||
# Miauth method breaks the ability to log out and log back into forget.
|
||||
|
||||
app = MisskeyApp()
|
||||
app.instance = instance_url
|
||||
app.protocol = proto
|
||||
|
||||
# register the app
|
||||
r = session.post('{}://{}/api/app/create'.format(app.protocol, app.instance), json = {
|
||||
'name': 'forget',
|
||||
'description': website,
|
||||
'permission': ['write:notes', 'read:reactions'],
|
||||
'callbackUrl': callback
|
||||
})
|
||||
r.raise_for_status()
|
||||
app.client_secret = r.json()['secret']
|
||||
|
||||
return app
|
||||
|
||||
def login_url(app, callback, session):
|
||||
# will use the callback we gave the server in `get_or_create_app`
|
||||
r = session.post('{}://{}/api/auth/session/generate'.format(app.protocol, app.instance), json = {
|
||||
'appSecret': app.client_secret
|
||||
})
|
||||
r.raise_for_status()
|
||||
# we already get the retrieval token here, but we get it again later so
|
||||
# we do not have to store it
|
||||
return r.json()['url']
|
||||
|
||||
def receive_token(token, app):
|
||||
session = make_session()
|
||||
|
||||
r = session.post('{}://{}/api/auth/session/userkey'.format(app.protocol, app.instance), json = {
|
||||
'appSecret': app.client_secret,
|
||||
'token': token
|
||||
})
|
||||
r.raise_for_status()
|
||||
|
||||
token = sha256(r.json()['accessToken'].encode('utf-8') + app.client_secret.encode('utf-8')).hexdigest()
|
||||
|
||||
acc = account_from_user(r.json()['user'], app.instance)
|
||||
acc = db.session.merge(acc)
|
||||
token = OAuthToken(token = token)
|
||||
token = db.session.merge(token)
|
||||
token.account = acc
|
||||
|
||||
return token
|
||||
|
||||
def check_auth(account, app, session):
|
||||
# there is no explicit check, we can only try getting user info
|
||||
r = session.post('{}://{}/api/i'.format(app.protocol, app.instance), json = {'i': account.tokens[0].token})
|
||||
|
||||
if r.status_code != 200:
|
||||
raise TemporaryError("{} {}".format(r.status_code, r.text))
|
||||
|
||||
if r.json()['isSuspended']:
|
||||
# this is technically a temporary error, but like for twitter
|
||||
# its handled as permanent to not make useless API calls
|
||||
raise PermanentError("Misskey account suspended")
|
||||
|
||||
def account_from_user(user, host):
|
||||
return Account(
|
||||
# in objects that get returned from misskey, the local host is
|
||||
# set to None
|
||||
misskey_instance=host,
|
||||
misskey_id=user['id'],
|
||||
screen_name='{}@{}'.format(user['username'], host),
|
||||
display_name=user['name'],
|
||||
avatar_url=user['avatarUrl'],
|
||||
# the notes count is not always included, especially not when
|
||||
# fetching posts. in that case assume its not needed
|
||||
reported_post_count=user.get('notesCount', None),
|
||||
)
|
||||
|
||||
def post_from_api_object(obj, host):
|
||||
return Post(
|
||||
# in objects that get returned from misskey, the local host is
|
||||
# set to None
|
||||
misskey_instance=host,
|
||||
misskey_id=obj['id'],
|
||||
favourite=('myReaction' in obj
|
||||
and bool(obj['myReaction'])),
|
||||
has_media=('fileIds' in obj
|
||||
and bool(obj['fileIds'])),
|
||||
created_at=obj['createdAt'],
|
||||
author_id=account_from_user(obj['user'], host).id,
|
||||
direct=obj['visibility'] == 'specified',
|
||||
is_reblog=obj['renoteId'] is not None,
|
||||
)
|
||||
|
||||
def fetch_posts(acc, max_id, since_id):
|
||||
app = MisskeyApp.query.get(acc.misskey_instance)
|
||||
session = make_session()
|
||||
check_auth(acc, app, session)
|
||||
|
||||
kwargs = dict(
|
||||
limit=100,
|
||||
userId=acc.misskey_id,
|
||||
# for some reason the token is needed so misskey also sends `myReaction`
|
||||
i=acc.tokens[0].token
|
||||
)
|
||||
if max_id:
|
||||
kwargs['untilId'] = max_id
|
||||
if since_id:
|
||||
kwargs['sinceId'] = since_id
|
||||
|
||||
notes = session.post('{}://{}/api/users/notes'.format(app.protocol, app.instance), json=kwargs)
|
||||
|
||||
if notes.status_code != 200:
|
||||
raise TemporaryError('{} {}'.format(notes.status_code, notes.text))
|
||||
|
||||
return [post_from_api_object(note, app.instance) for note in notes.json()]
|
||||
|
||||
|
||||
def refresh_posts(posts):
|
||||
acc = posts[0].author
|
||||
app = MisskeyApp.query.get(acc.misskey_instance)
|
||||
session = make_session()
|
||||
check_auth(acc, app, session)
|
||||
|
||||
new_posts = list()
|
||||
with db.session.no_autoflush:
|
||||
for post in posts:
|
||||
print('Refreshing {}'.format(post))
|
||||
r = session.post('{}://{}/api/notes/show'.format(app.protocol, app.instance), json={
|
||||
'i': acc.tokens[0].token,
|
||||
'noteId': post.misskey_id
|
||||
})
|
||||
if r.status_code != 200:
|
||||
try:
|
||||
if r.json()['error']['code'] == 'NO_SUCH_NOTE':
|
||||
db.session.delete(post)
|
||||
continue
|
||||
except Exception as e:
|
||||
raise TemporaryError(e)
|
||||
raise TemporaryError('{} {}'.format(r.status_code, r.text))
|
||||
|
||||
new_post = db.session.merge(post_from_api_object(r.json(), app.instance))
|
||||
new_post.touch()
|
||||
new_posts.append(new_post)
|
||||
return new_posts
|
||||
|
||||
def delete(post):
|
||||
acc = post.author
|
||||
app = MisskeyApp.query.get(post.misskey_instance)
|
||||
session = make_session()
|
||||
if not app:
|
||||
# how? if this happens, it doesnt make sense to repeat it,
|
||||
# so use a permanent error
|
||||
raise PermanentError("instance not registered for delete")
|
||||
|
||||
r = session.post('{}://{}/api/notes/delete'.format(app.protocol, app.instance), json = {
|
||||
'i': acc.tokens[0].token,
|
||||
'noteId': post.misskey_id
|
||||
})
|
||||
|
||||
if r.status_code != 204:
|
||||
raise TemporaryError("{} {}".format(r.status_code, r.text))
|
||||
|
||||
db.session.delete(post)
|
||||
|
||||
def suggested_instances(limit=5, min_popularity=5, blocklist=tuple()):
|
||||
return tuple((ins.instance for ins in (
|
||||
MisskeyInstance.query
|
||||
.filter(MisskeyInstance.popularity > min_popularity)
|
||||
.filter(~MisskeyInstance.instance.in_(blocklist))
|
||||
.order_by(db.desc(MisskeyInstance.popularity),
|
||||
MisskeyInstance.instance)
|
||||
.limit(limit).all())))
|
|
@ -115,10 +115,6 @@ def post_from_api_tweet_object(tweet, post=None):
|
|||
if 'entities' in tweet:
|
||||
post.has_media = bool(
|
||||
'media' in tweet['entities'] and tweet['entities']['media'])
|
||||
if 'favorite_count' in tweet:
|
||||
post.favourites = tweet['favorite_count']
|
||||
if 'retweet_count' in tweet:
|
||||
post.reblogs = tweet['retweet_count']
|
||||
post.is_reblog = 'retweeted_status' in tweet
|
||||
return post
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
from app import app
|
||||
|
||||
def url_for_version(ver):
|
||||
return app.config['COMMIT_URL'].format(hash=ver['full-revisionid'])
|
||||
return app.config['CHANGELOG_URL'].format(hash=ver['full-revisionid'])
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
"""add misskey
|
||||
|
||||
Revision ID: 740fe24a7712
|
||||
Revises: af763dccc0b4
|
||||
Create Date: 2021-11-10 00:13:37.344364
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '740fe24a7712'
|
||||
down_revision = 'af763dccc0b4'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table('misskey_instances',
|
||||
sa.Column('instance', sa.String(), nullable=False),
|
||||
sa.Column('popularity', sa.Float(), server_default='10', nullable=False),
|
||||
sa.PrimaryKeyConstraint('instance', name=op.f('pk_misskey_instances'))
|
||||
)
|
||||
op.execute("""
|
||||
INSERT INTO misskey_instances (instance, popularity) VALUES
|
||||
('misskey.io', 100),
|
||||
('cliq.social', 60),
|
||||
('misskey.dev', 50),
|
||||
('quietplace.xyz', 40),
|
||||
('mk.nixnet.social', 30),
|
||||
('jigglypuff.club', 20);
|
||||
""")
|
||||
|
||||
op.create_table('misskey_apps',
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('instance', sa.String(), nullable=False),
|
||||
sa.Column('client_secret', sa.String(), nullable=False),
|
||||
sa.Column('protocol', sa.Enum('http', 'https', name='enum_protocol_misskey'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('instance', name=op.f('pk_misskey_apps'))
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('misskey_instances')
|
||||
op.drop_table('misskey_apps')
|
||||
op.execute('DROP TYPE enum_protocol_misskey;')
|
|
@ -0,0 +1,64 @@
|
|||
"""remove fetch batch fk
|
||||
|
||||
things are real bad if the associated post is deleted and this is nulled
|
||||
keeping an opaque ID and associated date should work fine
|
||||
see GH-584
|
||||
|
||||
Revision ID: 7b0e9b8e0887
|
||||
Revises: 740fe24a7712
|
||||
Create Date: 2022-02-27 11:48:55.107299
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '7b0e9b8e0887'
|
||||
down_revision = '740fe24a7712'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.drop_constraint('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', type_='foreignkey')
|
||||
op.add_column('accounts', sa.Column('fetch_current_batch_end_date', sa.DateTime(timezone=True), nullable=True))
|
||||
op.execute('''
|
||||
UPDATE accounts SET fetch_current_batch_end_date = posts.created_at
|
||||
FROM posts WHERE accounts.fetch_current_batch_end_id = posts.id;
|
||||
''')
|
||||
|
||||
# update ids from "mastodon:69420@chitter.xyz" format to just "69420"
|
||||
op.execute('''
|
||||
UPDATE accounts SET fetch_current_batch_end_id =
|
||||
split_part(
|
||||
split_part(fetch_current_batch_end_id, ':', 2),
|
||||
'@', 1);
|
||||
''')
|
||||
|
||||
|
||||
|
||||
def downgrade():
|
||||
# converts ids like "69420" back to "mastodon:69420@chitter.xyz"
|
||||
# i sure hope there isn't a mastodon-compatible out there that can have
|
||||
# : or @ in its post ids
|
||||
op.execute('''
|
||||
WITH accounts_exploded_ids AS (
|
||||
SELECT
|
||||
id,
|
||||
split_part(id, ':', 1) || ':' AS service,
|
||||
CASE WHEN position('@' IN id) != 0
|
||||
THEN '@' || split_part(id, @, 2)
|
||||
ELSE ''
|
||||
END as instance
|
||||
FROM accounts
|
||||
)
|
||||
UPDATE accounts SET fetch_current_batch_end_id = e.service || fetch_current_batch_end_id || e.instance
|
||||
FROM accounts_exploded_ids AS e WHERE e.id = accounts.id AND fetch_current_batch_end_id IS NOT NULL;
|
||||
''')
|
||||
op.execute('''
|
||||
UPDATE accounts SET fetch_current_batch_end_id = NULL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM posts WHERE fetch_current_batch_end_id = posts.id);
|
||||
''')
|
||||
op.create_foreign_key('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', 'posts', ['fetch_current_batch_end_id'], ['id'], ondelete='SET NULL')
|
||||
op.drop_column('accounts', 'fetch_current_batch_end_date')
|
|
@ -0,0 +1,26 @@
|
|||
"""remove favourite, reblog count from posts
|
||||
|
||||
Revision ID: af763dccc0b4
|
||||
Revises: 4b56cde3ebd7
|
||||
Create Date: 2021-05-14 19:45:37.429645
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'af763dccc0b4'
|
||||
down_revision = '4b56cde3ebd7'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.drop_column('posts', 'reblogs')
|
||||
op.drop_column('posts', 'favourites')
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.add_column('posts', sa.Column('favourites', sa.Integer(), nullable=True))
|
||||
op.add_column('posts', sa.Column('reblogs', sa.Integer(), nullable=True))
|
65
model.py
65
model.py
|
@ -67,6 +67,34 @@ class RemoteIDMixin(object):
|
|||
@mastodon_id.setter
|
||||
def mastodon_id(self, id_):
|
||||
self.id = "mastodon:{}@{}".format(id_, self.mastodon_instance)
|
||||
|
||||
@property
|
||||
def misskey_instance(self):
|
||||
if not self.id:
|
||||
return None
|
||||
if self.service != "misskey":
|
||||
raise Exception(
|
||||
"tried to get misskey instance for a {} {}"
|
||||
.format(self.service, type(self)))
|
||||
return self.id.split(":", 1)[1].split('@')[1]
|
||||
|
||||
@misskey_instance.setter
|
||||
def misskey_instance(self, instance):
|
||||
self.id = "misskey:{}@{}".format(self.misskey_id, instance)
|
||||
|
||||
@property
|
||||
def misskey_id(self):
|
||||
if not self.id:
|
||||
return None
|
||||
if self.service != "misskey":
|
||||
raise Exception(
|
||||
"tried to get misskey id for a {} {}"
|
||||
.format(self.service, type(self)))
|
||||
return self.id.split(":", 1)[1].split('@')[0]
|
||||
|
||||
@misskey_id.setter
|
||||
def misskey_id(self, id_):
|
||||
self.id = "misskey:{}@{}".format(id_, self.misskey_instance)
|
||||
|
||||
@property
|
||||
def remote_id(self):
|
||||
|
@ -74,6 +102,8 @@ class RemoteIDMixin(object):
|
|||
return self.twitter_id
|
||||
elif self.service == 'mastodon':
|
||||
return self.mastodon_id
|
||||
elif self.service == 'misskey':
|
||||
return self.misskey_id
|
||||
|
||||
|
||||
ThreeWayPolicyEnum = db.Enum('keeponly', 'deleteonly', 'none',
|
||||
|
@ -116,15 +146,8 @@ class Account(TimestampMixin, RemoteIDMixin):
|
|||
fetch_history_complete = db.Column(db.Boolean, server_default='FALSE',
|
||||
nullable=False)
|
||||
|
||||
@declared_attr
|
||||
def fetch_current_batch_end_id(cls):
|
||||
return db.Column(db.String, db.ForeignKey('posts.id', ondelete='SET NULL'))
|
||||
@declared_attr
|
||||
def fetch_current_batch_end(cls):
|
||||
return db.relationship("Post", foreign_keys=(cls.fetch_current_batch_end_id,))
|
||||
# the declared_attr is necessary because of the foreign key
|
||||
# and because this class is technically one big mixin
|
||||
# https://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/mixins.html#mixing-in-relationships
|
||||
fetch_current_batch_end_id = db.Column(db.String)
|
||||
fetch_current_batch_end_date = db.Column(db.DateTime(timezone=True))
|
||||
|
||||
reason = db.Column(db.String)
|
||||
dormant = db.Column(db.Boolean, server_default='FALSE', nullable=False)
|
||||
|
@ -302,9 +325,6 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin):
|
|||
has_media = db.Column(db.Boolean, server_default='FALSE', nullable=False)
|
||||
direct = db.Column(db.Boolean, server_default='FALSE', nullable=False)
|
||||
|
||||
favourites = db.Column(db.Integer)
|
||||
reblogs = db.Column(db.Integer)
|
||||
|
||||
is_reblog = db.Column(db.Boolean, server_default='FALSE', nullable=False)
|
||||
|
||||
def __str__(self):
|
||||
|
@ -367,3 +387,24 @@ class MastodonInstance(db.Model):
|
|||
|
||||
def bump(self, value=1):
|
||||
self.popularity = (self.popularity or 10) + value
|
||||
|
||||
class MisskeyApp(db.Model, TimestampMixin):
|
||||
__tablename__ = 'misskey_apps'
|
||||
|
||||
instance = db.Column(db.String, primary_key=True)
|
||||
protocol = db.Column(db.String, nullable=False)
|
||||
client_secret = db.Column(db.String, nullable=False)
|
||||
|
||||
class MisskeyInstance(db.Model):
|
||||
"""
|
||||
this is for the autocomplete in the misskey login form
|
||||
it isn't coupled with anything else so that we can seed it with
|
||||
some popular instances ahead of time
|
||||
"""
|
||||
__tablename__ = 'misskey_instances'
|
||||
|
||||
instance = db.Column(db.String, primary_key=True)
|
||||
popularity = db.Column(db.Float, server_default='10', nullable=False)
|
||||
|
||||
def bump(self, value=1):
|
||||
self.popularity = (self.popularity or 10) + value
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"devDependencies": {
|
||||
"rollup": "^1.13.1",
|
||||
"rollup-plugin-node-resolve": "^4.2.4",
|
||||
"rollup-plugin-svelte": "^5.0.3",
|
||||
"svelte": "^3.1.0"
|
||||
"rollup": "^2.42.4",
|
||||
"rollup-plugin-node-resolve": "^5.2.0",
|
||||
"rollup-plugin-svelte": "^7.1.0",
|
||||
"svelte": "^3.35.0"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,2 +0,0 @@
|
|||
[pytest]
|
||||
redis_port = 15487
|
|
@ -1,27 +1,73 @@
|
|||
-i https://pypi.python.org/simple/
|
||||
atomicwrites==1.3.0
|
||||
attrs==19.1.0
|
||||
certifi==2019.3.9
|
||||
chardet==3.0.4
|
||||
codecov==2.0.15
|
||||
coverage==4.5.3
|
||||
idna==2.8
|
||||
importlib-metadata==0.17
|
||||
mirakuru==1.1.0
|
||||
more-itertools==7.0.0 ; python_version > '2.7'
|
||||
packaging==19.0
|
||||
pluggy==0.12.0
|
||||
port-for==0.4
|
||||
psutil==5.6.2
|
||||
py==1.8.0
|
||||
pyparsing==2.4.0
|
||||
pytest-cov==2.7.1
|
||||
pytest-redis==1.3.2
|
||||
pytest==4.6.1
|
||||
redis==3.2.1
|
||||
requests==2.22.0
|
||||
six==1.12.0
|
||||
urllib3==1.25.3
|
||||
versioneer==0.18
|
||||
wcwidth==0.1.7
|
||||
zipp==0.5.1
|
||||
#
|
||||
# These requirements were autogenerated by pipenv
|
||||
# To regenerate from the project's Pipfile, run:
|
||||
#
|
||||
# pipenv lock --requirements --dev
|
||||
#
|
||||
|
||||
# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
|
||||
# requirements. To emit only development requirements, pass "--dev-only".
|
||||
|
||||
-i https://pypi.python.org/simple
|
||||
alembic==1.7.6
|
||||
amqp==5.0.9; python_version >= '3.6'
|
||||
attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
billiard==3.6.4.0
|
||||
blinker==1.4
|
||||
blurhash==1.1.4
|
||||
brotli==1.0.9
|
||||
celery==5.2.3
|
||||
certifi==2021.10.8
|
||||
charset-normalizer==2.0.12; python_version >= '3'
|
||||
click-didyoumean==0.3.0; python_version < '4' and python_full_version >= '3.6.2'
|
||||
click-plugins==1.1.1
|
||||
click-repl==0.2.0
|
||||
click==8.0.4; python_version >= '3.6'
|
||||
cloudpickle==2.0.0; python_version >= '3.6'
|
||||
codecov==2.1.12
|
||||
coverage==6.3.2
|
||||
csscompressor==0.9.5
|
||||
decorator==5.1.1; python_version >= '3.5'
|
||||
deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
doit==0.34.2
|
||||
flask-migrate==3.1.0
|
||||
flask-sqlalchemy==2.5.1
|
||||
flask==2.0.3
|
||||
greenlet==1.1.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
|
||||
gunicorn==20.1.0
|
||||
honcho==1.1.0
|
||||
idna==3.3; python_version >= '3'
|
||||
iniconfig==1.1.1
|
||||
itsdangerous==2.1.0; python_version >= '3.7'
|
||||
jinja2==3.0.3; python_version >= '3.6'
|
||||
kombu==5.2.3; python_version >= '3.7'
|
||||
mako==1.1.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
markupsafe==2.1.0; python_version >= '3.7'
|
||||
mastodon.py==1.5.1
|
||||
packaging==21.3; python_version >= '3.6'
|
||||
pillow==9.0.1
|
||||
pluggy==1.0.0; python_version >= '3.6'
|
||||
prompt-toolkit==3.0.28; python_full_version >= '3.6.2'
|
||||
psycopg2==2.9.3
|
||||
py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pyinotify==0.9.6; sys_platform == 'linux'
|
||||
pyparsing==3.0.7; python_version >= '3.6'
|
||||
pytest-cov==3.0.0
|
||||
pytest==7.0.1
|
||||
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
python-magic==0.4.25; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pytz==2021.3
|
||||
raven==6.10.0
|
||||
redis==4.1.4
|
||||
requests==2.27.1
|
||||
setuptools==59.6.0; python_version >= '3.6'
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
sqlalchemy==1.4.31
|
||||
tomli==2.0.1; python_version >= '3.7'
|
||||
twitter==1.19.3
|
||||
urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
|
||||
versioneer==0.21
|
||||
vine==5.0.0; python_version >= '3.6'
|
||||
wcwidth==0.2.5
|
||||
werkzeug==2.0.3; python_version >= '3.6'
|
||||
wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
|
|
|
@ -1,43 +1,60 @@
|
|||
-i https://pypi.python.org/simple/
|
||||
alembic==1.0.10
|
||||
amqp==2.5.0
|
||||
billiard==3.6.0.0
|
||||
#
|
||||
# These requirements were autogenerated by pipenv
|
||||
# To regenerate from the project's Pipfile, run:
|
||||
#
|
||||
# pipenv lock --requirements
|
||||
#
|
||||
|
||||
-i https://pypi.python.org/simple
|
||||
alembic==1.7.6
|
||||
amqp==5.0.9; python_version >= '3.6'
|
||||
billiard==3.6.4.0
|
||||
blinker==1.4
|
||||
blurhash==1.1.3
|
||||
brotli==1.0.7
|
||||
celery==4.3.0
|
||||
certifi==2019.3.9
|
||||
chardet==3.0.4
|
||||
click==7.0
|
||||
cloudpickle==1.1.1
|
||||
blurhash==1.1.4
|
||||
brotli==1.0.9
|
||||
celery==5.2.3
|
||||
certifi==2021.10.8
|
||||
charset-normalizer==2.0.12; python_version >= '3'
|
||||
click-didyoumean==0.3.0; python_version < '4' and python_full_version >= '3.6.2'
|
||||
click-plugins==1.1.1
|
||||
click-repl==0.2.0
|
||||
click==8.0.4; python_version >= '3.6'
|
||||
cloudpickle==2.0.0; python_version >= '3.6'
|
||||
csscompressor==0.9.5
|
||||
decorator==4.4.0
|
||||
doit==0.31.1
|
||||
flask-migrate==2.5.2
|
||||
flask-sqlalchemy==2.4.0
|
||||
flask==1.0.3
|
||||
gunicorn==19.9.0
|
||||
honcho==1.0.1
|
||||
idna==2.8
|
||||
itsdangerous==1.1.0
|
||||
jinja2==2.10.1
|
||||
kombu==4.6.0
|
||||
mako==1.0.11
|
||||
markupsafe==1.1.1
|
||||
mastodon.py==1.4.3
|
||||
pillow==6.0.0
|
||||
psycopg2==2.8.2
|
||||
pyinotify==0.9.6 ; sys_platform == 'linux'
|
||||
python-dateutil==2.8.0
|
||||
python-editor==1.0.4
|
||||
python-magic==0.4.15
|
||||
pytz==2019.1
|
||||
decorator==5.1.1; python_version >= '3.5'
|
||||
deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
doit==0.34.2
|
||||
flask-migrate==3.1.0
|
||||
flask-sqlalchemy==2.5.1
|
||||
flask==2.0.3
|
||||
greenlet==1.1.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
|
||||
gunicorn==20.1.0
|
||||
honcho==1.1.0
|
||||
idna==3.3; python_version >= '3'
|
||||
itsdangerous==2.1.0; python_version >= '3.7'
|
||||
jinja2==3.0.3; python_version >= '3.6'
|
||||
kombu==5.2.3; python_version >= '3.7'
|
||||
mako==1.1.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
markupsafe==2.1.0; python_version >= '3.7'
|
||||
mastodon.py==1.5.1
|
||||
packaging==21.3; python_version >= '3.6'
|
||||
pillow==9.0.1
|
||||
prompt-toolkit==3.0.28; python_full_version >= '3.6.2'
|
||||
psycopg2==2.9.3
|
||||
pyinotify==0.9.6; sys_platform == 'linux'
|
||||
pyparsing==3.0.7; python_version >= '3.6'
|
||||
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
python-magic==0.4.25; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pytz==2021.3
|
||||
raven==6.10.0
|
||||
redis==3.2.1
|
||||
requests==2.22.0
|
||||
six==1.12.0
|
||||
sqlalchemy==1.3.3
|
||||
twitter==1.18.0
|
||||
urllib3==1.25.3
|
||||
vine==1.3.0
|
||||
werkzeug==0.15.4
|
||||
redis==4.1.4
|
||||
requests==2.27.1
|
||||
setuptools==59.6.0; python_version >= '3.6'
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
sqlalchemy==1.4.31
|
||||
twitter==1.19.3
|
||||
urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
|
||||
vine==5.0.0; python_version >= '3.6'
|
||||
wcwidth==0.2.5
|
||||
werkzeug==2.0.3; python_version >= '3.6'
|
||||
wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
|
|
|
@ -7,8 +7,10 @@ export default {
|
|||
},
|
||||
plugins: [
|
||||
svelte({
|
||||
extensions: ['.html'],
|
||||
include: 'components/**/*.html',
|
||||
hydratable: true,
|
||||
compilerOptions: {hydratable: true},
|
||||
emitCss: false,
|
||||
}),
|
||||
node_resolve(),
|
||||
]
|
||||
|
|
|
@ -3,13 +3,16 @@ from flask import render_template, url_for, redirect, request, g,\
|
|||
from datetime import datetime, timedelta, timezone
|
||||
import libforget.twitter
|
||||
import libforget.mastodon
|
||||
import libforget.misskey
|
||||
from libforget.auth import require_auth, csrf,\
|
||||
get_viewer
|
||||
from model import Session, TwitterArchive, MastodonApp
|
||||
from libforget.session import make_session
|
||||
from model import Session, TwitterArchive, MastodonApp, MisskeyApp
|
||||
from app import app, db, sentry, imgproxy
|
||||
import tasks
|
||||
from zipfile import BadZipFile
|
||||
from twitter import TwitterError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.error import URLError
|
||||
import libforget.version
|
||||
import libforget.settings
|
||||
|
@ -34,10 +37,13 @@ def index():
|
|||
|
||||
@app.route('/about/')
|
||||
def about():
|
||||
instances = libforget.mastodon.suggested_instances()
|
||||
blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
|
||||
mastodon_instances = libforget.mastodon.suggested_instances(blocklist=blocklist)
|
||||
misskey_instances = libforget.misskey.suggested_instances(blocklist=blocklist)
|
||||
return render_template(
|
||||
'about.html',
|
||||
mastodon_instances=instances,
|
||||
mastodon_instances=mastodon_instances,
|
||||
misskey_instances=misskey_instances,
|
||||
twitter_login_error='twitter_login_error' in request.args)
|
||||
|
||||
|
||||
|
@ -93,38 +99,9 @@ def twitter_login_step2():
|
|||
url_for('about', twitter_login_error='', _anchor='log_in'))
|
||||
|
||||
|
||||
class TweetArchiveEmptyException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@app.route('/upload_tweet_archive', methods=('POST',))
|
||||
@require_auth
|
||||
def upload_tweet_archive():
|
||||
ta = TwitterArchive(
|
||||
account=g.viewer.account,
|
||||
body=request.files['file'].read())
|
||||
db.session.add(ta)
|
||||
db.session.commit()
|
||||
|
||||
try:
|
||||
files = libforget.twitter.chunk_twitter_archive(ta.id)
|
||||
|
||||
ta.chunks = len(files)
|
||||
db.session.commit()
|
||||
|
||||
if not ta.chunks > 0:
|
||||
raise TweetArchiveEmptyException()
|
||||
|
||||
for filename in files:
|
||||
tasks.import_twitter_archive_month.s(ta.id, filename).apply_async()
|
||||
|
||||
return redirect(url_for('index', _anchor='recent_archives'))
|
||||
except (BadZipFile, TweetArchiveEmptyException):
|
||||
if sentry:
|
||||
sentry.captureException()
|
||||
return redirect(
|
||||
url_for('index', tweet_archive_failed='',
|
||||
_anchor='tweet_archive_import'))
|
||||
return 403, 'Tweet archive support is temporarily disabled, see banner on the front page.'
|
||||
|
||||
|
||||
@app.route('/settings', methods=('POST',))
|
||||
|
@ -200,6 +177,9 @@ def logout():
|
|||
return redirect(url_for('about'))
|
||||
|
||||
|
||||
def domain_from_url(url):
|
||||
return urlparse(url).netloc.lower() or urlparse("//"+url).netloc.lower()
|
||||
|
||||
@app.route('/login/mastodon', methods=('GET', 'POST'))
|
||||
def mastodon_login_step1(instance=None):
|
||||
|
||||
|
@ -207,9 +187,11 @@ def mastodon_login_step1(instance=None):
|
|||
or request.form.get('instance_url', None))
|
||||
|
||||
if not instance_url:
|
||||
blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
|
||||
instances = libforget.mastodon.suggested_instances(
|
||||
limit=30,
|
||||
min_popularity=1
|
||||
min_popularity=1,
|
||||
blocklist=blocklist,
|
||||
)
|
||||
return render_template(
|
||||
'mastodon_login.html', instances=instances,
|
||||
|
@ -217,28 +199,21 @@ def mastodon_login_step1(instance=None):
|
|||
generic_error='error' in request.args
|
||||
)
|
||||
|
||||
instance_url = instance_url.lower()
|
||||
# strip protocol
|
||||
instance_url = re.sub('^https?://', '', instance_url,
|
||||
count=1, flags=re.IGNORECASE)
|
||||
# strip username
|
||||
instance_url = instance_url.split("@")[-1]
|
||||
# strip trailing path
|
||||
instance_url = instance_url.split('/')[0]
|
||||
instance_url = domain_from_url(instance_url)
|
||||
|
||||
callback = url_for('mastodon_login_step2',
|
||||
instance_url=instance_url, _external=True)
|
||||
|
||||
try:
|
||||
app = libforget.mastodon.get_or_create_app(
|
||||
mastoapp = libforget.mastodon.get_or_create_app(
|
||||
instance_url,
|
||||
callback,
|
||||
url_for('index', _external=True))
|
||||
db.session.merge(app)
|
||||
db.session.merge(mastoapp)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return redirect(libforget.mastodon.login_url(app, callback))
|
||||
return redirect(libforget.mastodon.login_url(mastoapp, callback))
|
||||
|
||||
except Exception:
|
||||
if sentry:
|
||||
|
@ -249,14 +224,77 @@ def mastodon_login_step1(instance=None):
|
|||
@app.route('/login/mastodon/callback/<instance_url>')
|
||||
def mastodon_login_step2(instance_url):
|
||||
code = request.args.get('code', None)
|
||||
app = MastodonApp.query.get(instance_url)
|
||||
if not code or not app:
|
||||
mastoapp = MastodonApp.query.get(instance_url)
|
||||
if not code or not mastoapp:
|
||||
return redirect(url_for('mastodon_login_step1', error=True))
|
||||
|
||||
callback = url_for('mastodon_login_step2',
|
||||
instance_url=instance_url, _external=True)
|
||||
|
||||
token = libforget.mastodon.receive_code(code, app, callback)
|
||||
token = libforget.mastodon.receive_code(code, mastoapp, callback)
|
||||
account = token.account
|
||||
|
||||
session = login(account.id)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
g.viewer = session
|
||||
|
||||
resp = redirect(url_for('index', _anchor='bump_instance'))
|
||||
return resp
|
||||
|
||||
|
||||
@app.route('/login/misskey', methods=('GET', 'POST'))
|
||||
def misskey_login(instance=None):
|
||||
instance_url = (request.args.get('instance_url', None)
|
||||
or request.form.get('instance_url', None))
|
||||
|
||||
if not instance_url:
|
||||
blocklist = app.config.get('HIDDEN_INSTANCES', '').split()
|
||||
instances = libforget.misskey.suggested_instances(
|
||||
limit = 30,
|
||||
min_popularity = 1,
|
||||
blocklist=blocklist,
|
||||
)
|
||||
return render_template(
|
||||
'misskey_login.html', instances=instances,
|
||||
address_error=request.method == 'POST',
|
||||
generic_error='error' in request.args
|
||||
)
|
||||
|
||||
instance_url = domain_from_url(instance_url)
|
||||
|
||||
callback = url_for('misskey_callback',
|
||||
instance_url=instance_url, _external=True)
|
||||
|
||||
try:
|
||||
session = make_session()
|
||||
mkapp = libforget.misskey.get_or_create_app(
|
||||
instance_url,
|
||||
callback,
|
||||
url_for('index', _external=True),
|
||||
session)
|
||||
db.session.merge(mkapp)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return redirect(libforget.misskey.login_url(mkapp, callback, session))
|
||||
|
||||
except Exception:
|
||||
if sentry:
|
||||
sentry.captureException()
|
||||
return redirect(url_for('misskey_login', error=True))
|
||||
|
||||
|
||||
@app.route('/login/misskey/callback/<instance_url>')
|
||||
def misskey_callback(instance_url):
|
||||
# legacy auth and miauth use different parameter names
|
||||
token = request.args.get('token', None) or request.args.get('session', None)
|
||||
mkapp = MisskeyApp.query.get(instance_url)
|
||||
if not token or not mkapp:
|
||||
return redirect(url_for('misskey_login', error=True))
|
||||
|
||||
token = libforget.misskey.receive_token(token, mkapp)
|
||||
account = token.account
|
||||
|
||||
session = login(account.id)
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
from app import app, db
|
||||
from app import app, db, imgproxy
|
||||
from libforget.auth import require_auth_api, get_viewer
|
||||
from flask import jsonify, redirect, make_response, request, Response
|
||||
from model import Account
|
||||
|
@ -6,13 +6,20 @@ import libforget.settings
|
|||
import libforget.json
|
||||
import random
|
||||
|
||||
@app.route('/api/health_check')
|
||||
def health_check():
|
||||
@app.route('/api/health_check') # deprecated 2021-03-12
|
||||
@app.route('/api/status_check')
|
||||
def api_status_check():
|
||||
try:
|
||||
db.session.execute('SELECT 1')
|
||||
return 'ok'
|
||||
except Exception:
|
||||
return ('bad', 500)
|
||||
return ('PostgreSQL bad', 500)
|
||||
|
||||
try:
|
||||
imgproxy.redis.set('forget-status-check', 'howdy', ex=5)
|
||||
except Exception:
|
||||
return ('Redis bad', 500)
|
||||
|
||||
return 'OK'
|
||||
|
||||
|
||||
@app.route('/api/settings', methods=('PUT',))
|
||||
|
@ -60,22 +67,3 @@ def users_badge():
|
|||
return redirect(
|
||||
"https://img.shields.io/badge/active%20users-{}-blue.svg"
|
||||
.format(count))
|
||||
|
||||
|
||||
@app.route('/api/known_instances', methods=('GET', 'DELETE'))
|
||||
def known_instances():
|
||||
if request.method == 'GET':
|
||||
known = request.cookies.get('forget_known_instances', '')
|
||||
if not known:
|
||||
return Response('[]', 404, mimetype='application/json')
|
||||
|
||||
# pad to avoid oracle attacks
|
||||
for _ in range(random.randint(0, 1000)):
|
||||
known += random.choice((' ', '\t', '\n'))
|
||||
|
||||
return Response(known, mimetype='application/json')
|
||||
|
||||
elif request.method == 'DELETE':
|
||||
resp = Response('', 204)
|
||||
resp.set_cookie('forget_known_instances', '', max_age=0)
|
||||
return resp
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
from app import app, db, sentry
|
||||
from flask import g, render_template, make_response, redirect
|
||||
from flask import g, render_template, make_response, redirect, request
|
||||
import version
|
||||
import libforget.version
|
||||
from libforget.auth import get_viewer_session, set_session_cookie
|
||||
|
@ -48,6 +48,8 @@ def not_found(e):
|
|||
|
||||
@app.errorhandler(500)
|
||||
def internal_server_error(e):
|
||||
if request.endpoint and request.endpoint.startswith('api_'):
|
||||
return e.get_response()
|
||||
return (render_template('500.html', e=e), 500)
|
||||
|
||||
|
||||
|
|
91
tasks.py
91
tasks.py
|
@ -2,10 +2,12 @@ from celery import Celery, Task
|
|||
from app import app as flaskapp
|
||||
from app import db
|
||||
from model import Session, Account, TwitterArchive, Post, OAuthToken,\
|
||||
MastodonInstance
|
||||
MastodonInstance, MisskeyInstance
|
||||
import libforget.twitter
|
||||
import libforget.mastodon
|
||||
import libforget.misskey
|
||||
from datetime import timedelta, datetime, timezone
|
||||
from time import time
|
||||
from zipfile import ZipFile
|
||||
from io import BytesIO, TextIOWrapper
|
||||
import json
|
||||
|
@ -15,11 +17,13 @@ from libforget.exceptions import PermanentError, TemporaryError
|
|||
import redis
|
||||
from functools import wraps
|
||||
import pickle
|
||||
import logging
|
||||
|
||||
app = Celery(
|
||||
'tasks',
|
||||
broker=flaskapp.config['CELERY_BROKER'],
|
||||
task_serializer='pickle',
|
||||
accept_content={'pickle',},
|
||||
task_soft_time_limit=600,
|
||||
task_time_limit=1200,
|
||||
)
|
||||
|
@ -64,14 +68,18 @@ def unique(fun):
|
|||
|
||||
@wraps(fun)
|
||||
def wrapper(*args, **kwargs):
|
||||
key = 'celery_unique_lock:{}'.format(
|
||||
pickle.dumps((fun.__name__, args, kwargs)))
|
||||
logging.info('Checking for dupes for unique task %s', (fun.__name__, args, kwargs))
|
||||
key = 'celery_unique_lock:epoch1:{}:{}'.format(
|
||||
fun.__name__, pickle.dumps((args, kwargs)))
|
||||
has_lock = False
|
||||
result = None
|
||||
try:
|
||||
if r.set(key, 1, nx=True, ex=60 * 5):
|
||||
logging.info('No dupes for unique, running task %s', (fun.__name__, args, kwargs))
|
||||
has_lock = True
|
||||
result = fun(*args, **kwargs)
|
||||
else:
|
||||
logging.info('Unique task has a dupe, skipping %s', (fun.__name__, args, kwargs))
|
||||
finally:
|
||||
if has_lock:
|
||||
r.delete(key)
|
||||
|
@ -112,10 +120,10 @@ def fetch_acc(id_):
|
|||
else:
|
||||
max_id = None
|
||||
since_id = None
|
||||
elif account.fetch_current_batch_end:
|
||||
elif account.fetch_current_batch_end_date:
|
||||
oldest = (db.session.query(Post)
|
||||
.with_parent(account, 'posts')
|
||||
.filter(Post.created_at > account.fetch_current_batch_end.created_at)
|
||||
.filter(Post.created_at > account.fetch_current_batch_end_date)
|
||||
.order_by(db.asc(Post.created_at))
|
||||
.first())
|
||||
# ^ None if this is our first fetch of this batch, otherwise oldest of this batch
|
||||
|
@ -123,7 +131,7 @@ def fetch_acc(id_):
|
|||
max_id = oldest.remote_id
|
||||
else:
|
||||
max_id = None
|
||||
since_id = account.fetch_current_batch_end.remote_id
|
||||
since_id = account.fetch_current_batch_end_id
|
||||
else:
|
||||
# we shouldn't get here unless the user had no posts on the service last time we fetched
|
||||
max_id = None
|
||||
|
@ -145,22 +153,30 @@ def fetch_acc(id_):
|
|||
fetch_posts = libforget.twitter.fetch_posts
|
||||
elif (account.service == 'mastodon'):
|
||||
fetch_posts = libforget.mastodon.fetch_posts
|
||||
elif (account.service == 'misskey'):
|
||||
fetch_posts = libforget.misskey.fetch_posts
|
||||
posts = fetch_posts(account, max_id, since_id)
|
||||
|
||||
if posts is None:
|
||||
# ???
|
||||
raise TemporaryError("Fetching posts went horribly wrong")
|
||||
|
||||
if len(posts) == 0:
|
||||
if (
|
||||
len([post for post in posts if post.remote_id not in (max_id, since_id)])
|
||||
== 0
|
||||
):
|
||||
# if there are no posts other than the edges
|
||||
# we either finished the historic fetch
|
||||
# or we finished the current batch
|
||||
account.fetch_history_complete = True
|
||||
batch_end = (Post.query.with_parent(account, 'posts').order_by(
|
||||
db.desc(Post.created_at)).first())
|
||||
if batch_end:
|
||||
account.fetch_current_batch_end_id = batch_end.id
|
||||
account.fetch_current_batch_end_id = batch_end.remote_id
|
||||
account.fetch_current_batch_end_date = batch_end.created_at
|
||||
else:
|
||||
account.fetch_current_batch_end_id = None
|
||||
account.fetch_current_batch_end_date = None
|
||||
db.session.commit()
|
||||
|
||||
else:
|
||||
|
@ -170,7 +186,8 @@ def fetch_acc(id_):
|
|||
|
||||
if not account.fetch_history_complete:
|
||||
# reschedule immediately if we're still doing the historic fetch
|
||||
fetch_acc.apply_async((id_,))
|
||||
print("{} is not done fetching history, resheduling.".format(account))
|
||||
fetch_acc.apply_async((id_,), countdown=1)
|
||||
|
||||
|
||||
except TemporaryError:
|
||||
|
@ -280,6 +297,10 @@ def delete_from_account(account_id):
|
|||
if refreshed and is_eligible(refreshed[0]):
|
||||
to_delete = refreshed[0]
|
||||
break
|
||||
elif account.service == 'misskey':
|
||||
action = libforget.misskey.delete
|
||||
posts = refresh_posts(posts)
|
||||
to_delete = next(filter(is_eligible, posts), None)
|
||||
|
||||
if to_delete:
|
||||
print("Deleting {}".format(to_delete))
|
||||
|
@ -306,6 +327,8 @@ def refresh_posts(posts):
|
|||
return libforget.twitter.refresh_posts(posts)
|
||||
elif posts[0].service == 'mastodon':
|
||||
return libforget.mastodon.refresh_posts(posts)
|
||||
elif posts[0].service == 'misskey':
|
||||
return libforget.misskey.refresh_posts(posts)
|
||||
|
||||
|
||||
@app.task()
|
||||
|
@ -407,12 +430,22 @@ def queue_deletes():
|
|||
@app.task
|
||||
@unique
|
||||
def refresh_account_with_oldest_post():
|
||||
post = (Post.query.outerjoin(Post.author).join(Account.tokens)
|
||||
.filter(Account.backoff_until < db.func.now())
|
||||
.filter(~Account.dormant).group_by(Post).order_by(
|
||||
db.asc(Post.updated_at)).first())
|
||||
then = time()
|
||||
post = db.session.query(Post).from_statement(db.text("""
|
||||
SELECT posts.id, posts.author_id
|
||||
FROM posts, accounts, oauth_tokens
|
||||
WHERE accounts.id = posts.author_id
|
||||
AND accounts.id = oauth_tokens.account_id
|
||||
AND accounts.backoff_until < now()
|
||||
AND NOT accounts.dormant
|
||||
ORDER BY posts.updated_at ASC
|
||||
LIMIT 1;
|
||||
""").columns(Post.id, Post.author_id)).one_or_none()
|
||||
if post:
|
||||
refresh_account(post.author_id)
|
||||
aid = post.author_id
|
||||
refresh_account(aid)
|
||||
now = time()
|
||||
logging.info('Refreshed posts for {} for having oldest post in {}s'.format(aid, now-then))
|
||||
|
||||
|
||||
@app.task
|
||||
|
@ -453,13 +486,41 @@ def update_mastodon_instances_popularity():
|
|||
})
|
||||
db.session.commit()
|
||||
|
||||
@app.task
|
||||
def update_misskey_instances_popularity():
|
||||
# bump score for each active account
|
||||
for acct in (Account.query.options(db.joinedload(Account.sessions))
|
||||
.filter(~Account.dormant).filter(
|
||||
Account.id.like('misskey:%'))):
|
||||
instance = MisskeyInstance.query.get(acct.misskey_instance)
|
||||
if not instance:
|
||||
instance = MisskeyInstance(
|
||||
instance=acct.misskey_instance, popularity=10)
|
||||
db.session.add(instance)
|
||||
amount = 0.01
|
||||
if acct.policy_enabled:
|
||||
amount = 0.5
|
||||
for _ in acct.sessions:
|
||||
amount += 0.1
|
||||
instance.bump(amount / max(1, instance.popularity))
|
||||
|
||||
# normalise scores so the top is 20
|
||||
top_pop = (db.session.query(db.func.max(MisskeyInstance.popularity))
|
||||
.scalar())
|
||||
MisskeyInstance.query.update({
|
||||
MisskeyInstance.popularity:
|
||||
MisskeyInstance.popularity * 20 / top_pop
|
||||
})
|
||||
db.session.commit()
|
||||
|
||||
|
||||
app.add_periodic_task(40, queue_fetch_for_most_stale_accounts)
|
||||
app.add_periodic_task(9, queue_deletes)
|
||||
app.add_periodic_task(25, refresh_account_with_oldest_post)
|
||||
app.add_periodic_task(6, refresh_account_with_oldest_post)
|
||||
app.add_periodic_task(50, refresh_account_with_longest_time_since_refresh)
|
||||
app.add_periodic_task(300, periodic_cleanup)
|
||||
app.add_periodic_task(300, update_mastodon_instances_popularity)
|
||||
app.add_periodic_task(300, update_misskey_instances_popularity)
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.worker_main()
|
||||
|
|
|
@ -4,18 +4,7 @@
|
|||
|
||||
<section>
|
||||
{% include "lib/greet.html" %}
|
||||
<p>Forget is a service that automatically deletes your old posts that everyone has forgotten about. Shouldn't databases forget too?</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Features</h2>
|
||||
<ul>
|
||||
<li>Delete your stale bad posts without even having to look at them again!</li>
|
||||
<li>Set it and <em>forget</em> it. Once you set up an post age limit and/or a post count limit, posts will be considered for deletion as soon as they age past these limits.</li>
|
||||
<li>Choose your pace: delete one post every minute, one post a day, etc...</li>
|
||||
<li>Optionally mark posts that you want to keep, by giving them a favourite.</li>
|
||||
<li>Optionally keep posts with media.</li>
|
||||
</ul>
|
||||
<p>Forget is a service that automatically deletes your old posts. Shouldn't databases forget too?</p>
|
||||
</section>
|
||||
|
||||
{% if not g.viewer %}
|
||||
|
@ -28,7 +17,7 @@
|
|||
{% endif %}
|
||||
|
||||
<p>
|
||||
<a style='background-color:#1da1f2' class='btn primary' href="/login/twitter">
|
||||
<a class='btn primary twitter-colored' href="/login/twitter">
|
||||
{{picture(st, 'twitter', (20,40,80), ('webp', 'png'))}}
|
||||
Log in with Twitter
|
||||
</a>
|
||||
|
@ -37,7 +26,7 @@
|
|||
<p id='mastodon_instance_buttons'>
|
||||
|
||||
{% for instance in mastodon_instances %}
|
||||
<a style='background-color:#282c37' class='btn primary' href="{{ url_for('mastodon_login_step1', instance_url=instance) }}">
|
||||
<a class='btn primary mastodon-colored' href="{{ url_for('mastodon_login_step1', instance_url=instance) }}">
|
||||
{% if loop.first %}
|
||||
{{picture(st, 'mastodon', (20,40,80), ('webp', 'png'))}}
|
||||
Log in with
|
||||
|
@ -45,7 +34,7 @@
|
|||
{{instance}}
|
||||
</a>
|
||||
{% else %}
|
||||
<a style='background-color:#282c37' class='btn primary' href="{{ url_for('mastodon_login_step1') }}">
|
||||
<a class='btn primary mastodon-colored' href="{{ url_for('mastodon_login_step1') }}">
|
||||
{{picture(st, 'mastodon', (20,40,80), ('webp', 'png'))}}
|
||||
Log in with Mastodon
|
||||
</a>
|
||||
|
@ -57,11 +46,37 @@
|
|||
</a>
|
||||
{% endif %}
|
||||
|
||||
</p>
|
||||
|
||||
<p id='misskey_instance_buttons'>
|
||||
{% for instance in misskey_instances %}
|
||||
<a class='btn primary misskey-colored' href="{{ url_for('misskey_login', instance_url=instance) }}">
|
||||
{% if loop.first %}
|
||||
{{picture(st, 'misskey', (20,40,80), ('webp', 'png'))}}
|
||||
Log in with
|
||||
{% endif %}
|
||||
{{instance}}
|
||||
</a>
|
||||
{% else %}
|
||||
<a class='btn primary misskey-colored' href="{{ url_for('misskey_login') }}">
|
||||
{{picture(st, 'misskey', (20,40,80), ('webp', 'png'))}}
|
||||
Log in with Misskey
|
||||
</a>
|
||||
{% endfor %}
|
||||
|
||||
|
||||
{% if misskey_instances %}
|
||||
<a class='btn secondary' href="{{ url_for('misskey_login') }}">
|
||||
Another Misskey instance
|
||||
</a>
|
||||
{% endif %}
|
||||
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<!-- Mastodon -->
|
||||
|
||||
<script type="application/json" id="top_instances">
|
||||
<script type="application/json" id="mastodon_top_instances">
|
||||
[
|
||||
{% for instance in mastodon_instances %}
|
||||
{"instance": "{{instance}}"}
|
||||
|
@ -72,8 +87,8 @@
|
|||
]
|
||||
</script>
|
||||
|
||||
<script type="text/html+template" id="instance_button_template">
|
||||
<a style='background-color:#282c37' class='btn primary'
|
||||
<script type="text/html+template" id="mastodon_instance_button_template">
|
||||
<a class='btn primary mastodon-colored'
|
||||
href="{{ url_for('mastodon_login_step1') }}?instance_url=${encodeURIComponent(instance)}">
|
||||
${ !first? '' : `
|
||||
{{picture(st, 'mastodon', (20,40,80), ('webp', 'png'))}}
|
||||
|
@ -83,14 +98,65 @@
|
|||
</a>
|
||||
</script>
|
||||
|
||||
<script type="text/html+template" id="another_instance_button_template">
|
||||
<script type="text/html+template" id="mastodon_another_instance_button_template">
|
||||
<a class='btn secondary' href="{{ url_for('mastodon_login_step1') }}">
|
||||
Another Mastodon instance
|
||||
</a>
|
||||
</script>
|
||||
|
||||
<!-- Misskey -->
|
||||
|
||||
<script type="application/json" id="misskey_top_instances">
|
||||
[
|
||||
{% for instance in misskey_instances %}
|
||||
{"instance": "{{instance}}"}
|
||||
{%- if not loop.last -%}
|
||||
,
|
||||
{%- endif %}
|
||||
{% endfor %}
|
||||
]
|
||||
</script>
|
||||
|
||||
<script type="text/html+template" id="misskey_instance_button_template">
|
||||
<a class='btn primary misskey-colored'
|
||||
href="{{ url_for('misskey_login') }}?instance_url=${encodeURIComponent(instance)}">
|
||||
${ !first? '' : `
|
||||
{{picture(st, 'misskey', (20,40,80), ('webp', 'png'))}}
|
||||
Log in with
|
||||
`}
|
||||
${ instance }
|
||||
</a>
|
||||
</script>
|
||||
|
||||
<script type="text/html+template" id="misskey_another_instance_button_template">
|
||||
<a class='btn secondary' href="{{ url_for('misskey_login') }}">
|
||||
Another Misskey instance
|
||||
</a>
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
{% endif %}
|
||||
|
||||
<section>
|
||||
<h2>Features</h2>
|
||||
<ul>
|
||||
<li>Delete your posts when they cross an age threshold.</li>
|
||||
<li>Or keep your post count in check, deleting old posts when you go over.</li>
|
||||
<li>Preserve old posts that matter by giving them a favourite or a reaction.</li>
|
||||
<li>Set it and <i>forget</i> it. Forget works continuously in the background.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Non-features</h2>
|
||||
<p>
|
||||
Forget is not a one-time purging tool.
|
||||
It is designed to prune your account continuously, not quickly.
|
||||
If you need a lot of posts gone fast, you may want to look for another more-suited tool.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
|
||||
|
|
|
@ -68,18 +68,19 @@
|
|||
{{interval_input(g.viewer.account, 'policy_keep_younger', scales)}}
|
||||
old and are not one of your
|
||||
<input type=number name=policy_keep_latest min=0 step=1 style='max-width:8ch' value={{g.viewer.account.policy_keep_latest}}>
|
||||
most recent posts will expire
|
||||
most recent posts will be considered for deletion
|
||||
</p>
|
||||
<p>Keep
|
||||
{% if g.viewer.account.service == 'misskey' %}
|
||||
<p>…unless you
|
||||
<span class="radiostrip">
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_favourites value=keeponly id=policy_keep_favourites_keeponly {{ "checked" if g.viewer.account.policy_keep_favourites == 'keeponly' }}>
|
||||
<label for=policy_keep_favourites_keeponly>favourited posts</label>
|
||||
<label for=policy_keep_favourites_keeponly>reacted to them</label>
|
||||
</span>
|
||||
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_favourites value=deleteonly id=policy_keep_favourites_deleteonly {{ "checked" if g.viewer.account.policy_keep_favourites == 'deleteonly' }}>
|
||||
<label for=policy_keep_favourites_deleteonly>non-favourited posts</label>
|
||||
<label for=policy_keep_favourites_deleteonly>have not reacted to them</label>
|
||||
</span>
|
||||
|
||||
<span class="choice">
|
||||
|
@ -88,16 +89,36 @@
|
|||
</span>
|
||||
</span>
|
||||
</p>
|
||||
<p>Keep
|
||||
{%- else %}
|
||||
<p>…unless you
|
||||
<span class="radiostrip">
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_favourites value=keeponly id=policy_keep_favourites_keeponly {{ "checked" if g.viewer.account.policy_keep_favourites == 'keeponly' }}>
|
||||
<label for=policy_keep_favourites_keeponly>favourited them</label>
|
||||
</span>
|
||||
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_favourites value=deleteonly id=policy_keep_favourites_deleteonly {{ "checked" if g.viewer.account.policy_keep_favourites == 'deleteonly' }}>
|
||||
<label for=policy_keep_favourites_deleteonly>have not favourited them</label>
|
||||
</span>
|
||||
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_favourites value=none id=policy_keep_favourites_none {{ "checked" if g.viewer.account.policy_keep_favourites == 'none' }}>
|
||||
<label for=policy_keep_favourites_none>neither</label>
|
||||
</span>
|
||||
</span>
|
||||
</p>
|
||||
{%- endif %}
|
||||
<p>…or unless they
|
||||
<span class="radiostrip">
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_media value=keeponly id=policy_keep_media_keeponly {{ "checked" if g.viewer.account.policy_keep_media == 'keeponly' }}>
|
||||
<label for=policy_keep_media_keeponly>media posts</label>
|
||||
<label for=policy_keep_media_keeponly>have media</label>
|
||||
</span>
|
||||
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_media value=deleteonly id=policy_keep_media_deleteonly {{ "checked" if g.viewer.account.policy_keep_media == 'deleteonly' }}>
|
||||
<label for=policy_keep_media_deleteonly>non-media posts</label>
|
||||
<label for=policy_keep_media_deleteonly>do not have media</label>
|
||||
</span>
|
||||
|
||||
<span class="choice">
|
||||
|
@ -105,8 +126,8 @@
|
|||
<label for=policy_keep_media_none>neither</label>
|
||||
</span>
|
||||
</p>
|
||||
{% if g.viewer.account.service == 'mastodon' %}
|
||||
<p>Keep direct messages
|
||||
{% if g.viewer.account.service == 'mastodon' or g.viewer.account.service == 'misskey' %}
|
||||
<p>Keep direct messages:
|
||||
<span class="radiostrip">
|
||||
<span class="choice">
|
||||
<input type=radio name=policy_keep_direct value=true id=policy_keep_direct_true {{ "checked" if g.viewer.account.policy_keep_direct }}>
|
||||
|
@ -122,7 +143,7 @@
|
|||
{% endif %}
|
||||
<p>Every
|
||||
{{interval_input(g.viewer.account, 'policy_delete_every', scales)}},
|
||||
one expired post will be picked at random and deleted.
|
||||
one post matching these rules will be picked at random and deleted.
|
||||
</p>
|
||||
<input type=submit value='Save settings'>
|
||||
<input type='hidden' name='csrf-token' value='{{g.viewer.csrf_token}}'>
|
||||
|
@ -148,8 +169,10 @@
|
|||
<div class="banner error">The file you uploaded is not a valid tweet archive. No posts have been imported.</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="banner warning">Archive imports are temporarily disabled. The tweet archive format has been quietly dropped by Twitter and replaced by a more comprehensive data archive format. Support for this format will come soon.</div>
|
||||
|
||||
<form action='{{url_for('upload_tweet_archive')}}' method='post' enctype='multipart/form-data' id='archive-form'>
|
||||
<input type="file" name='file' accept='application/zip,.zip'><input type="submit" value="Upload">
|
||||
<input disabled type="file" name='file' accept='application/zip,.zip'><input disabled type="submit" value="Upload">
|
||||
<input type='hidden' name='csrf-token' value='{{g.viewer.csrf_token}}'>
|
||||
</form>
|
||||
|
||||
|
|
|
@ -0,0 +1,29 @@
|
|||
{% extends 'lib/layout.html' %}
|
||||
{% block body %}
|
||||
<section>
|
||||
<h2>Log in with Misskey</h2>
|
||||
|
||||
{% if generic_error %}
|
||||
<div class='banner error'>Something went wrong while logging in. Try again?</div>
|
||||
{% endif %}
|
||||
|
||||
{% if address_error %}
|
||||
<div class='banner error'>This doesn't look like a misskey instance url. Try again?</div>
|
||||
{% endif %}
|
||||
|
||||
<form method='post'>
|
||||
<label>misskey instance:
|
||||
<input type='text' name='instance_url' list='instances' placeholder='social.example.net'/>
|
||||
</label>
|
||||
<datalist id='instances'>
|
||||
<option value=''>
|
||||
{% for instance in instances %}
|
||||
<option value='{{instance}}'>
|
||||
{% endfor %}
|
||||
</datalist>
|
||||
<input name='confirm' value='Log in' type='submit'/>
|
||||
</form>
|
||||
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
|
@ -12,13 +12,12 @@
|
|||
<li>A unique post identifier</li>
|
||||
<li>The post's time and date of publishing</li>
|
||||
<li>Whether the post has any media attached</li>
|
||||
<li>Whether the post has been favourited by you</li>
|
||||
<li>How many favourites and reblogs / retweets the post has</li>
|
||||
<li>(Mastodon only) Whether the post is a direct message</li>
|
||||
<li>Whether the post has been favourited by you (only Twitter or Mastodon); or if (not how) you reacted to the post (Misskey only)</li>
|
||||
<li>Whether the post is a direct message (only Mastodon or Misskey)</li>
|
||||
</ul>
|
||||
<p>No other post metadata and no post contents are stored by Forget.</p>
|
||||
|
||||
<p>Last updated on 2017-12-27. <a href="https://github.com/codl/forget/commits/master/templates/privacy.html">History</a>.</p>
|
||||
<p>Last updated on 2021-11-11. <a href="https://github.com/codl/forget/commits/master/templates/privacy.html">History</a>.</p>
|
||||
</section>
|
||||
|
||||
{% endblock %}
|
||||
|
|
|
@ -7,10 +7,10 @@ TIMEOUT_TARGET = 0.2
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def app(redisdb):
|
||||
def app():
|
||||
from flask import Flask
|
||||
app_ = Flask(__name__)
|
||||
app_.config['REDIS_URI'] = 'redis://localhost:15487'
|
||||
app_.config['REDIS_URI'] = 'redis://localhost:6379'
|
||||
app_.debug = True
|
||||
|
||||
@app_.route('/')
|
||||
|
|
252
version.py
252
version.py
|
@ -6,7 +6,7 @@
|
|||
# that just contains the computed version number.
|
||||
|
||||
# This file is released into the public domain. Generated by
|
||||
# versioneer-0.18 (https://github.com/warner/python-versioneer)
|
||||
# versioneer-0.21 (https://github.com/python-versioneer/python-versioneer)
|
||||
|
||||
"""Git implementation of _version.py."""
|
||||
|
||||
|
@ -15,6 +15,7 @@ import os
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Callable, Dict
|
||||
|
||||
|
||||
def get_keywords():
|
||||
|
@ -52,12 +53,12 @@ class NotThisMethod(Exception):
|
|||
"""Exception raised if a method is not valid for the current scenario."""
|
||||
|
||||
|
||||
LONG_VERSION_PY = {}
|
||||
HANDLERS = {}
|
||||
LONG_VERSION_PY: Dict[str, str] = {}
|
||||
HANDLERS: Dict[str, Dict[str, Callable]] = {}
|
||||
|
||||
|
||||
def register_vcs_handler(vcs, method): # decorator
|
||||
"""Decorator to mark a method as the handler for a particular VCS."""
|
||||
"""Create decorator to mark a method as the handler of a VCS."""
|
||||
def decorate(f):
|
||||
"""Store f in HANDLERS[vcs][method]."""
|
||||
if vcs not in HANDLERS:
|
||||
|
@ -71,17 +72,17 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
|
|||
env=None):
|
||||
"""Call the given command(s)."""
|
||||
assert isinstance(commands, list)
|
||||
p = None
|
||||
for c in commands:
|
||||
process = None
|
||||
for command in commands:
|
||||
try:
|
||||
dispcmd = str([c] + args)
|
||||
dispcmd = str([command] + args)
|
||||
# remember shell=False, so use git.cmd on windows, not just git
|
||||
p = subprocess.Popen([c] + args, cwd=cwd, env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=(subprocess.PIPE if hide_stderr
|
||||
else None))
|
||||
process = subprocess.Popen([command] + args, cwd=cwd, env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=(subprocess.PIPE if hide_stderr
|
||||
else None))
|
||||
break
|
||||
except EnvironmentError:
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
if e.errno == errno.ENOENT:
|
||||
continue
|
||||
|
@ -93,15 +94,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
|
|||
if verbose:
|
||||
print("unable to find command, tried %s" % (commands,))
|
||||
return None, None
|
||||
stdout = p.communicate()[0].strip()
|
||||
if sys.version_info[0] >= 3:
|
||||
stdout = stdout.decode()
|
||||
if p.returncode != 0:
|
||||
stdout = process.communicate()[0].strip().decode()
|
||||
if process.returncode != 0:
|
||||
if verbose:
|
||||
print("unable to run %s (error)" % dispcmd)
|
||||
print("stdout was %s" % stdout)
|
||||
return None, p.returncode
|
||||
return stdout, p.returncode
|
||||
return None, process.returncode
|
||||
return stdout, process.returncode
|
||||
|
||||
|
||||
def versions_from_parentdir(parentdir_prefix, root, verbose):
|
||||
|
@ -113,15 +112,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
|
|||
"""
|
||||
rootdirs = []
|
||||
|
||||
for i in range(3):
|
||||
for _ in range(3):
|
||||
dirname = os.path.basename(root)
|
||||
if dirname.startswith(parentdir_prefix):
|
||||
return {"version": dirname[len(parentdir_prefix):],
|
||||
"full-revisionid": None,
|
||||
"dirty": False, "error": None, "date": None}
|
||||
else:
|
||||
rootdirs.append(root)
|
||||
root = os.path.dirname(root) # up a level
|
||||
rootdirs.append(root)
|
||||
root = os.path.dirname(root) # up a level
|
||||
|
||||
if verbose:
|
||||
print("Tried directories %s but none started with prefix %s" %
|
||||
|
@ -138,22 +136,21 @@ def git_get_keywords(versionfile_abs):
|
|||
# _version.py.
|
||||
keywords = {}
|
||||
try:
|
||||
f = open(versionfile_abs, "r")
|
||||
for line in f.readlines():
|
||||
if line.strip().startswith("git_refnames ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["refnames"] = mo.group(1)
|
||||
if line.strip().startswith("git_full ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["full"] = mo.group(1)
|
||||
if line.strip().startswith("git_date ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["date"] = mo.group(1)
|
||||
f.close()
|
||||
except EnvironmentError:
|
||||
with open(versionfile_abs, "r") as fobj:
|
||||
for line in fobj:
|
||||
if line.strip().startswith("git_refnames ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["refnames"] = mo.group(1)
|
||||
if line.strip().startswith("git_full ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["full"] = mo.group(1)
|
||||
if line.strip().startswith("git_date ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["date"] = mo.group(1)
|
||||
except OSError:
|
||||
pass
|
||||
return keywords
|
||||
|
||||
|
@ -161,10 +158,14 @@ def git_get_keywords(versionfile_abs):
|
|||
@register_vcs_handler("git", "keywords")
|
||||
def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
||||
"""Get version information from git keywords."""
|
||||
if not keywords:
|
||||
raise NotThisMethod("no keywords at all, weird")
|
||||
if "refnames" not in keywords:
|
||||
raise NotThisMethod("Short version file found")
|
||||
date = keywords.get("date")
|
||||
if date is not None:
|
||||
# Use only the last line. Previous lines may contain GPG signature
|
||||
# information.
|
||||
date = date.splitlines()[-1]
|
||||
|
||||
# git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
|
||||
# datestamp. However we prefer "%ci" (which expands to an "ISO-8601
|
||||
# -like" string, which we must then edit to make compliant), because
|
||||
|
@ -177,11 +178,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
|||
if verbose:
|
||||
print("keywords are unexpanded, not using")
|
||||
raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
|
||||
refs = set([r.strip() for r in refnames.strip("()").split(",")])
|
||||
refs = {r.strip() for r in refnames.strip("()").split(",")}
|
||||
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
|
||||
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
|
||||
TAG = "tag: "
|
||||
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
|
||||
tags = {r[len(TAG):] for r in refs if r.startswith(TAG)}
|
||||
if not tags:
|
||||
# Either we're using git < 1.8.3, or there really are no tags. We use
|
||||
# a heuristic: assume all version tags have a digit. The old git %d
|
||||
|
@ -190,7 +191,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
|||
# between branches and tags. By ignoring refnames without digits, we
|
||||
# filter out many common branch names like "release" and
|
||||
# "stabilization", as well as "HEAD" and "master".
|
||||
tags = set([r for r in refs if re.search(r'\d', r)])
|
||||
tags = {r for r in refs if re.search(r'\d', r)}
|
||||
if verbose:
|
||||
print("discarding '%s', no digits" % ",".join(refs - tags))
|
||||
if verbose:
|
||||
|
@ -199,6 +200,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
|||
# sorting will prefer e.g. "2.0" over "2.0rc1"
|
||||
if ref.startswith(tag_prefix):
|
||||
r = ref[len(tag_prefix):]
|
||||
# Filter out refs that exactly match prefix or that don't start
|
||||
# with a number once the prefix is stripped (mostly a concern
|
||||
# when prefix is '')
|
||||
if not re.match(r'\d', r):
|
||||
continue
|
||||
if verbose:
|
||||
print("picking %s" % r)
|
||||
return {"version": r,
|
||||
|
@ -214,7 +220,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
|||
|
||||
|
||||
@register_vcs_handler("git", "pieces_from_vcs")
|
||||
def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
||||
def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command):
|
||||
"""Get version from 'git describe' in the root of the source tree.
|
||||
|
||||
This only gets called if the git-archive 'subst' keywords were *not*
|
||||
|
@ -222,11 +228,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
|||
version string, meaning we're inside a checked out source tree.
|
||||
"""
|
||||
GITS = ["git"]
|
||||
TAG_PREFIX_REGEX = "*"
|
||||
if sys.platform == "win32":
|
||||
GITS = ["git.cmd", "git.exe"]
|
||||
TAG_PREFIX_REGEX = r"\*"
|
||||
|
||||
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
|
||||
hide_stderr=True)
|
||||
_, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root,
|
||||
hide_stderr=True)
|
||||
if rc != 0:
|
||||
if verbose:
|
||||
print("Directory %s not under git control" % root)
|
||||
|
@ -234,15 +242,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
|||
|
||||
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
|
||||
# if there isn't one, this yields HEX[-dirty] (no NUM)
|
||||
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
|
||||
"--always", "--long",
|
||||
"--match", "%s*" % tag_prefix],
|
||||
cwd=root)
|
||||
describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty",
|
||||
"--always", "--long",
|
||||
"--match",
|
||||
"%s%s" % (tag_prefix, TAG_PREFIX_REGEX)],
|
||||
cwd=root)
|
||||
# --long was added in git-1.5.5
|
||||
if describe_out is None:
|
||||
raise NotThisMethod("'git describe' failed")
|
||||
describe_out = describe_out.strip()
|
||||
full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
|
||||
full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root)
|
||||
if full_out is None:
|
||||
raise NotThisMethod("'git rev-parse' failed")
|
||||
full_out = full_out.strip()
|
||||
|
@ -252,6 +261,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
|||
pieces["short"] = full_out[:7] # maybe improved later
|
||||
pieces["error"] = None
|
||||
|
||||
branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=root)
|
||||
# --abbrev-ref was added in git-1.6.3
|
||||
if rc != 0 or branch_name is None:
|
||||
raise NotThisMethod("'git rev-parse --abbrev-ref' returned error")
|
||||
branch_name = branch_name.strip()
|
||||
|
||||
if branch_name == "HEAD":
|
||||
# If we aren't exactly on a branch, pick a branch which represents
|
||||
# the current commit. If all else fails, we are on a branchless
|
||||
# commit.
|
||||
branches, rc = runner(GITS, ["branch", "--contains"], cwd=root)
|
||||
# --contains was added in git-1.5.4
|
||||
if rc != 0 or branches is None:
|
||||
raise NotThisMethod("'git branch --contains' returned error")
|
||||
branches = branches.split("\n")
|
||||
|
||||
# Remove the first line if we're running detached
|
||||
if "(" in branches[0]:
|
||||
branches.pop(0)
|
||||
|
||||
# Strip off the leading "* " from the list of branches.
|
||||
branches = [branch[2:] for branch in branches]
|
||||
if "master" in branches:
|
||||
branch_name = "master"
|
||||
elif not branches:
|
||||
branch_name = None
|
||||
else:
|
||||
# Pick the first branch that is returned. Good or bad.
|
||||
branch_name = branches[0]
|
||||
|
||||
pieces["branch"] = branch_name
|
||||
|
||||
# parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
|
||||
# TAG might have hyphens.
|
||||
git_describe = describe_out
|
||||
|
@ -268,7 +310,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
|||
# TAG-NUM-gHEX
|
||||
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
|
||||
if not mo:
|
||||
# unparseable. Maybe git-describe is misbehaving?
|
||||
# unparsable. Maybe git-describe is misbehaving?
|
||||
pieces["error"] = ("unable to parse git-describe output: '%s'"
|
||||
% describe_out)
|
||||
return pieces
|
||||
|
@ -293,13 +335,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
|||
else:
|
||||
# HEX: no tags
|
||||
pieces["closest-tag"] = None
|
||||
count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
|
||||
cwd=root)
|
||||
count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root)
|
||||
pieces["distance"] = int(count_out) # total number of commits
|
||||
|
||||
# commit date: see ISO-8601 comment in git_versions_from_keywords()
|
||||
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
|
||||
cwd=root)[0].strip()
|
||||
date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip()
|
||||
# Use only the last line. Previous lines may contain GPG signature
|
||||
# information.
|
||||
date = date.splitlines()[-1]
|
||||
pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
|
||||
|
||||
return pieces
|
||||
|
@ -337,19 +380,67 @@ def render_pep440(pieces):
|
|||
return rendered
|
||||
|
||||
|
||||
def render_pep440_pre(pieces):
|
||||
"""TAG[.post.devDISTANCE] -- No -dirty.
|
||||
def render_pep440_branch(pieces):
|
||||
"""TAG[[.dev0]+DISTANCE.gHEX[.dirty]] .
|
||||
|
||||
The ".dev0" means not master branch. Note that .dev0 sorts backwards
|
||||
(a feature branch will appear "older" than the master branch).
|
||||
|
||||
Exceptions:
|
||||
1: no tags. 0.post.devDISTANCE
|
||||
1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty]
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"]:
|
||||
rendered += ".post.dev%d" % pieces["distance"]
|
||||
if pieces["distance"] or pieces["dirty"]:
|
||||
if pieces["branch"] != "master":
|
||||
rendered += ".dev0"
|
||||
rendered += plus_or_dot(pieces)
|
||||
rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dirty"
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0.post.dev%d" % pieces["distance"]
|
||||
rendered = "0"
|
||||
if pieces["branch"] != "master":
|
||||
rendered += ".dev0"
|
||||
rendered += "+untagged.%d.g%s" % (pieces["distance"],
|
||||
pieces["short"])
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dirty"
|
||||
return rendered
|
||||
|
||||
|
||||
def pep440_split_post(ver):
|
||||
"""Split pep440 version string at the post-release segment.
|
||||
|
||||
Returns the release segments before the post-release and the
|
||||
post-release version number (or -1 if no post-release segment is present).
|
||||
"""
|
||||
vc = str.split(ver, ".post")
|
||||
return vc[0], int(vc[1] or 0) if len(vc) == 2 else None
|
||||
|
||||
|
||||
def render_pep440_pre(pieces):
|
||||
"""TAG[.postN.devDISTANCE] -- No -dirty.
|
||||
|
||||
Exceptions:
|
||||
1: no tags. 0.post0.devDISTANCE
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
if pieces["distance"]:
|
||||
# update the post release segment
|
||||
tag_version, post_version = pep440_split_post(pieces["closest-tag"])
|
||||
rendered = tag_version
|
||||
if post_version is not None:
|
||||
rendered += ".post%d.dev%d" % (post_version+1, pieces["distance"])
|
||||
else:
|
||||
rendered += ".post0.dev%d" % (pieces["distance"])
|
||||
else:
|
||||
# no commits, use the tag as the version
|
||||
rendered = pieces["closest-tag"]
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0.post0.dev%d" % pieces["distance"]
|
||||
return rendered
|
||||
|
||||
|
||||
|
@ -380,12 +471,41 @@ def render_pep440_post(pieces):
|
|||
return rendered
|
||||
|
||||
|
||||
def render_pep440_post_branch(pieces):
|
||||
"""TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] .
|
||||
|
||||
The ".dev0" means not master branch.
|
||||
|
||||
Exceptions:
|
||||
1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty]
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"] or pieces["dirty"]:
|
||||
rendered += ".post%d" % pieces["distance"]
|
||||
if pieces["branch"] != "master":
|
||||
rendered += ".dev0"
|
||||
rendered += plus_or_dot(pieces)
|
||||
rendered += "g%s" % pieces["short"]
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dirty"
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0.post%d" % pieces["distance"]
|
||||
if pieces["branch"] != "master":
|
||||
rendered += ".dev0"
|
||||
rendered += "+g%s" % pieces["short"]
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dirty"
|
||||
return rendered
|
||||
|
||||
|
||||
def render_pep440_old(pieces):
|
||||
"""TAG[.postDISTANCE[.dev0]] .
|
||||
|
||||
The ".dev0" means dirty.
|
||||
|
||||
Eexceptions:
|
||||
Exceptions:
|
||||
1: no tags. 0.postDISTANCE[.dev0]
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
|
@ -456,10 +576,14 @@ def render(pieces, style):
|
|||
|
||||
if style == "pep440":
|
||||
rendered = render_pep440(pieces)
|
||||
elif style == "pep440-branch":
|
||||
rendered = render_pep440_branch(pieces)
|
||||
elif style == "pep440-pre":
|
||||
rendered = render_pep440_pre(pieces)
|
||||
elif style == "pep440-post":
|
||||
rendered = render_pep440_post(pieces)
|
||||
elif style == "pep440-post-branch":
|
||||
rendered = render_pep440_post_branch(pieces)
|
||||
elif style == "pep440-old":
|
||||
rendered = render_pep440_old(pieces)
|
||||
elif style == "git-describe":
|
||||
|
@ -495,7 +619,7 @@ def get_versions():
|
|||
# versionfile_source is the relative path from the top of the source
|
||||
# tree (where the .git directory might live) to this file. Invert
|
||||
# this to find the root from __file__.
|
||||
for i in cfg.versionfile_source.split('/'):
|
||||
for _ in cfg.versionfile_source.split('/'):
|
||||
root = os.path.dirname(root)
|
||||
except NameError:
|
||||
return {"version": "0+unknown", "full-revisionid": None,
|
||||
|
|
719
versioneer.py
719
versioneer.py
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue