From 09ce33579aef9e9de3a45a29dff542bab45ef9ce Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 27 Jun 2022 20:55:44 +0200 Subject: [PATCH] Custom emoji support --- app/admin.py | 41 +++-------- app/boxes.py | 2 +- app/config.py | 7 ++ app/main.py | 11 +++ app/scss/main.scss | 2 +- app/source.py | 6 ++ app/static/emoji/.gitignore | 2 + app/static/twemoji/.gitignore | 2 + app/templates.py | 35 ++++++--- app/templates/admin_new.html | 40 ++++++++++ app/utils/emoji.py | 45 ++++++++++++ poetry.lock | 89 ++++++++++++++++++++++- pyproject.toml | 3 + tasks.py | 24 ++++++ tests/factories.py | 5 ++ tests/test_emoji.py | 61 ++++++++++++++++ tests/test_process_outgoing_activities.py | 52 ++++++------- 17 files changed, 357 insertions(+), 70 deletions(-) create mode 100644 app/static/emoji/.gitignore create mode 100644 app/static/twemoji/.gitignore create mode 100644 app/utils/emoji.py create mode 100644 tests/test_emoji.py diff --git a/app/admin.py b/app/admin.py index 84ecf8c..929f93d 100644 --- a/app/admin.py +++ b/app/admin.py @@ -18,6 +18,7 @@ from app.actor import get_actors_metadata from app.boxes import get_inbox_object_by_ap_id from app.boxes import get_outbox_object_by_ap_id from app.boxes import send_follow +from app.config import EMOJIS from app.config import generate_csrf_token from app.config import session_serializer from app.config import verify_csrf_token @@ -25,6 +26,7 @@ from app.config import verify_password from app.database import get_db from app.lookup import lookup from app.uploads import save_upload +from app.utils.emoji import EMOJIS_BY_NAME def user_session_or_redirect( @@ -123,36 +125,11 @@ def admin_new( (v.name, ap.VisibilityEnum.get_display_name(v)) for v in ap.VisibilityEnum ], - }, - ) - - -@router.get("/stream") -def stream( - request: Request, - db: Session = Depends(get_db), -) -> templates.TemplateResponse: - stream = ( - db.query(models.InboxObject) - .filter( - models.InboxObject.ap_type.in_(["Note", "Article", "Video", "Announce"]), - models.InboxObject.is_hidden_from_stream.is_(False), - models.InboxObject.undone_by_inbox_object_id.is_(None), - ) - .options( - # joinedload(models.InboxObject.relates_to_inbox_object), - joinedload(models.InboxObject.relates_to_outbox_object), - ) - .order_by(models.InboxObject.ap_published_at.desc()) - .limit(20) - .all() - ) - return templates.render_template( - db, - request, - "admin_stream.html", - { - "stream": stream, + "emojis": EMOJIS.split(" "), + "custom_emojis": sorted( + [dat for name, dat in EMOJIS_BY_NAME.items()], + key=lambda obj: obj["name"], + ), }, ) @@ -452,7 +429,7 @@ def admin_actions_unpin( @router.post("/actions/new") def admin_actions_new( request: Request, - files: list[UploadFile], + files: list[UploadFile] = [], content: str = Form(), redirect_url: str = Form(), in_reply_to: str | None = Form(None), @@ -501,7 +478,7 @@ def login_validation( if not verify_password(password): raise HTTPException(status_code=401) - resp = RedirectResponse("/admin", status_code=302) + resp = RedirectResponse("/admin/inbox", status_code=302) resp.set_cookie("session", session_serializer.dumps({"is_logged_in": True})) # type: ignore # noqa: E501 return resp diff --git a/app/boxes.py b/app/boxes.py index 1bcff63..1e216f8 100644 --- a/app/boxes.py +++ b/app/boxes.py @@ -341,7 +341,7 @@ def _compute_recipients(db: Session, ap_object: ap.RawObject) -> set[str]: db.query(models.Actor).filter(models.Actor.ap_id == r).one_or_none() ) if known_actor: - recipients.add(known_actor.shared_inbox_url or actor.inbox_url) + recipients.add(known_actor.shared_inbox_url or known_actor.inbox_url) continue # Fetch the object diff --git a/app/config.py b/app/config.py index 15ffbd9..ac65ebd 100644 --- a/app/config.py +++ b/app/config.py @@ -10,6 +10,8 @@ from fastapi import Request from itsdangerous import TimedSerializer from itsdangerous import TimestampSigner +from app.utils.emoji import _load_emojis + ROOT_DIR = Path().parent.resolve() _CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "me.toml") @@ -76,6 +78,11 @@ SQLALCHEMY_DATABASE_URL = CONFIG.sqlalchemy_database_url or f"sqlite:///{DB_PATH KEY_PATH = ( (ROOT_DIR / CONFIG.key_path) if CONFIG.key_path else ROOT_DIR / "data" / "key.pem" ) +EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾" +# Emoji template for the FE +EMOJI_TPL = '{raw}' + +_load_emojis(ROOT_DIR, BASE_URL) session_serializer = TimedSerializer(CONFIG.secret, salt="microblogpub.login") diff --git a/app/main.py b/app/main.py index 2bdd049..8ef8812 100644 --- a/app/main.py +++ b/app/main.py @@ -52,6 +52,7 @@ from app.config import verify_csrf_token from app.database import get_db from app.templates import is_current_user_admin from app.uploads import UPLOAD_DIR +from app.utils.emoji import EMOJIS_BY_NAME from app.webfinger import get_remote_follow_template # TODO(ts): @@ -520,6 +521,16 @@ def tag_by_name( ) +@app.get("/e/{name}") +def emoji_by_name(name: str) -> ActivityPubResponse: + try: + emoji = EMOJIS_BY_NAME[f":{name}:"] + except KeyError: + raise HTTPException(status_code=404) + + return ActivityPubResponse({"@context": ap.AS_CTX, **emoji}) + + @app.post("/inbox") async def inbox( request: Request, diff --git a/app/scss/main.scss b/app/scss/main.scss index 1744b8f..e1d3125 100644 --- a/app/scss/main.scss +++ b/app/scss/main.scss @@ -140,6 +140,6 @@ nav.flexbox { float: right; } } -.custom-emoji { +.emoji, .custom-emoji { max-width: 25px; } diff --git a/app/source.py b/app/source.py index eacf67a..053a16e 100644 --- a/app/source.py +++ b/app/source.py @@ -8,6 +8,7 @@ from app import webfinger from app.actor import Actor from app.actor import fetch_actor from app.config import BASE_URL +from app.utils import emoji def _set_a_attrs(attrs, new=False): @@ -78,5 +79,10 @@ def markdownify( if mentionify: content, mention_tags, mentioned_actors = _mentionify(db, content) tags.extend(mention_tags) + + # Handle custom emoji + tags.extend(emoji.tags(content)) + content = markdown(content, extensions=["mdx_linkify"]) + return content, tags, mentioned_actors diff --git a/app/static/emoji/.gitignore b/app/static/emoji/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/app/static/emoji/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/app/static/twemoji/.gitignore b/app/static/twemoji/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/app/static/twemoji/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/app/templates.py b/app/templates.py index 7d9f529..6739ade 100644 --- a/app/templates.py +++ b/app/templates.py @@ -6,6 +6,7 @@ from typing import Any from urllib.parse import urlparse import bleach +import emoji import html2text import timeago # type: ignore from bs4 import BeautifulSoup # type: ignore @@ -16,6 +17,7 @@ from sqlalchemy.orm import Session from starlette.templating import _TemplateResponse as TemplateResponse from app import activitypub as ap +from app import config from app import models from app.actor import LOCAL_ACTOR from app.ap_object import Attachment @@ -171,14 +173,16 @@ def _update_inline_imgs(content): def _clean_html(html: str, note: Object) -> str: try: - return _replace_custom_emojis( - bleach.clean( - _update_inline_imgs(highlight(html)), - tags=ALLOWED_TAGS, - attributes=ALLOWED_ATTRIBUTES, - strip=True, - ), - note, + return _emojify( + _replace_custom_emojis( + bleach.clean( + _update_inline_imgs(highlight(html)), + tags=ALLOWED_TAGS, + attributes=ALLOWED_ATTRIBUTES, + strip=True, + ), + note, + ) ) except Exception: raise @@ -229,11 +233,24 @@ def _html2text(content: str) -> str: return H2T.handle(content) +def _replace_emoji(u, data): + filename = hex(ord(u))[2:] + return config.EMOJI_TPL.format(filename=filename, raw=u) + + +def _emojify(text: str): + return emoji.replace_emoji( + text, + replace=_replace_emoji, + ) + + _templates.env.filters["domain"] = _filter_domain _templates.env.filters["media_proxy_url"] = _media_proxy_url _templates.env.filters["clean_html"] = _clean_html _templates.env.filters["timeago"] = _timeago _templates.env.filters["format_date"] = _format_date _templates.env.filters["has_media_type"] = _has_media_type -_templates.env.filters["pluralize"] = _pluralize _templates.env.filters["html2text"] = _html2text +_templates.env.filters["emojify"] = _emojify +_templates.env.filters["pluralize"] = _pluralize diff --git a/app/templates/admin_new.html b/app/templates/admin_new.html index 689bb89..faa694f 100644 --- a/app/templates/admin_new.html +++ b/app/templates/admin_new.html @@ -17,6 +17,13 @@ {% endfor %}

+ {% for emoji in emojis %} + {{ emoji | emojify | safe }} + {% endfor %} + {% for emoji in custom_emojis %} + {{ emoji.name }} + {% endfor %} +

@@ -26,5 +33,38 @@

+ {% endblock %} diff --git a/app/utils/emoji.py b/app/utils/emoji.py new file mode 100644 index 0000000..6484788 --- /dev/null +++ b/app/utils/emoji.py @@ -0,0 +1,45 @@ +import mimetypes +import re +import typing +from pathlib import Path + +if typing.TYPE_CHECKING: + from app.activitypub import RawObject + +EMOJI_REGEX = re.compile(r"(:[\d\w]+:)") + +EMOJIS: dict[str, "RawObject"] = {} +EMOJIS_BY_NAME: dict[str, "RawObject"] = {} + + +def _load_emojis(root_dir: Path, base_url: str) -> None: + if EMOJIS: + return + for emoji in (root_dir / "app" / "static" / "emoji").iterdir(): + mt = mimetypes.guess_type(emoji.name)[0] + if mt and mt.startswith("image/"): + name = emoji.name.split(".")[0] + ap_emoji: "RawObject" = { + "type": "Emoji", + "name": f":{name}:", + "updated": "1970-01-01T00:00:00Z", # XXX: we don't track date + "id": f"{base_url}/e/{name}", + "icon": { + "mediaType": mt, + "type": "Image", + "url": f"{base_url}/static/emoji/{emoji.name}", + }, + } + EMOJIS[emoji.name] = ap_emoji + EMOJIS_BY_NAME[ap_emoji["name"]] = ap_emoji + + +def tags(content: str) -> list["RawObject"]: + tags = [] + added = set() + for e in re.findall(EMOJI_REGEX, content): + if e not in added and e in EMOJIS_BY_NAME: + tags.append(EMOJIS_BY_NAME[e]) + added.add(e) + + return tags diff --git a/poetry.lock b/poetry.lock index 7d9640b..24b92fe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -176,6 +176,14 @@ python-versions = "*" [package.dependencies] beautifulsoup4 = "*" +[[package]] +name = "cachetools" +version = "5.2.0" +description = "Extensible memoizing collections and decorators" +category = "main" +optional = false +python-versions = "~=3.7" + [[package]] name = "certifi" version = "2022.5.18.1" @@ -239,6 +247,17 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} [package.extras] development = ["black", "flake8", "mypy", "pytest", "types-colorama"] +[[package]] +name = "emoji" +version = "1.7.0" +description = "Emoji for Python" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["pytest", "coverage", "coveralls"] + [[package]] name = "factory-boy" version = "3.2.1" @@ -308,6 +327,14 @@ mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.8.0,<2.9.0" pyflakes = ">=2.4.0,<2.5.0" +[[package]] +name = "frozendict" +version = "2.3.2" +description = "A simple immutable dictionary" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "greenlet" version = "1.1.2" @@ -725,6 +752,25 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "pyld" +version = "2.0.3" +description = "Python implementation of the JSON-LD API" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +cachetools = "*" +frozendict = "*" +lxml = "*" + +[package.extras] +aiohttp = ["aiohttp"] +cachetools = ["cachetools"] +frozendict = ["frozendict"] +requests = ["requests"] + [[package]] name = "pyparsing" version = "3.0.9" @@ -959,6 +1005,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-emoji" +version = "1.7.2" +description = "Typing stubs for emoji" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-markdown" version = "3.3.28" @@ -1080,7 +1134,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"] [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "2bb47cf626fbd4c87803f8b0362470892a04239b2530676c861401e4352f7483" +content-hash = "e8f20d21a8c7822fbc3c183376d694fc0109e90851377bc6b7316c5c72e880b0" [metadata.files] alembic = [ @@ -1168,6 +1222,10 @@ boussole = [ bs4 = [ {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"}, ] +cachetools = [ + {file = "cachetools-5.2.0-py3-none-any.whl", hash = "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db"}, + {file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"}, +] certifi = [ {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, @@ -1240,6 +1298,9 @@ colorlog = [ {file = "colorlog-6.6.0-py2.py3-none-any.whl", hash = "sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e"}, {file = "colorlog-6.6.0.tar.gz", hash = "sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8"}, ] +emoji = [ + {file = "emoji-1.7.0.tar.gz", hash = "sha256:65c54533ea3c78f30d0729288998715f418d7467de89ec258a31c0ce8660a1d1"}, +] factory-boy = [ {file = "factory_boy-3.2.1-py2.py3-none-any.whl", hash = "sha256:eb02a7dd1b577ef606b75a253b9818e6f9eaf996d94449c9d5ebb124f90dc795"}, {file = "factory_boy-3.2.1.tar.gz", hash = "sha256:a98d277b0c047c75eb6e4ab8508a7f81fb03d2cb21986f627913546ef7a2a55e"}, @@ -1259,6 +1320,25 @@ flake8 = [ {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, ] +frozendict = [ + {file = "frozendict-2.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4fb171d1e84d17335365877e19d17440373b47ca74a73c06f65ac0b16d01e87f"}, + {file = "frozendict-2.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a3640e9d7533d164160b758351aa49d9e85bbe0bd76d219d4021e90ffa6a52"}, + {file = "frozendict-2.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:87cfd00fafbc147d8cd2590d1109b7db8ac8d7d5bdaa708ba46caee132b55d4d"}, + {file = "frozendict-2.3.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fb09761e093cfabb2f179dbfdb2521e1ec5701df714d1eb5c51fa7849027be19"}, + {file = "frozendict-2.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82176dc7adf01cf8f0193e909401939415a230a1853f4a672ec1629a06ceae18"}, + {file = "frozendict-2.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:c1c70826aa4a50fa283fe161834ac4a3ac7c753902c980bb8b595b0998a38ddb"}, + {file = "frozendict-2.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1db5035ddbed995badd1a62c4102b5e207b5aeb24472df2c60aba79639d7996b"}, + {file = "frozendict-2.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4246fc4cb1413645ba4d3513939b90d979a5bae724be605a10b2b26ee12f839c"}, + {file = "frozendict-2.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:680cd42fb0a255da1ce45678ccbd7f69da750d5243809524ebe8f45b2eda6e6b"}, + {file = "frozendict-2.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a7f3a181d6722c92a9fab12d0c5c2b006a18ca5666098531f316d1e1c8984e3"}, + {file = "frozendict-2.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1cb866eabb3c1384a7fe88e1e1033e2b6623073589012ab637c552bf03f6364"}, + {file = "frozendict-2.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:952c5e5e664578c5c2ce8489ee0ab6a1855da02b58ef593ee728fc10d672641a"}, + {file = "frozendict-2.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:608b77904cd0117cd816df605a80d0043a5326ee62529327d2136c792165a823"}, + {file = "frozendict-2.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0eed41fd326f0bcc779837d8d9e1374da1bc9857fe3b9f2910195bbd5fff3aeb"}, + {file = "frozendict-2.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:bde28db6b5868dd3c45b3555f9d1dc5a1cca6d93591502fa5dcecce0dde6a335"}, + {file = "frozendict-2.3.2-py3-none-any.whl", hash = "sha256:6882a9bbe08ab9b5ff96ce11bdff3fe40b114b9813bc6801261e2a7b45e20012"}, + {file = "frozendict-2.3.2.tar.gz", hash = "sha256:7fac4542f0a13fbe704db4942f41ba3abffec5af8b100025973e59dff6a09d0d"}, +] greenlet = [ {file = "greenlet-1.1.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6"}, {file = "greenlet-1.1.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a"}, @@ -1683,6 +1763,9 @@ pygments = [ {file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"}, {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"}, ] +pyld = [ + {file = "PyLD-2.0.3.tar.gz", hash = "sha256:287445f888c3a332ccbd20a14844c66c2fcbaeab3c99acd506a0788e2ebb2f82"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1823,6 +1906,10 @@ types-bleach = [ {file = "types-bleach-5.0.2.tar.gz", hash = "sha256:e1498c512a62117496cf82be3d129972bb89fd1d6482b001cdeb2759ab3c82f5"}, {file = "types_bleach-5.0.2-py3-none-any.whl", hash = "sha256:6fcb75ee4b69190fe60340147b66442cecddaefe3c0629433a4240da1ec2dcf6"}, ] +types-emoji = [ + {file = "types-emoji-1.7.2.tar.gz", hash = "sha256:a7660fb507b30cb80bcec2d01417d828f1258b9b2cd9fa80918e8e5470c5e037"}, + {file = "types_emoji-1.7.2-py3-none-any.whl", hash = "sha256:f4c18bb43e33dc267c650b73d7ae0cd71708c75c79063706d0b91fa9416190c8"}, +] types-markdown = [ {file = "types-Markdown-3.3.28.tar.gz", hash = "sha256:733ba19dad58d5dca1206390f55fa285573535b7c369b94dd367bbc34bf7e4de"}, {file = "types_Markdown-3.3.28-py3-none-any.whl", hash = "sha256:7868cfa3f8a2304d9ecea2ca9b02c14fcb2e34bd26fdbaf01d8c4d362a85d345"}, diff --git a/pyproject.toml b/pyproject.toml index ac8f9ef..98aa982 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ Pillow = "^9.1.1" blurhash-python = "^1.1.3" html2text = "^2020.1.16" feedgen = "^0.9.0" +emoji = "^1.7.0" +PyLD = "^2.0.3" [tool.poetry.dev-dependencies] black = "^22.3.0" @@ -53,6 +55,7 @@ types-Markdown = "^3.3.28" factory-boy = "^3.2.1" pytest-asyncio = "^0.18.3" types-Pillow = "^9.0.20" +types-emoji = "^1.7.2" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tasks.py b/tasks.py index 9c28785..5f709d2 100644 --- a/tasks.py +++ b/tasks.py @@ -1,5 +1,9 @@ +import io +import tarfile +from pathlib import Path from typing import Optional +import httpx from invoke import Context # type: ignore from invoke import run # type: ignore from invoke import task # type: ignore @@ -67,3 +71,23 @@ def tests(ctx, k=None): pty=True, echo=True, ) + + +@task +def download_twemoji(ctx): + # type: (Context) -> None + resp = httpx.get( + "https://github.com/twitter/twemoji/archive/refs/tags/v14.0.2.tar.gz", + follow_redirects=True, + ) + resp.raise_for_status() + tf = tarfile.open(fileobj=io.BytesIO(resp.content)) + members = [ + member + for member in tf.getmembers() + if member.name.startswith("twemoji-14.0.2/assets/svg/") + ] + for member in members: + emoji_name = Path(member.name).name + with open(f"app/static/twemoji/{emoji_name}", "wb") as f: + f.write(tf.extractfile(member).read()) # type: ignore diff --git a/tests/factories.py b/tests/factories.py index b12411b..f4a290f 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -204,3 +204,8 @@ class InboxObjectFactory(factory.alchemy.SQLAlchemyModelFactory): # Hide replies from the stream is_hidden_from_stream=True if ro.in_reply_to else False, ) + + +class FollowerFactory(factory.alchemy.SQLAlchemyModelFactory): + class Meta(BaseModelMeta): + model = models.Follower diff --git a/tests/test_emoji.py b/tests/test_emoji.py new file mode 100644 index 0000000..380f5a8 --- /dev/null +++ b/tests/test_emoji.py @@ -0,0 +1,61 @@ +from fastapi.testclient import TestClient + +from app import activitypub as ap +from app import models +from app.config import generate_csrf_token +from app.database import Session +from app.utils.emoji import EMOJIS_BY_NAME +from tests.utils import generate_admin_session_cookies + + +def test_emoji_are_loaded() -> None: + assert len(EMOJIS_BY_NAME) >= 1 + + +def test_emoji_ap_endpoint(db: Session, client: TestClient) -> None: + response = client.get("/e/goose_hacker", headers={"Accept": ap.AP_CONTENT_TYPE}) + assert response.status_code == 200 + assert response.headers["content-type"] == ap.AP_CONTENT_TYPE + emoji_resp = response.json() + assert emoji_resp["type"] == "Emoji" + + +def test_emoji_ap_endpoint__not_found(db: Session, client: TestClient) -> None: + response = client.get("/e/goose_hacker2", headers={"Accept": ap.AP_CONTENT_TYPE}) + assert response.status_code == 404 + + +def test_emoji_note_with_emoji(db: Session, client: TestClient) -> None: + # Call admin endpoint to create a note with + note_content = "😺 :goose_hacker:" + + response = client.post( + "/admin/actions/new", + data={ + "redirect_url": "http://testserver/", + "content": note_content, + "visibility": ap.VisibilityEnum.PUBLIC.name, + "csrf_token": generate_csrf_token(), + }, + cookies=generate_admin_session_cookies(), + ) + + # Then the server returns a 302 + assert response.status_code == 302 + + # And the Follow activity was created in the outbox + outbox_object = db.query(models.OutboxObject).one() + assert outbox_object.ap_type == "Note" + assert len(outbox_object.tags) == 1 + emoji_tag = outbox_object.tags[0] + assert emoji_tag["type"] == "Emoji" + assert emoji_tag["name"] == ":goose_hacker:" + url = emoji_tag["icon"]["url"] + + # And the custom emoji is rendered in the HTML version + html_resp = client.get("/o/" + outbox_object.public_id) + html_resp.raise_for_status() + assert html_resp.status_code == 200 + assert url in html_resp.text + # And the unicode emoji is rendered with twemoji + assert f'/static/twemoji/{hex(ord("😺"))[2:]}.svg' in html_resp.text diff --git a/tests/test_process_outgoing_activities.py b/tests/test_process_outgoing_activities.py index 7c312cb..a5e5422 100644 --- a/tests/test_process_outgoing_activities.py +++ b/tests/test_process_outgoing_activities.py @@ -120,32 +120,6 @@ def test_process_next_outgoing_activity__error_500( assert outgoing_activity.tries == 1 -def test_process_next_outgoing_activity__connect_error( - db: Session, - respx_mock: respx.MockRouter, -) -> None: - outbox_object = _setup_outbox_object() - recipient_inbox_url = "https://example.com/inbox" - respx_mock.post(recipient_inbox_url).mock(side_effect=httpx.ConnectError) - - # And an outgoing activity - outgoing_activity = factories.OutgoingActivityFactory( - recipient=recipient_inbox_url, - outbox_object_id=outbox_object.id, - ) - - # When processing the next outgoing activity - # Then it is processed - assert process_next_outgoing_activity(db) is True - - assert respx_mock.calls.call_count == 1 - - outgoing_activity = db.query(models.OutgoingActivity).one() - assert outgoing_activity.is_sent is False - assert outgoing_activity.error is not None - assert outgoing_activity.tries == 1 - - def test_process_next_outgoing_activity__errored( db: Session, respx_mock: respx.MockRouter, @@ -179,5 +153,31 @@ def test_process_next_outgoing_activity__errored( assert process_next_outgoing_activity(db) is False +def test_process_next_outgoing_activity__connect_error( + db: Session, + respx_mock: respx.MockRouter, +) -> None: + outbox_object = _setup_outbox_object() + recipient_inbox_url = "https://example.com/inbox" + respx_mock.post(recipient_inbox_url).mock(side_effect=httpx.ConnectError) + + # And an outgoing activity + outgoing_activity = factories.OutgoingActivityFactory( + recipient=recipient_inbox_url, + outbox_object_id=outbox_object.id, + ) + + # When processing the next outgoing activity + # Then it is processed + assert process_next_outgoing_activity(db) is True + + assert respx_mock.calls.call_count == 1 + + outgoing_activity = db.query(models.OutgoingActivity).one() + assert outgoing_activity.is_sent is False + assert outgoing_activity.error is not None + assert outgoing_activity.tries == 1 + + # TODO(ts): # - parse retry after