Remove data/_theme.scss from git history

- Remove data/_theme.scss from git history, to prevent accidental overwrites/commits
- Add a step to the configuration wizard to create a blank file if one doesn't exist
- Move the file existence check in compile-scss to before favicon building (as it errors if the file is missing)

Fixes https://todo.sr.ht/~tsileo/microblog.pub/67

AUTHORS

@@ -0,0 +1,8 @@
+Thomas Sileo <t@a4.io>
+Kevin Wallace <doof@doof.net>
+Miguel Jacq <mig@mig5.net>
+Josh Washburne <josh@jodh.us>
+Alexey Shpakovsky <alexey@shpakovsky.ru>
+Ash McAllan <acegiak@gmail.com>
+Cassio Zen <cassio@hey.com>
+Cocoa <momijizukamori@gmail.com>

Dockerfile

@@ -10,13 +10,18 @@ ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
 
 FROM python-base as builder-base
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends curl build-essential gcc 
+RUN apt-get install -y --no-install-recommends curl build-essential gcc libffi-dev libssl-dev libxml2-dev libxslt1-dev zlib1g-dev libxslt-dev gcc libjpeg-dev zlib1g-dev libwebp-dev
+# rustc is needed to compile Python packages
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
 RUN curl -sSL https://install.python-poetry.org | python3 - 
 WORKDIR $PYSETUP_PATH
 COPY poetry.lock pyproject.toml ./
-RUN poetry install --no-dev
+RUN poetry install --only main
 
 FROM python-base as production
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends libjpeg-dev libxslt1-dev libxml2-dev libxslt-dev
 RUN groupadd --gid 1000 microblogpub \
   && useradd --uid 1000 --gid microblogpub --shell /bin/bash microblogpub
 COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH

Makefile

@@ -9,12 +9,35 @@ build:
 config:
 	# Run and remove instantly
 	-docker run --rm -it --volume `pwd`/data:/app/data microblogpub/microblogpub inv configuration-wizard
-	-docker run --env MICROBLOGPUB_CONFIG_FILE=tests.toml --rm -it --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv configuration-wizard
 
 .PHONY: update
 update:
-	-docker run --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv update
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv update --no-update-deps
 
 .PHONY: prune-old-data
 prune-old-data:
-	-docker run --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv prune-old-data
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv prune-old-data
+
+.PHONY: webfinger
+webfinger:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv webfinger $(account)
+
+.PHONY: move-to
+move-to:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv move-to $(account)
+
+.PHONY: self-destruct
+self-destruct:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv self-destruct
+
+.PHONY: reset-password
+reset-password:
+	-docker run --rm -it --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv reset-password
+
+.PHONY: check-config
+check-config:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv check-config
+
+.PHONY: compile-scss
+compile-scss:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv compile-scss

README.md

@@ -22,7 +22,7 @@ There are still some rough edges, but the server is mostly functional.
     - Author notes in Markdown, with code highlighting support
     - Dedicated section for articles/blog posts (enabled when the first article is posted)
  - Lightweight
-    - Uses SQLite, and no external dependencies except Python 3.10+
+    - Uses SQLite, and Python 3.10+
     - Can be deployed on small VPS
  - Privacy-aware
     - EXIF metadata (like GPS location) are stripped before storage
@@ -58,7 +58,7 @@ All the development takes place on [sourcehut](https://sr.ht/~tsileo/microblog.p
  - [Issue tracker](https://todo.sr.ht/~tsileo/microblog.pub)
  - [Mailing list](https://sr.ht/~tsileo/microblog.pub/lists)
 
-Contributions are welcomed, check out the [documentation](https://docs.microblog.pub) for more details.
+Contributions are welcomed, check out the [contributing section of the documentation](https://docs.microblog.pub/developer_guide.html#contributing) for more details.
 
 
 ## License

alembic/versions/2022_10_30_1409-b28c0551c236_add_a_slug_field_for_outbox_objects.py

@@ -0,0 +1,48 @@
+"""Add a slug field for outbox objects
+
+Revision ID: b28c0551c236
+Revises: 604d125ea2fb
+Create Date: 2022-10-30 14:09:14.540461+00:00
+
+"""
+import sqlalchemy as sa
+from sqlalchemy import select
+from sqlalchemy.orm.session import Session
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'b28c0551c236'
+down_revision = '604d125ea2fb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('outbox', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('slug', sa.String(), nullable=True))
+        batch_op.create_index(batch_op.f('ix_outbox_slug'), ['slug'], unique=False)
+
+    # ### end Alembic commands ###
+
+    # Backfill the slug for existing articles
+    from app.models import OutboxObject
+    from app.utils.text import slugify
+    sess = Session(op.get_bind())
+    articles = sess.execute(select(OutboxObject).where(
+        OutboxObject.ap_type == "Article")
+    ).scalars()
+    for article in articles:
+        title = article.ap_object["name"]
+        article.slug = slugify(title)
+    sess.commit()
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('outbox', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_outbox_slug'))
+        batch_op.drop_column('slug')
+
+    # ### end Alembic commands ###

alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py

@@ -0,0 +1,32 @@
+"""Add Webmention.webmention_type
+
+Revision ID: fadfd359ce78
+Revises: b28c0551c236
+Create Date: 2022-11-16 19:42:56.925512+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'fadfd359ce78'
+down_revision = 'b28c0551c236'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('webmention_type', sa.Enum('UNKNOWN', 'LIKE', 'REPLY', 'REPOST', name='webmentiontype'), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.drop_column('webmention_type')
+
+    # ### end Alembic commands ###

app/activitypub.py

@@ -6,12 +6,15 @@ from typing import Any
 
 import httpx
 from loguru import logger
-from markdown import markdown
 
 from app import config
+from app.config import ALSO_KNOWN_AS
 from app.config import AP_CONTENT_TYPE  # noqa: F401
+from app.config import MOVED_TO
 from app.httpsig import auth
 from app.key import get_pubkey_as_pem
+from app.source import dedup_tags
+from app.source import hashtagify
 from app.utils.url import check_url
 
 if TYPE_CHECKING:
@@ -32,6 +35,7 @@ AS_EXTENDED_CTX = [
         "sensitive": "as:sensitive",
         "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
         "alsoKnownAs": {"@id": "as:alsoKnownAs", "@type": "@id"},
+        "movedTo": {"@id": "as:movedTo", "@type": "@id"},
         # toot
         "toot": "http://joinmastodon.org/ns#",
         "featured": {"@id": "toot:featured", "@type": "@id"},
@@ -49,11 +53,26 @@ AS_EXTENDED_CTX = [
 ]
 
 
-class ObjectIsGoneError(Exception):
+class FetchError(Exception):
+    def __init__(self, url: str, resp: httpx.Response | None = None) -> None:
+        resp_part = ""
+        if resp:
+            resp_part = f", got HTTP {resp.status_code}: {resp.text}"
+        message = f"Failed to fetch {url}{resp_part}"
+        super().__init__(message)
+        self.resp = resp
+        self.url = url
+
+
+class ObjectIsGoneError(FetchError):
     pass
 
 
-class ObjectNotFoundError(Exception):
+class ObjectNotFoundError(FetchError):
+    pass
+
+
+class ObjectUnavailableError(FetchError):
     pass
 
 
@@ -81,6 +100,21 @@ class VisibilityEnum(str, enum.Enum):
         }[key]
 
 
+_LOCAL_ACTOR_SUMMARY, _LOCAL_ACTOR_TAGS = hashtagify(config.CONFIG.summary)
+_LOCAL_ACTOR_METADATA = []
+if config.CONFIG.metadata:
+    for kv in config.CONFIG.metadata:
+        kv_value, kv_tags = hashtagify(kv.value)
+        _LOCAL_ACTOR_METADATA.append(
+            {
+                "name": kv.key,
+                "type": "PropertyValue",
+                "value": kv_value,
+            }
+        )
+        _LOCAL_ACTOR_TAGS.extend(kv_tags)
+
+
 ME = {
     "@context": AS_EXTENDED_CTX,
     "type": "Person",
@@ -92,7 +126,7 @@ ME = {
     "outbox": config.BASE_URL + "/outbox",
     "preferredUsername": config.USERNAME,
     "name": config.CONFIG.name,
-    "summary": config.CONFIG.summary,
+    "summary": _LOCAL_ACTOR_SUMMARY,
     "endpoints": {
         # For compat with servers expecting a sharedInbox...
         "sharedInbox": config.BASE_URL
@@ -100,28 +134,35 @@ ME = {
     },
     "url": config.ID + "/",  # XXX: the path is important for Mastodon compat
     "manuallyApprovesFollowers": config.CONFIG.manually_approves_followers,
-    "attachment": [
+    "attachment": _LOCAL_ACTOR_METADATA,
-        {
-            "name": kv.key,
-            "type": "PropertyValue",
-            "value": markdown(kv.value, extensions=["mdx_linkify", "fenced_code"]),
-        }
-        for kv in config.CONFIG.metadata
-    ]
-    if config.CONFIG.metadata
-    else [],
-    "icon": {
-        "mediaType": mimetypes.guess_type(config.CONFIG.icon_url)[0],
-        "type": "Image",
-        "url": config.CONFIG.icon_url,
-    },
     "publicKey": {
         "id": f"{config.ID}#main-key",
         "owner": config.ID,
         "publicKeyPem": get_pubkey_as_pem(config.KEY_PATH),
     },
+    "tag": dedup_tags(_LOCAL_ACTOR_TAGS),
 }
 
+if config.CONFIG.icon_url:
+    ME["icon"] = {
+        "mediaType": mimetypes.guess_type(config.CONFIG.icon_url)[0],
+        "type": "Image",
+        "url": config.CONFIG.icon_url,
+    }
+
+if ALSO_KNOWN_AS:
+    ME["alsoKnownAs"] = [ALSO_KNOWN_AS]
+
+if MOVED_TO:
+    ME["movedTo"] = MOVED_TO
+
+if config.CONFIG.image_url:
+    ME["image"] = {
+        "mediaType": mimetypes.guess_type(config.CONFIG.image_url)[0],
+        "type": "Image",
+        "url": config.CONFIG.image_url,
+    }
+
 
 class NotAnObjectError(Exception):
     def __init__(self, url: str, resp: httpx.Response | None = None) -> None:
@@ -153,11 +194,17 @@ async def fetch(
 
     # Special handling for deleted object
     if resp.status_code == 410:
-        raise ObjectIsGoneError(f"{url} is gone")
+        raise ObjectIsGoneError(url, resp)
+    elif resp.status_code in [401, 403]:
+        raise ObjectUnavailableError(url, resp)
     elif resp.status_code == 404:
-        raise ObjectNotFoundError(f"{url} not found")
+        raise ObjectNotFoundError(url, resp)
+
+    try:
+        resp.raise_for_status()
+    except httpx.HTTPError as http_error:
+        raise FetchError(url, resp) from http_error
 
-    resp.raise_for_status()
     try:
         return resp.json()
     except json.JSONDecodeError:

app/actor.py

@@ -1,16 +1,21 @@
 import hashlib
 import typing
 from dataclasses import dataclass
+from datetime import timedelta
 from functools import cached_property
 from typing import Union
 from urllib.parse import urlparse
 
+from loguru import logger
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 
 from app import activitypub as ap
 from app import media
+from app.config import BASE_URL
 from app.database import AsyncSession
+from app.utils.datetime import as_utc
+from app.utils.datetime import now
 
 if typing.TYPE_CHECKING:
     from app.models import Actor as ActorModel
@@ -78,11 +83,21 @@ class Actor:
 
     @property
     def icon_url(self) -> str | None:
-        return self.ap_actor.get("icon", {}).get("url")
+        if icon := self.ap_actor.get("icon"):
+            return icon.get("url")
+        return None
 
     @property
     def icon_media_type(self) -> str | None:
-        return self.ap_actor.get("icon", {}).get("mediaType")
+        if icon := self.ap_actor.get("icon"):
+            return icon.get("mediaType")
+        return None
+
+    @property
+    def image_url(self) -> str | None:
+        if image := self.ap_actor.get("image"):
+            return image.get("url")
+        return None
 
     @property
     def public_key_as_pem(self) -> str:
@@ -97,18 +112,18 @@ class Actor:
         if self.icon_url:
             return media.proxied_media_url(self.icon_url)
         else:
-            return "/static/nopic.png"
+            return BASE_URL + "/static/nopic.png"
 
     @property
     def resized_icon_url(self) -> str:
         if self.icon_url:
             return media.resized_media_url(self.icon_url, 50)
         else:
-            return "/static/nopic.png"
+            return BASE_URL + "/static/nopic.png"
 
     @property
     def tags(self) -> list[ap.RawObject]:
-        return self.ap_actor.get("tag", [])
+        return ap.as_list(self.ap_actor.get("tag", []))
 
     @property
     def followers_collection_id(self) -> str | None:
@@ -118,6 +133,10 @@ class Actor:
     def attachments(self) -> list[ap.RawObject]:
         return ap.as_list(self.ap_actor.get("attachment", []))
 
+    @cached_property
+    def moved_to(self) -> str | None:
+        return self.ap_actor.get("movedTo")
+
     @cached_property
     def server(self) -> str:
         return urlparse(self.ap_id).hostname  # type: ignore
@@ -154,9 +173,9 @@ async def save_actor(db_session: AsyncSession, ap_actor: ap.RawObject) -> "Actor
         raise ValueError(f"Invalid type {ap_type} for actor {ap_actor}")
 
     actor = models.Actor(
-        ap_id=ap_actor["id"],
+        ap_id=ap.get_id(ap_actor["id"]),
         ap_actor=ap_actor,
-        ap_type=ap_actor["type"],
+        ap_type=ap.as_list(ap_actor["type"])[0],
         handle=_handle(ap_actor),
     )
     db_session.add(actor)
@@ -184,13 +203,64 @@ async def fetch_actor(
     if existing_actor:
         if existing_actor.is_deleted:
             raise ap.ObjectNotFoundError(f"{actor_id} was deleted")
-        return existing_actor
+
-    else:
+        if now() - as_utc(existing_actor.updated_at) > timedelta(hours=24):
-        if save_if_not_found:
+            logger.info(
-            ap_actor = await ap.fetch(actor_id)
+                f"Refreshing {actor_id=} last updated {existing_actor.updated_at}"
-            return await save_actor(db_session, ap_actor)
+            )
+            try:
+                ap_actor = await ap.fetch(actor_id)
+                await update_actor_if_needed(
+                    db_session,
+                    existing_actor,
+                    RemoteActor(ap_actor),
+                )
+                return existing_actor
+            except Exception:
+                logger.exception(f"Failed to refresh {actor_id}")
+                # If we fail to refresh the actor, return the cached one
+                return existing_actor
         else:
-            raise ap.ObjectNotFoundError
+            return existing_actor
+
+    if save_if_not_found:
+        ap_actor = await ap.fetch(actor_id)
+        # Some softwares uses URL when we expect ID or uses a different casing
+        # (like Birdsite LIVE) , which mean we may already have it in DB
+        existing_actor_by_url = (
+            await db_session.scalars(
+                select(models.Actor).where(
+                    models.Actor.ap_id == ap.get_id(ap_actor),
+                )
+            )
+        ).one_or_none()
+        if existing_actor_by_url:
+            # Update the actor as we had to fetch it anyway
+            await update_actor_if_needed(
+                db_session,
+                existing_actor_by_url,
+                RemoteActor(ap_actor),
+            )
+            return existing_actor_by_url
+
+        return await save_actor(db_session, ap_actor)
+    else:
+        raise ap.ObjectNotFoundError(actor_id)
+
+
+async def update_actor_if_needed(
+    db_session: AsyncSession,
+    actor_in_db: "ActorModel",
+    ra: RemoteActor,
+) -> None:
+    # Check if we actually need to udpte the actor in DB
+    if _actor_hash(ra) != _actor_hash(actor_in_db):
+        actor_in_db.ap_actor = ra.ap_actor
+        actor_in_db.handle = ra.handle
+        actor_in_db.ap_type = ra.ap_type
+
+    actor_in_db.updated_at = now()
+    await db_session.flush()
 
 
 @dataclass
@@ -199,8 +269,11 @@ class ActorMetadata:
     is_following: bool
     is_follower: bool
     is_follow_request_sent: bool
+    is_follow_request_rejected: bool
     outbox_follow_ap_id: str | None
     inbox_follow_ap_id: str | None
+    moved_to: typing.Optional["ActorModel"]
+    has_blocked_local_actor: bool
 
 
 ActorsMetadata = dict[str, ActorMetadata]
@@ -243,17 +316,57 @@ async def get_actors_metadata(
             )
         )
     }
+    rejected_follow_requests = {
+        reject.activity_object_ap_id
+        for reject in await db_session.execute(
+            select(models.InboxObject.activity_object_ap_id).where(
+                models.InboxObject.ap_type == "Reject",
+                models.InboxObject.ap_actor_id.in_(ap_actor_ids),
+            )
+        )
+    }
+    blocks = {
+        block.ap_actor_id
+        for block in await db_session.execute(
+            select(models.InboxObject.ap_actor_id).where(
+                models.InboxObject.ap_type == "Block",
+                models.InboxObject.undone_by_inbox_object_id.is_(None),
+                models.InboxObject.ap_actor_id.in_(ap_actor_ids),
+            )
+        )
+    }
+
     idx: ActorsMetadata = {}
     for actor in actors:
         if not actor.ap_id:
             raise ValueError("Should never happen")
+        moved_to = None
+        if actor.moved_to:
+            try:
+                moved_to = await fetch_actor(
+                    db_session,
+                    actor.moved_to,
+                    save_if_not_found=False,
+                )
+            except ap.ObjectNotFoundError:
+                pass
+            except Exception:
+                logger.exception(f"Failed to fetch {actor.moved_to=}")
+
         idx[actor.ap_id] = ActorMetadata(
             ap_actor_id=actor.ap_id,
             is_following=actor.ap_id in following,
             is_follower=actor.ap_id in followers,
             is_follow_request_sent=actor.ap_id in sent_follow_requests,
+            is_follow_request_rejected=bool(
+                sent_follow_requests[actor.ap_id] in rejected_follow_requests
+            )
+            if actor.ap_id in sent_follow_requests
+            else False,
             outbox_follow_ap_id=sent_follow_requests.get(actor.ap_id),
             inbox_follow_ap_id=followers.get(actor.ap_id),
+            moved_to=moved_to,
+            has_blocked_local_actor=actor.ap_id in blocks,
         )
     return idx
 
@@ -278,6 +391,9 @@ def _actor_hash(actor: Actor) -> bytes:
     if actor.icon_url:
         h.update(actor.icon_url.encode())
 
+    if actor.image_url:
+        h.update(actor.image_url.encode())
+
     if actor.attachments:
         for a in actor.attachments:
             if a.get("type") != "PropertyValue":
@@ -289,4 +405,7 @@ def _actor_hash(actor: Actor) -> bytes:
     h.update(actor.public_key_id.encode())
     h.update(actor.public_key_as_pem.encode())
 
+    if actor.moved_to:
+        h.update(actor.moved_to.encode())
+
     return h.digest()

app/admin.py

@@ -11,6 +11,7 @@ from fastapi.exceptions import HTTPException
 from fastapi.responses import RedirectResponse
 from loguru import logger
 from sqlalchemy import and_
+from sqlalchemy import delete
 from sqlalchemy import func
 from sqlalchemy import or_
 from sqlalchemy import select
@@ -25,8 +26,11 @@ from app.actor import fetch_actor
 from app.actor import get_actors_metadata
 from app.boxes import get_inbox_object_by_ap_id
 from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import send_block
 from app.boxes import send_follow
+from app.boxes import send_unblock
 from app.config import EMOJIS
+from app.config import SESSION_TIMEOUT
 from app.config import generate_csrf_token
 from app.config import session_serializer
 from app.config import verify_csrf_token
@@ -34,29 +38,42 @@ from app.config import verify_password
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.lookup import lookup
+from app.templates import is_current_user_admin
 from app.uploads import save_upload
 from app.utils import pagination
 from app.utils.emoji import EMOJIS_BY_NAME
 
 
-def user_session_or_redirect(
+async def user_session_or_redirect(
     request: Request,
     session: str | None = Cookie(default=None),
 ) -> None:
+    if request.method == "POST":
+        form_data = await request.form()
+        if "redirect_url" in form_data:
+            redirect_url = form_data["redirect_url"]
+        else:
+            redirect_url = request.url_for("admin_stream")
+    else:
+        redirect_url = str(request.url)
+
     _RedirectToLoginPage = HTTPException(
         status_code=302,
-        headers={"Location": request.url_for("login") + f"?redirect={request.url}"},
+        headers={"Location": request.url_for("login") + f"?redirect={redirect_url}"},
     )
 
     if not session:
+        logger.info("No existing admin session")
         raise _RedirectToLoginPage
 
     try:
-        loaded_session = session_serializer.loads(session, max_age=3600 * 12)
+        loaded_session = session_serializer.loads(session, max_age=SESSION_TIMEOUT)
     except Exception:
+        logger.exception("Failed to validate admin session")
         raise _RedirectToLoginPage
 
     if not loaded_session.get("is_logged_in"):
+        logger.info(f"Admin session invalidated: {loaded_session}")
         raise _RedirectToLoginPage
 
     return None
@@ -68,16 +85,6 @@ router = APIRouter(
 unauthenticated_router = APIRouter()
 
 
-@router.get("/")
-async def admin_index(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-) -> templates.TemplateResponse:
-    return await templates.render_template(
-        db_session, request, "index.html", {"request": request}
-    )
-
-
 @router.get("/lookup")
 async def get_lookup(
     request: Request,
@@ -94,6 +101,8 @@ async def get_lookup(
             error = ap.FetchErrorTypeEnum.TIMEOUT
         except (ap.ObjectNotFoundError, ap.ObjectIsGoneError):
             error = ap.FetchErrorTypeEnum.NOT_FOUND
+        except (ap.ObjectUnavailableError):
+            error = ap.FetchErrorTypeEnum.UNAUHTORIZED
         except Exception:
             logger.exception(f"Failed to lookup {query}")
             error = ap.FetchErrorTypeEnum.INTERNAL_ERROR
@@ -122,7 +131,9 @@ async def get_lookup(
                 )
                 if requested_object:
                     return RedirectResponse(
-                        request.url_for("admin_object") + f"?ap_id={ap_object.ap_id}",
+                        request.url_for("admin_object")
+                        + f"?ap_id={ap_object.ap_id}#"
+                        + requested_object.permalink_id,
                         status_code=302,
                     )
 
@@ -211,6 +222,7 @@ async def admin_bookmarks(
     request: Request,
     db_session: AsyncSession = Depends(get_db_session),
 ) -> templates.TemplateResponse:
+    # TODO: support pagination
     stream = (
         (
             await db_session.scalars(
@@ -335,6 +347,7 @@ async def admin_inbox(
                 "Update",
                 "Undo",
                 "Read",
+                "Reject",
                 "Add",
                 "Remove",
                 "EmojiReact",
@@ -431,6 +444,7 @@ async def admin_direct_messages(
                     models.InboxObject.ap_context.is_not(None),
                     # Skip transient object like poll relies
                     models.InboxObject.is_transient.is_(False),
+                    models.InboxObject.is_deleted.is_(False),
                 )
                 .group_by(models.InboxObject.ap_context, models.InboxObject.actor_id)
             )
@@ -453,6 +467,7 @@ async def admin_direct_messages(
                     models.OutboxObject.ap_context.is_not(None),
                     # Skip transient object like poll relies
                     models.OutboxObject.is_transient.is_(False),
+                    models.OutboxObject.is_deleted.is_(False),
                 )
                 .group_by(models.OutboxObject.ap_context)
             )
@@ -667,15 +682,30 @@ async def admin_outbox(
 
 @router.get("/notifications")
 async def get_notifications(
-    request: Request, db_session: AsyncSession = Depends(get_db_session)
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    cursor: str | None = None,
 ) -> templates.TemplateResponse:
+    where = []
+    if cursor:
+        decoded_cursor = pagination.decode_cursor(cursor)
+        where.append(models.Notification.created_at < decoded_cursor)
+
+    page_size = 20
+    remaining_count = await db_session.scalar(
+        select(func.count(models.Notification.id)).where(*where)
+    )
+
     notifications = (
         (
             await db_session.scalars(
                 select(models.Notification)
+                .where(*where)
                 .options(
                     joinedload(models.Notification.actor),
-                    joinedload(models.Notification.inbox_object),
+                    joinedload(models.Notification.inbox_object).options(
+                        joinedload(models.InboxObject.actor)
+                    ),
                     joinedload(models.Notification.outbox_object).options(
                         joinedload(
                             models.OutboxObject.outbox_object_attachments
@@ -684,6 +714,7 @@ async def get_notifications(
                     joinedload(models.Notification.webmention),
                 )
                 .order_by(models.Notification.created_at.desc())
+                .limit(page_size)
             )
         )
         .unique()
@@ -692,21 +723,42 @@ async def get_notifications(
     actors_metadata = await get_actors_metadata(
         db_session, [notif.actor for notif in notifications if notif.actor]
     )
+    more_unread_count = 0
+    next_cursor = None
 
-    for notif in notifications:
+    if notifications and remaining_count > page_size:
-        notif.is_new = False
+        decoded_next_cursor = notifications[-1].created_at
-    await db_session.commit()
+        next_cursor = pagination.encode_cursor(decoded_next_cursor)
 
-    return await templates.render_template(
+        # If on the "see more" page there's more unread notification, we want
+        # to display it next to the link
+        more_unread_count = await db_session.scalar(
+            select(func.count(models.Notification.id)).where(
+                models.Notification.is_new.is_(True),
+                models.Notification.created_at < decoded_next_cursor,
+            )
+        )
+
+    # Render the template before we change the new flag on notifications
+    tpl_resp = await templates.render_template(
         db_session,
         request,
         "notifications.html",
         {
             "notifications": notifications,
             "actors_metadata": actors_metadata,
+            "next_cursor": next_cursor,
+            "more_unread_count": more_unread_count,
         },
     )
 
+    if len({notif.id for notif in notifications if notif.is_new}):
+        for notif in notifications:
+            notif.is_new = False
+        await db_session.commit()
+
+    return tpl_resp
+
 
 @router.get("/object")
 async def admin_object(
@@ -715,7 +767,7 @@ async def admin_object(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> templates.TemplateResponse:
     requested_object = await boxes.get_anybox_object_by_ap_id(db_session, ap_id)
-    if not requested_object:
+    if not requested_object or requested_object.is_deleted:
         raise HTTPException(status_code=404)
 
     replies_tree = await boxes.get_replies_tree(
@@ -736,8 +788,10 @@ async def admin_object(
 async def admin_profile(
     request: Request,
     actor_id: str,
+    cursor: str | None = None,
     db_session: AsyncSession = Depends(get_db_session),
 ) -> templates.TemplateResponse:
+    # TODO: show featured/pinned
     actor = (
         await db_session.execute(
             select(models.Actor).where(models.Actor.ap_id == actor_id)
@@ -748,17 +802,27 @@ async def admin_profile(
 
     actors_metadata = await get_actors_metadata(db_session, [actor])
 
+    where = [
+        models.InboxObject.is_deleted.is_(False),
+        models.InboxObject.actor_id == actor.id,
+        models.InboxObject.ap_type.in_(
+            ["Note", "Article", "Video", "Page", "Announce"]
+        ),
+    ]
+    if cursor:
+        decoded_cursor = pagination.decode_cursor(cursor)
+        where.append(models.InboxObject.ap_published_at < decoded_cursor)
+
+    page_size = 20
+    remaining_count = await db_session.scalar(
+        select(func.count(models.InboxObject.id)).where(*where)
+    )
+
     inbox_objects = (
         (
             await db_session.scalars(
                 select(models.InboxObject)
-                .where(
+                .where(*where)
-                    models.InboxObject.is_deleted.is_(False),
-                    models.InboxObject.actor_id == actor.id,
-                    models.InboxObject.ap_type.in_(
-                        ["Note", "Article", "Video", "Page", "Announce"]
-                    ),
-                )
                 .options(
                     joinedload(models.InboxObject.relates_to_inbox_object).options(
                         joinedload(models.InboxObject.actor)
@@ -771,12 +835,19 @@ async def admin_profile(
                     joinedload(models.InboxObject.actor),
                 )
                 .order_by(models.InboxObject.ap_published_at.desc())
+                .limit(page_size)
             )
         )
         .unique()
         .all()
     )
 
+    next_cursor = (
+        pagination.encode_cursor(inbox_objects[-1].created_at)
+        if inbox_objects and remaining_count > page_size
+        else None
+    )
+
     return await templates.render_template(
         db_session,
         request,
@@ -785,10 +856,71 @@ async def admin_profile(
             "actors_metadata": actors_metadata,
             "actor": actor,
             "inbox_objects": inbox_objects,
+            "next_cursor": next_cursor,
         },
     )
 
 
+@router.post("/actions/force_delete")
+async def admin_actions_force_delete(
+    request: Request,
+    ap_object_id: str = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    ap_object_to_delete = await get_inbox_object_by_ap_id(db_session, ap_object_id)
+    if not ap_object_to_delete:
+        raise ValueError(f"Cannot find {ap_object_id}")
+
+    logger.info(f"Deleting {ap_object_to_delete.ap_type}/{ap_object_to_delete.ap_id}")
+    await boxes._revert_side_effect_for_deleted_object(
+        db_session,
+        None,
+        ap_object_to_delete,
+        None,
+    )
+    ap_object_to_delete.is_deleted = True
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
+@router.post("/actions/force_delete_webmention")
+async def admin_actions_force_delete_webmention(
+    request: Request,
+    webmention_id: int = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    webmention = await boxes.get_webmention_by_id(db_session, webmention_id)
+    if not webmention:
+        raise ValueError(f"Cannot find {webmention_id}")
+    if not webmention.outbox_object:
+        raise ValueError(f"Missing related outbox object for {webmention_id}")
+
+    # TODO: move this
+    logger.info(f"Deleting {webmention_id}")
+    webmention.is_deleted = True
+    await db_session.flush()
+    from app.webmentions import _handle_webmention_side_effects
+
+    await _handle_webmention_side_effects(
+        db_session, webmention, webmention.outbox_object
+    )
+    # Delete related notifications
+    notif_deletion_result = await db_session.execute(
+        delete(models.Notification)
+        .where(models.Notification.webmention_id == webmention.id)
+        .execution_options(synchronize_session=False)
+    )
+    logger.info(
+        f"Deleted {notif_deletion_result.rowcount} notifications"  # type: ignore
+    )
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
 @router.post("/actions/follow")
 async def admin_actions_follow(
     request: Request,
@@ -810,10 +942,7 @@ async def admin_actions_block(
     csrf_check: None = Depends(verify_csrf_token),
     db_session: AsyncSession = Depends(get_db_session),
 ) -> RedirectResponse:
-    logger.info(f"Blocking {ap_actor_id}")
+    await send_block(db_session, ap_actor_id)
-    actor = await fetch_actor(db_session, ap_actor_id)
-    actor.is_blocked = True
-    await db_session.commit()
     return RedirectResponse(redirect_url, status_code=302)
 
 
@@ -826,9 +955,7 @@ async def admin_actions_unblock(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> RedirectResponse:
     logger.info(f"Unblocking {ap_actor_id}")
-    actor = await fetch_actor(db_session, ap_actor_id)
+    await send_unblock(db_session, ap_actor_id)
-    actor.is_blocked = False
-    await db_session.commit()
     return RedirectResponse(redirect_url, status_code=302)
 
 
@@ -974,7 +1101,7 @@ async def admin_actions_unpin(
 async def admin_actions_new(
     request: Request,
     files: list[UploadFile] = [],
-    content: str = Form(),
+    content: str | None = Form(None),
     redirect_url: str = Form(),
     in_reply_to: str | None = Form(None),
     content_warning: str | None = Form(None),
@@ -985,6 +1112,19 @@ async def admin_actions_new(
     csrf_check: None = Depends(verify_csrf_token),
     db_session: AsyncSession = Depends(get_db_session),
 ) -> RedirectResponse:
+    if not content and not content_warning:
+        raise HTTPException(status_code=422, detail="Error: object must have a content")
+
+    # Do like Mastodon, if there's only a CW with no content and some attachments,
+    # swap the CW and the content
+    if not content and content_warning and len(files) >= 1:
+        content = content_warning
+        is_sensitive = True
+        content_warning = None
+
+    if not content:
+        raise HTTPException(status_code=422, detail="Error: objec must have a content")
+
     # XXX: for some reason, no files restuls in an empty single file
     uploads = []
     raw_form_data = await request.form()
@@ -1054,7 +1194,10 @@ async def admin_actions_vote(
 async def login(
     request: Request,
     db_session: AsyncSession = Depends(get_db_session),
-) -> templates.TemplateResponse:
+) -> templates.TemplateResponse | RedirectResponse:
+    if is_current_user_admin(request):
+        return RedirectResponse(request.url_for("admin_stream"), status_code=302)
+
     return await templates.render_template(
         db_session,
         request,
@@ -1072,11 +1215,25 @@ async def login_validation(
     password: str = Form(),
     redirect: str | None = Form(None),
     csrf_check: None = Depends(verify_csrf_token),
-) -> RedirectResponse:
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse | templates.TemplateResponse:
     if not verify_password(password):
-        raise HTTPException(status_code=401)
+        logger.warning("Invalid password")
+        return await templates.render_template(
+            db_session,
+            request,
+            "login.html",
+            {
+                "error": "Invalid password",
+                "csrf_token": generate_csrf_token(),
+                "redirect": request.query_params.get("redirect", ""),
+            },
+            status_code=403,
+        )
 
-    resp = RedirectResponse(redirect or "/admin/stream", status_code=302)
+    resp = RedirectResponse(
+        redirect or request.url_for("admin_stream"), status_code=302
+    )
     resp.set_cookie("session", session_serializer.dumps({"is_logged_in": True}))  # type: ignore  # noqa: E501
 
     return resp

app/ap_object.py

@@ -1,11 +1,12 @@
 import hashlib
+import mimetypes
 from datetime import datetime
 from functools import cached_property
 from typing import Any
 
 import pydantic
 from bs4 import BeautifulSoup  # type: ignore
-from markdown import markdown
+from mistletoe import markdown  # type: ignore
 
 from app import activitypub as ap
 from app.actor import LOCAL_ACTOR
@@ -95,6 +96,9 @@ class Object:
     def attachments(self) -> list["Attachment"]:
         attachments = []
         for obj in ap.as_list(self.ap_object.get("attachment", [])):
+            if obj.get("type") == "PropertyValue":
+                continue
+
             if obj.get("type") == "Link":
                 attachments.append(
                     Attachment.parse_obj(
@@ -155,7 +159,7 @@ class Object:
     @cached_property
     def url(self) -> str | None:
         obj_url = self.ap_object.get("url")
-        if isinstance(obj_url, str):
+        if isinstance(obj_url, str) and obj_url:
             return obj_url
         elif obj_url:
             for u in ap.as_list(obj_url):
@@ -175,7 +179,7 @@ class Object:
 
         # PeerTube returns the content as markdown
         if self.ap_object.get("mediaType") == "text/markdown":
-            content = markdown(content, extensions=["mdx_linkify"])
+            content = markdown(content)
 
         return content
 
@@ -208,6 +212,13 @@ class Object:
     def in_reply_to(self) -> str | None:
         return self.ap_object.get("inReplyTo")
 
+    @property
+    def is_in_reply_to_from_inbox(self) -> bool | None:
+        if not self.in_reply_to:
+            return None
+
+        return not self.in_reply_to.startswith(LOCAL_ACTOR.ap_id)
+
     @property
     def has_ld_signature(self) -> bool:
         return bool(self.ap_object.get("signature"))
@@ -269,6 +280,20 @@ class Attachment(BaseModel):
     proxied_url: str | None = None
     resized_url: str | None = None
 
+    width: int | None = None
+    height: int | None = None
+
+    @property
+    def mimetype(self) -> str:
+        mimetype = self.media_type
+        if not mimetype:
+            mimetype, _ = mimetypes.guess_type(self.url)
+
+        if not mimetype:
+            return "unknown"
+
+        return mimetype.split("/")[-1]
+
 
 class RemoteObject(Object):
     def __init__(self, raw_object: ap.RawObject, actor: Actor):

app/config.py

@@ -1,4 +1,5 @@
 import hashlib
+import hmac
 import os
 import secrets
 from pathlib import Path
@@ -12,8 +13,13 @@ from fastapi import HTTPException
 from fastapi import Request
 from itsdangerous import URLSafeTimedSerializer
 from loguru import logger
+from mistletoe import markdown  # type: ignore
 
+from app.customization import _CUSTOM_ROUTES
+from app.customization import _StreamVisibilityCallback
+from app.customization import default_stream_visibility_callback
 from app.utils.emoji import _load_emojis
+from app.utils.version import get_version_commit
 
 ROOT_DIR = Path().parent.resolve()
 
@@ -24,7 +30,7 @@ VERSION_COMMIT = "dev"
 try:
     from app._version import VERSION_COMMIT  # type: ignore
 except ImportError:
-    pass
+    VERSION_COMMIT = get_version_commit()
 
 # Force reloading cache when the CSS is updated
 CSS_HASH = "none"
@@ -34,6 +40,31 @@ try:
 except FileNotFoundError:
     pass
 
+# Force reloading cache when the JS is changed
+JS_HASH = "none"
+try:
+    # To keep things simple, we keep a single hash for the 2 files
+    js_data_common = (ROOT_DIR / "app" / "static" / "common-admin.js").read_bytes()
+    js_data_new = (ROOT_DIR / "app" / "static" / "new.js").read_bytes()
+    JS_HASH = hashlib.md5(
+        js_data_common + js_data_new, usedforsecurity=False
+    ).hexdigest()
+except FileNotFoundError:
+    pass
+
+MOVED_TO_FILE = ROOT_DIR / "data" / "moved_to.dat"
+
+
+def _get_moved_to() -> str | None:
+    if not MOVED_TO_FILE.exists():
+        return None
+
+    return MOVED_TO_FILE.read_text()
+
+
+def set_moved_to(moved_to: str) -> None:
+    MOVED_TO_FILE.write_text(moved_to)
+
 
 VERSION = f"2.0.0+{VERSION_COMMIT}"
 USER_AGENT = f"microblogpub/{VERSION}"
@@ -62,7 +93,8 @@ class Config(pydantic.BaseModel):
     name: str
     summary: str
     https: bool
-    icon_url: str
+    icon_url: str | None = None
+    image_url: str | None = None
     secret: str
     debug: bool = False
     trusted_hosts: list[str] = ["127.0.0.1"]
@@ -71,13 +103,26 @@ class Config(pydantic.BaseModel):
     metadata: list[_ProfileMetadata] | None = None
     code_highlighting_theme = "friendly_grayscale"
     blocked_servers: list[_BlockedServer] = []
+    custom_footer: str | None = None
+    emoji: str | None = None
+    also_known_as: str | None = None
+
+    hides_followers: bool = False
+    hides_following: bool = False
 
     inbox_retention_days: int = 15
 
+    custom_content_security_policy: str | None = None
+
     # Config items to make tests easier
     sqlalchemy_database: str | None = None
     key_path: str | None = None
 
+    session_timeout: int = 3600 * 24 * 3  # in seconds, 3 days by default
+
+    # Only set when the app is served on a non-root path
+    id: str | None = None
+
 
 def load_config() -> Config:
     try:
@@ -112,15 +157,30 @@ CONFIG = load_config()
 DOMAIN = CONFIG.domain
 _SCHEME = "https" if CONFIG.https else "http"
 ID = f"{_SCHEME}://{DOMAIN}"
+
+# When running the app on a path, the ID maybe set by the config, but in this
+# case, a valid webfinger must be served on the root domain
+if CONFIG.id:
+    ID = CONFIG.id
 USERNAME = CONFIG.username
 MANUALLY_APPROVES_FOLLOWERS = CONFIG.manually_approves_followers
+HIDES_FOLLOWERS = CONFIG.hides_followers
+HIDES_FOLLOWING = CONFIG.hides_following
 PRIVACY_REPLACE = None
 if CONFIG.privacy_replace:
     PRIVACY_REPLACE = {pr.domain: pr.replace_by for pr in CONFIG.privacy_replace}
 
 BLOCKED_SERVERS = {blocked_server.hostname for blocked_server in CONFIG.blocked_servers}
+ALSO_KNOWN_AS = CONFIG.also_known_as
+CUSTOM_CONTENT_SECURITY_POLICY = CONFIG.custom_content_security_policy
 
 INBOX_RETENTION_DAYS = CONFIG.inbox_retention_days
+SESSION_TIMEOUT = CONFIG.session_timeout
+CUSTOM_FOOTER = (
+    markdown(CONFIG.custom_footer.replace("{version}", VERSION))
+    if CONFIG.custom_footer
+    else None
+)
 
 BASE_URL = ID
 DEBUG = CONFIG.debug
@@ -130,13 +190,45 @@ KEY_PATH = (
     (ROOT_DIR / CONFIG.key_path) if CONFIG.key_path else ROOT_DIR / "data" / "key.pem"
 )
 EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾"
+if CONFIG.emoji:
+    EMOJIS = CONFIG.emoji
+
 # Emoji template for the FE
-EMOJI_TPL = '<img src="/static/twemoji/{filename}.svg" alt="{raw}" class="emoji">'
+EMOJI_TPL = (
+    '<img src="{base_url}/static/twemoji/{filename}.svg" alt="{raw}" class="emoji">'
+)
 
 _load_emojis(ROOT_DIR, BASE_URL)
 
 CODE_HIGHLIGHTING_THEME = CONFIG.code_highlighting_theme
 
+MOVED_TO = _get_moved_to()
+
+
+_NavBarItem = tuple[str, str]
+
+
+class NavBarItems:
+    EXTRA_NAVBAR_ITEMS: list[_NavBarItem] = []
+    INDEX_NAVBAR_ITEM: _NavBarItem | None = None
+    NOTES_PATH = "/"
+
+
+def load_custom_routes() -> None:
+    try:
+        from data import custom_routes  # type: ignore  # noqa: F401
+    except ImportError:
+        pass
+
+    for path, custom_handler in _CUSTOM_ROUTES.items():
+        # If a handler wants to replace the root, move the index to /notes
+        if path == "/":
+            NavBarItems.NOTES_PATH = "/notes"
+            NavBarItems.INDEX_NAVBAR_ITEM = (path, custom_handler.title)
+        else:
+            if custom_handler.show_in_navbar:
+                NavBarItems.EXTRA_NAVBAR_ITEMS.append((path, custom_handler.title))
+
 
 session_serializer = URLSafeTimedSerializer(
     CONFIG.secret,
@@ -152,10 +244,34 @@ def generate_csrf_token() -> str:
     return csrf_serializer.dumps(secrets.token_hex(16))  # type: ignore
 
 
-def verify_csrf_token(csrf_token: str = Form()) -> None:
+def verify_csrf_token(
+    csrf_token: str = Form(),
+    redirect_url: str | None = Form(None),
+) -> None:
+    please_try_again = "please try again"
+    if redirect_url:
+        please_try_again = f'<a href="{redirect_url}">please try again</a>'
     try:
         csrf_serializer.loads(csrf_token, max_age=1800)
     except (itsdangerous.BadData, itsdangerous.SignatureExpired):
         logger.exception("Failed to verify CSRF token")
-        raise HTTPException(status_code=403, detail="CSRF error")
+        raise HTTPException(
+            status_code=403,
+            detail=f"The security token has expired, {please_try_again}",
+        )
     return None
+
+
+def hmac_sha256() -> hmac.HMAC:
+    return hmac.new(CONFIG.secret.encode(), digestmod=hashlib.sha256)
+
+
+stream_visibility_callback: _StreamVisibilityCallback
+try:
+    from data.stream import (  # type: ignore  # noqa: F401, E501
+        custom_stream_visibility_callback,
+    )
+
+    stream_visibility_callback = custom_stream_visibility_callback
+except ImportError:
+    stream_visibility_callback = default_stream_visibility_callback

app/customization.py

@@ -0,0 +1,154 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import Callable
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import Request
+from loguru import logger
+from starlette.responses import JSONResponse
+
+if TYPE_CHECKING:
+    from app.ap_object import RemoteObject
+
+
+_DATA_DIR = Path().parent.resolve() / "data"
+_Handler = Callable[..., Any]
+
+
+class HTMLPage:
+    def __init__(
+        self,
+        title: str,
+        html_file: str,
+        show_in_navbar: bool,
+    ) -> None:
+        self.title = title
+        self.html_file = _DATA_DIR / html_file
+        self.show_in_navbar = show_in_navbar
+
+
+class RawHandler:
+    def __init__(
+        self,
+        title: str,
+        handler: Any,
+        show_in_navbar: bool,
+    ) -> None:
+        self.title = title
+        self.handler = handler
+        self.show_in_navbar = show_in_navbar
+
+
+_CUSTOM_ROUTES: dict[str, HTMLPage | RawHandler] = {}
+
+
+def register_html_page(
+    path: str,
+    *,
+    title: str,
+    html_file: str,
+    show_in_navbar: bool = True,
+) -> None:
+    if path in _CUSTOM_ROUTES:
+        raise ValueError(f"{path} is already registered")
+
+    _CUSTOM_ROUTES[path] = HTMLPage(title, html_file, show_in_navbar)
+
+
+def register_raw_handler(
+    path: str,
+    *,
+    title: str,
+    handler: _Handler,
+    show_in_navbar: bool = True,
+) -> None:
+    if path in _CUSTOM_ROUTES:
+        raise ValueError(f"{path} is already registered")
+
+    _CUSTOM_ROUTES[path] = RawHandler(title, handler, show_in_navbar)
+
+
+class ActivityPubResponse(JSONResponse):
+    media_type = "application/activity+json"
+
+
+def _custom_page_handler(path: str, html_page: HTMLPage) -> Any:
+    from app import templates
+    from app.actor import LOCAL_ACTOR
+    from app.config import is_activitypub_requested
+    from app.database import AsyncSession
+    from app.database import get_db_session
+
+    async def _handler(
+        request: Request,
+        db_session: AsyncSession = Depends(get_db_session),
+    ) -> templates.TemplateResponse | ActivityPubResponse:
+        if path == "/" and is_activitypub_requested(request):
+            return ActivityPubResponse(LOCAL_ACTOR.ap_actor)
+
+        return await templates.render_template(
+            db_session,
+            request,
+            "custom_page.html",
+            {
+                "page_content": html_page.html_file.read_text(),
+                "title": html_page.title,
+            },
+        )
+
+    return _handler
+
+
+def get_custom_router() -> APIRouter | None:
+    if not _CUSTOM_ROUTES:
+        return None
+
+    router = APIRouter()
+
+    for path, handler in _CUSTOM_ROUTES.items():
+        if isinstance(handler, HTMLPage):
+            router.add_api_route(
+                path, _custom_page_handler(path, handler), methods=["GET"]
+            )
+        else:
+            router.add_api_route(path, handler.handler)
+
+    return router
+
+
+@dataclass
+class ObjectInfo:
+    # Is it a reply?
+    is_reply: bool
+
+    # Is it a reply to an outbox object
+    is_local_reply: bool
+
+    # Is the object mentioning the local actor
+    is_mention: bool
+
+    # Is it from someone the local actor is following
+    is_from_following: bool
+
+    # List of hashtags, e.g. #microblogpub
+    hashtags: list[str]
+
+    # @dev@microblog.pub
+    actor_handle: str
+
+    remote_object: "RemoteObject"
+
+
+_StreamVisibilityCallback = Callable[[ObjectInfo], bool]
+
+
+def default_stream_visibility_callback(object_info: ObjectInfo) -> bool:
+    logger.info(f"{object_info=}")
+    return (
+        (not object_info.is_reply and object_info.is_from_following)
+        or object_info.is_mention
+        or object_info.is_local_reply
+    )

app/database.py

@@ -9,6 +9,7 @@ from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
 from app.config import DB_PATH
+from app.config import DEBUG
 from app.config import SQLALCHEMY_DATABASE_URL
 
 engine = create_engine(
@@ -18,7 +19,7 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 DATABASE_URL = f"sqlite+aiosqlite:///{DB_PATH}"
 async_engine = create_async_engine(
-    DATABASE_URL, future=True, echo=False, connect_args={"timeout": 15}
+    DATABASE_URL, future=True, echo=DEBUG, connect_args={"timeout": 15}
 )
 async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False)
 

app/httpsig.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+import json
 import typing
 from dataclasses import dataclass
 from datetime import datetime
@@ -88,8 +89,12 @@ def _body_digest(body: bytes) -> str:
     return "SHA-256=" + base64.b64encode(h.digest()).decode("utf-8")
 
 
-async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
+async def _get_public_key(
-    if cached_key := _KEY_CACHE.get(key_id):
+    db_session: AsyncSession,
+    key_id: str,
+    should_skip_cache: bool = False,
+) -> Key:
+    if not should_skip_cache and (cached_key := _KEY_CACHE.get(key_id)):
         logger.info(f"Key {key_id} found in cache")
         return cached_key
 
@@ -101,25 +106,25 @@ async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
             select(models.Actor).where(models.Actor.ap_id == key_id.split("#")[0])
         )
     ).one_or_none()
-    if existing_actor and existing_actor.public_key_id == key_id:
+    if not should_skip_cache:
-        k = Key(existing_actor.ap_id, key_id)
+        if existing_actor and existing_actor.public_key_id == key_id:
-        k.load_pub(existing_actor.public_key_as_pem)
+            k = Key(existing_actor.ap_id, key_id)
-        logger.info(f"Found {key_id} on an existing actor")
+            k.load_pub(existing_actor.public_key_as_pem)
-        _KEY_CACHE[key_id] = k
+            logger.info(f"Found {key_id} on an existing actor")
-        return k
+            _KEY_CACHE[key_id] = k
+            return k
 
     # Fetch it
     from app import activitypub as ap
+    from app.actor import RemoteActor
+    from app.actor import update_actor_if_needed
 
     # Without signing the request as if it's the first contact, the 2 servers
     # might race to fetch each other key
     try:
         actor = await ap.fetch(key_id, disable_httpsig=True)
-    except httpx.HTTPStatusError as http_err:
+    except ap.ObjectUnavailableError:
-        if http_err.response.status_code in [401, 403]:
+        actor = await ap.fetch(key_id, disable_httpsig=False)
-            actor = await ap.fetch(key_id, disable_httpsig=False)
-        else:
-            raise
 
     if actor["type"] == "Key":
         # The Key is not embedded in the Person
@@ -130,11 +135,18 @@ async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
         k.load_pub(actor["publicKey"]["publicKeyPem"])
 
     # Ensure the right key was fetch
+    # TODO: some server have the key ID `http://` but fetching it return `https`
     if key_id not in [k.key_id(), k.owner]:
         raise ValueError(
             f"failed to fetch requested key {key_id}: got {actor['publicKey']}"
         )
 
+    if should_skip_cache and actor["type"] != "Key" and existing_actor:
+        # We had to skip the cache, which means the actor key probably changed
+        # and we want to update our cached version
+        await update_actor_if_needed(db_session, existing_actor, RemoteActor(actor))
+        await db_session.commit()
+
     _KEY_CACHE[key_id] = k
     return k
 
@@ -187,6 +199,32 @@ async def httpsig_checker(
             server=server,
         )
 
+    # Try to drop Delete activity spams early on, this prevent making an extra
+    # HTTP requests trying to fetch an unavailable actor to verify the HTTP sig
+    try:
+        if request.method == "POST" and request.url.path.endswith("/inbox"):
+            from app import models  # TODO: solve this circular import
+
+            activity = json.loads(body)
+            actor_id = ap.get_id(activity["actor"])
+            if (
+                ap.as_list(activity["type"])[0] == "Delete"
+                and actor_id == ap.get_id(activity["object"])
+                and not (
+                    await db_session.scalars(
+                        select(models.Actor).where(
+                            models.Actor.ap_id == actor_id,
+                        )
+                    )
+                ).one_or_none()
+            ):
+                logger.info(f"Dropping Delete activity early for {body=}")
+                raise fastapi.HTTPException(status_code=202)
+    except fastapi.HTTPException as http_exc:
+        raise http_exc
+    except Exception:
+        logger.exception("Failed to check for Delete spam")
+
     # logger.debug(f"hsig={hsig}")
     signed_string, signature_date = _build_signed_string(
         hsig["headers"],
@@ -215,10 +253,23 @@ async def httpsig_checker(
         logger.exception(f'Failed to fetch HTTP sig key {hsig["keyId"]}')
         return HTTPSigInfo(has_valid_signature=False)
 
+    has_valid_signature = _verify_h(
+        signed_string, base64.b64decode(hsig["signature"]), k.pubkey
+    )
+
+    # If the signature is not valid, we may have to update the cached actor
+    if not has_valid_signature:
+        logger.info("Invalid signature, trying to refresh actor")
+        try:
+            k = await _get_public_key(db_session, hsig["keyId"], should_skip_cache=True)
+            has_valid_signature = _verify_h(
+                signed_string, base64.b64decode(hsig["signature"]), k.pubkey
+            )
+        except Exception:
+            logger.exception("Failed to refresh actor")
+
     httpsig_info = HTTPSigInfo(
-        has_valid_signature=_verify_h(
+        has_valid_signature=has_valid_signature,
-            signed_string, base64.b64decode(hsig["signature"]), k.pubkey
-        ),
         signed_by_ap_actor_id=k.owner,
         server=server,
     )

app/incoming_activities.py

@@ -3,7 +3,6 @@ import traceback
 from datetime import datetime
 from datetime import timedelta
 
-import httpx
 from loguru import logger
 from sqlalchemy import func
 from sqlalchemy import select
@@ -26,7 +25,7 @@ async def new_ap_incoming_activity(
     raw_object: ap.RawObject,
 ) -> models.IncomingActivity | None:
     ap_id: str
-    if "id" not in raw_object:
+    if "id" not in raw_object or ap.as_list(raw_object["type"])[0] in ap.ACTOR_TYPES:
         if "@context" not in raw_object:
             logger.warning(f"Dropping invalid object: {raw_object}")
             return None
@@ -108,22 +107,29 @@ async def process_next_incoming_activity(
 
     next_activity.tries = next_activity.tries + 1
     next_activity.last_try = now()
+    await db_session.commit()
 
     if next_activity.ap_object and next_activity.sent_by_ap_actor_id:
         try:
             async with db_session.begin_nested():
-                await save_to_inbox(
+                await asyncio.wait_for(
-                    db_session,
+                    save_to_inbox(
-                    next_activity.ap_object,
+                        db_session,
-                    next_activity.sent_by_ap_actor_id,
+                        next_activity.ap_object,
+                        next_activity.sent_by_ap_actor_id,
+                    ),
+                    timeout=60,
                 )
-        except httpx.TimeoutException as exc:
+        except asyncio.exceptions.TimeoutError:
-            url = exc._request.url if exc._request else None
+            logger.error("Activity took too long to process")
-            logger.error(f"Failed, HTTP timeout when fetching {url}")
+            await db_session.rollback()
+            await db_session.refresh(next_activity)
             next_activity.error = traceback.format_exc()
             _set_next_try(next_activity)
         except Exception:
             logger.exception("Failed")
+            await db_session.rollback()
+            await db_session.refresh(next_activity)
             next_activity.error = traceback.format_exc()
             _set_next_try(next_activity)
         else:

app/indieauth.py

@@ -276,7 +276,7 @@ async def _check_access_token(
     if now() > access_token_info.created_at.replace(tzinfo=timezone.utc) + timedelta(
         seconds=access_token_info.expires_in
     ):
-        logger.info("Access token is expired")
+        logger.info("Access token has expired")
         return False, None
 
     return True, access_token_info

app/lookup.py

@@ -10,6 +10,7 @@ from app.source import _MENTION_REGEX
 
 
 async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
+    query = query.strip()
     if query.startswith("@") or _MENTION_REGEX.match("@" + query):
         query = await webfinger.get_actor_url(query)  # type: ignore  # None check below
 
@@ -37,4 +38,9 @@ async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
     if ap.as_list(ap_obj["type"])[0] in ap.ACTOR_TYPES:
         return RemoteActor(ap_obj)
     else:
+        # Some software return objects wrapped in a Create activity (like
+        # python-federation)
+        if ap.as_list(ap_obj["type"])[0] == "Create":
+            ap_obj = await ap.get_object(ap_obj)
+
         return await RemoteObject.from_raw_object(ap_obj)

app/main.py

@@ -8,6 +8,7 @@ from typing import Any
 from typing import MutableMapping
 from typing import Type
 
+import fastapi
 import httpx
 import starlette
 from asgiref.typing import ASGI3Application
@@ -20,6 +21,7 @@ from fastapi import FastAPI
 from fastapi import Form
 from fastapi import Request
 from fastapi import Response
+from fastapi.exception_handlers import http_exception_handler
 from fastapi.exceptions import HTTPException
 from fastapi.responses import FileResponse
 from fastapi.responses import PlainTextResponse
@@ -35,6 +37,7 @@ from sqlalchemy.orm import joinedload
 from starlette.background import BackgroundTask
 from starlette.datastructures import Headers
 from starlette.datastructures import MutableHeaders
+from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.responses import JSONResponse
 from starlette.types import Message
 from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware  # type: ignore
@@ -45,6 +48,7 @@ from app import boxes
 from app import config
 from app import httpsig
 from app import indieauth
+from app import media
 from app import micropub
 from app import models
 from app import templates
@@ -60,21 +64,30 @@ from app.config import USER_AGENT
 from app.config import USERNAME
 from app.config import is_activitypub_requested
 from app.config import verify_csrf_token
+from app.customization import get_custom_router
 from app.database import AsyncSession
+from app.database import async_session
 from app.database import get_db_session
 from app.incoming_activities import new_ap_incoming_activity
 from app.templates import is_current_user_admin
 from app.uploads import UPLOAD_DIR
 from app.utils import pagination
 from app.utils.emoji import EMOJIS_BY_NAME
+from app.utils.facepile import Face
+from app.utils.facepile import WebmentionReply
+from app.utils.facepile import merge_faces
+from app.utils.highlight import HIGHLIGHT_CSS_HASH
 from app.utils.url import check_url
 from app.webfinger import get_remote_follow_template
 
-_RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCache(32)
+# Only images <1MB will be cached, so 64MB of data will be cached
+_RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCache(64)
 
 
 # TODO(ts):
 # Next:
+# - self-destruct + move support and actions/tasks for
+# - doc for prune/move/delete
 # - fix issue with followers from a blocked server (skip it?)
 # - allow to share old notes
 # - only show 10 most recent threads in DMs
@@ -118,21 +131,27 @@ class CustomMiddleware:
                 # And add the security headers
                 headers = MutableHeaders(scope=message)
                 headers["X-Request-ID"] = request_id
-                headers["Server"] = "microblogpub"
+                headers["x-powered-by"] = "microblogpub"
                 headers[
                     "referrer-policy"
                 ] = "no-referrer, strict-origin-when-cross-origin"
                 headers["x-content-type-options"] = "nosniff"
                 headers["x-xss-protection"] = "1; mode=block"
-                headers["x-frame-options"] = "SAMEORIGIN"
+                headers["x-frame-options"] = "DENY"
-                # TODO(ts): disallow inline CSS?
+                headers["permissions-policy"] = "interest-cohort=()"
-                headers[
+                headers["content-security-policy"] = (
-                    "content-security-policy"
+                    (
-                ] = "default-src 'self'; style-src 'self' 'unsafe-inline';"
+                        f"default-src 'self'; "
+                        f"style-src 'self' 'sha256-{HIGHLIGHT_CSS_HASH}'; "
+                        f"frame-ancestors 'none'; base-uri 'self'; form-action 'self';"
+                    )
+                    if not config.CUSTOM_CONTENT_SECURITY_POLICY
+                    else config.CUSTOM_CONTENT_SECURITY_POLICY.format(
+                        HIGHLIGHT_CSS_HASH=HIGHLIGHT_CSS_HASH
+                    )
+                )
                 if not DEBUG:
-                    headers[
+                    headers["strict-transport-security"] = "max-age=63072000;"
-                        "strict-transport-security"
-                    ] = "max-age=63072000; includeSubdomains"
 
             await send(message)  # type: ignore
 
@@ -164,7 +183,15 @@ class CustomMiddleware:
         return None
 
 
-app = FastAPI(docs_url=None, redoc_url=None)
+def _check_0rtt_early_data(request: Request) -> None:
+    """Disable TLS1.3 0-RTT requests for non-GET."""
+    if request.headers.get("Early-Data", None) == "1" and request.method != "GET":
+        raise fastapi.HTTPException(status_code=425, detail="Too early")
+
+
+app = FastAPI(
+    docs_url=None, redoc_url=None, dependencies=[Depends(_check_0rtt_early_data)]
+)
 app.mount(
     "/custom_emoji",
     StaticFiles(directory="data/custom_emoji"),
@@ -176,6 +203,9 @@ app.include_router(admin.unauthenticated_router, prefix="/admin")
 app.include_router(indieauth.router)
 app.include_router(micropub.router)
 app.include_router(webmentions.router)
+config.load_custom_routes()
+if custom_router := get_custom_router():
+    app.include_router(custom_router)
 
 # XXX: order matters, the proxy middleware needs to be last
 app.add_middleware(CustomMiddleware)
@@ -192,11 +222,66 @@ logger_format = (
 logger.add(sys.stdout, format=logger_format, level="DEBUG" if DEBUG else "INFO")
 
 
+@app.exception_handler(StarletteHTTPException)
+async def custom_http_exception_handler(
+    request: Request,
+    exc: StarletteHTTPException,
+) -> templates.TemplateResponse | JSONResponse:
+    accept_value = request.headers.get("accept")
+    if (
+        accept_value
+        and accept_value.startswith("text/html")
+        and 400 <= exc.status_code < 600
+    ):
+        async with async_session() as db_session:
+            title = (
+                {
+                    404: "Oops, nothing to see here",
+                    500: "Oops, something went wrong",
+                }
+            ).get(exc.status_code, exc.detail)
+            try:
+                return await templates.render_template(
+                    db_session,
+                    request,
+                    "error.html",
+                    {"title": title},
+                    status_code=exc.status_code,
+                )
+            finally:
+                await db_session.close()
+    return await http_exception_handler(request, exc)
+
+
 class ActivityPubResponse(JSONResponse):
     media_type = "application/activity+json"
 
 
-@app.get("/")
+async def redirect_to_remote_instance(
+    request: Request,
+    db_session: AsyncSession,
+    url: str,
+) -> templates.TemplateResponse:
+    """
+    Similar to RedirectResponse, but uses a 200 response with HTML.
+
+    Needed for remote redirects on form submission endpoints,
+    since our CSP policy disallows remote form submission.
+    https://github.com/w3c/webappsec-csp/issues/8#issuecomment-810108984
+    """
+    return await templates.render_template(
+        db_session,
+        request,
+        "redirect_to_remote_instance.html",
+        {
+            "request": request,
+            "url": url,
+        },
+        headers={"Refresh": "0;url=" + url},
+    )
+
+
+@app.get(config.NavBarItems.NOTES_PATH)
 async def index(
     request: Request,
     db_session: AsyncSession = Depends(get_db_session),
@@ -204,6 +289,7 @@ async def index(
     page: int | None = None,
 ) -> templates.TemplateResponse | ActivityPubResponse:
     if is_activitypub_requested(request):
+
         return ActivityPubResponse(LOCAL_ACTOR.ap_actor)
 
     page = page or 1
@@ -234,6 +320,7 @@ async def index(
                 ),
             ),
         )
+        .order_by(models.OutboxObject.is_pinned.desc())
         .order_by(models.OutboxObject.ap_published_at.desc())
         .offset(page_offset)
         .limit(page_size)
@@ -354,6 +441,20 @@ async def _build_followx_collection(
     return collection_page
 
 
+async def _empty_followx_collection(
+    db_session: AsyncSession,
+    model_cls: Type[models.Following | models.Follower],
+    path: str,
+) -> ap.RawObject:
+    total_items = await db_session.scalar(select(func.count(model_cls.id)))
+    return {
+        "@context": ap.AS_CTX,
+        "id": ID + path,
+        "type": "OrderedCollection",
+        "totalItems": total_items,
+    }
+
+
 @app.get("/followers")
 async def followers(
     request: Request,
@@ -364,15 +465,27 @@ async def followers(
     _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse:
     if is_activitypub_requested(request):
-        return ActivityPubResponse(
+        if config.HIDES_FOLLOWERS:
-            await _build_followx_collection(
+            return ActivityPubResponse(
-                db_session=db_session,
+                await _empty_followx_collection(
-                model_cls=models.Follower,
+                    db_session=db_session,
-                path="/followers",
+                    model_cls=models.Follower,
-                page=page,
+                    path="/followers",
-                next_cursor=next_cursor,
+                )
             )
-        )
+        else:
+            return ActivityPubResponse(
+                await _build_followx_collection(
+                    db_session=db_session,
+                    model_cls=models.Follower,
+                    path="/followers",
+                    page=page,
+                    next_cursor=next_cursor,
+                )
+            )
+
+    if config.HIDES_FOLLOWERS and not is_current_user_admin(request):
+        raise HTTPException(status_code=404)
 
     # We only show the most recent 20 followers on the public website
     followers_result = await db_session.scalars(
@@ -411,15 +524,27 @@ async def following(
     _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse:
     if is_activitypub_requested(request):
-        return ActivityPubResponse(
+        if config.HIDES_FOLLOWING:
-            await _build_followx_collection(
+            return ActivityPubResponse(
-                db_session=db_session,
+                await _empty_followx_collection(
-                model_cls=models.Following,
+                    db_session=db_session,
-                path="/following",
+                    model_cls=models.Following,
-                page=page,
+                    path="/following",
-                next_cursor=next_cursor,
+                )
             )
-        )
+        else:
+            return ActivityPubResponse(
+                await _build_followx_collection(
+                    db_session=db_session,
+                    model_cls=models.Following,
+                    path="/following",
+                    page=page,
+                    next_cursor=next_cursor,
+                )
+            )
+
+    if config.HIDES_FOLLOWING and not is_current_user_admin(request):
+        raise HTTPException(status_code=404)
 
     # We only show the most recent 20 follows on the public website
     following = (
@@ -545,13 +670,75 @@ async def _check_outbox_object_acl(
     raise HTTPException(status_code=404)
 
 
+async def _fetch_likes(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> list[models.InboxObject]:
+    return (
+        (
+            await db_session.scalars(
+                select(models.InboxObject)
+                .where(
+                    models.InboxObject.ap_type == "Like",
+                    models.InboxObject.activity_object_ap_id == outbox_object.ap_id,
+                    models.InboxObject.is_deleted.is_(False),
+                )
+                .options(joinedload(models.InboxObject.actor))
+                .order_by(models.InboxObject.ap_published_at.desc())
+                .limit(10)
+            )
+        )
+        .unique()
+        .all()
+    )
+
+
+async def _fetch_shares(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> list[models.InboxObject]:
+    return (
+        (
+            await db_session.scalars(
+                select(models.InboxObject)
+                .filter(
+                    models.InboxObject.ap_type == "Announce",
+                    models.InboxObject.activity_object_ap_id == outbox_object.ap_id,
+                    models.InboxObject.is_deleted.is_(False),
+                )
+                .options(joinedload(models.InboxObject.actor))
+                .order_by(models.InboxObject.ap_published_at.desc())
+                .limit(10)
+            )
+        )
+        .unique()
+        .all()
+    )
+
+
+async def _fetch_webmentions(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> list[models.Webmention]:
+    return (
+        await db_session.scalars(
+            select(models.Webmention)
+            .filter(
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.is_deleted.is_(False),
+            )
+            .limit(50)
+        )
+    ).all()
+
+
 @app.get("/o/{public_id}")
 async def outbox_by_public_id(
     public_id: str,
     request: Request,
     db_session: AsyncSession = Depends(get_db_session),
     httpsig_info: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
-) -> ActivityPubResponse | templates.TemplateResponse:
+) -> ActivityPubResponse | templates.TemplateResponse | RedirectResponse:
     maybe_object = (
         (
             await db_session.execute(
@@ -578,69 +765,148 @@ async def outbox_by_public_id(
     if is_activitypub_requested(request):
         return ActivityPubResponse(maybe_object.ap_object)
 
+    if maybe_object.ap_type == "Article":
+        return RedirectResponse(
+            f"{BASE_URL}/articles/{public_id[:7]}/{maybe_object.slug}",
+            status_code=301,
+        )
+
     replies_tree = await boxes.get_replies_tree(
         db_session,
         maybe_object,
         is_current_user_admin=is_current_user_admin(request),
     )
 
-    likes = (
+    webmentions = await _fetch_webmentions(db_session, maybe_object)
-        (
+    likes = await _fetch_likes(db_session, maybe_object)
-            await db_session.scalars(
+    shares = await _fetch_shares(db_session, maybe_object)
-                select(models.InboxObject)
-                .where(
-                    models.InboxObject.ap_type == "Like",
-                    models.InboxObject.activity_object_ap_id == maybe_object.ap_id,
-                    models.InboxObject.is_deleted.is_(False),
-                )
-                .options(joinedload(models.InboxObject.actor))
-                .order_by(models.InboxObject.ap_published_at.desc())
-                .limit(10)
-            )
-        )
-        .unique()
-        .all()
-    )
-
-    shares = (
-        (
-            await db_session.scalars(
-                select(models.InboxObject)
-                .filter(
-                    models.InboxObject.ap_type == "Announce",
-                    models.InboxObject.activity_object_ap_id == maybe_object.ap_id,
-                    models.InboxObject.is_deleted.is_(False),
-                )
-                .options(joinedload(models.InboxObject.actor))
-                .order_by(models.InboxObject.ap_published_at.desc())
-                .limit(10)
-            )
-        )
-        .unique()
-        .all()
-    )
-
-    webmentions = (
-        await db_session.scalars(
-            select(models.Webmention)
-            .filter(
-                models.Webmention.outbox_object_id == maybe_object.id,
-                models.Webmention.is_deleted.is_(False),
-            )
-            .limit(10)
-        )
-    ).all()
-
     return await templates.render_template(
         db_session,
         request,
         "object.html",
         {
-            "replies_tree": replies_tree,
+            "replies_tree": _merge_replies(replies_tree, webmentions),
             "outbox_object": maybe_object,
-            "likes": likes,
+            "likes": _merge_faces_from_inbox_object_and_webmentions(
-            "shares": shares,
+                likes,
-            "webmentions": webmentions,
+                webmentions,
+                models.WebmentionType.LIKE,
+            ),
+            "shares": _merge_faces_from_inbox_object_and_webmentions(
+                shares,
+                webmentions,
+                models.WebmentionType.REPOST,
+            ),
+            "webmentions": _filter_webmentions(webmentions),
+        },
+    )
+
+
+def _filter_webmentions(
+    webmentions: list[models.Webmention],
+) -> list[models.Webmention]:
+    return [
+        wm
+        for wm in webmentions
+        if wm.webmention_type
+        not in [
+            models.WebmentionType.LIKE,
+            models.WebmentionType.REPOST,
+            models.WebmentionType.REPLY,
+        ]
+    ]
+
+
+def _merge_faces_from_inbox_object_and_webmentions(
+    inbox_objects: list[models.InboxObject],
+    webmentions: list[models.Webmention],
+    webmention_type: models.WebmentionType,
+) -> list[Face]:
+    wm_faces = []
+    for wm in webmentions:
+        if wm.webmention_type != webmention_type:
+            continue
+        if face := Face.from_webmention(wm):
+            wm_faces.append(face)
+
+    return merge_faces(
+        [Face.from_inbox_object(obj) for obj in inbox_objects] + wm_faces
+    )
+
+
+def _merge_replies(
+    reply_tree_node: boxes.ReplyTreeNode,
+    webmentions: list[models.Webmention],
+) -> boxes.ReplyTreeNode:
+    # TODO: return None as we update the object in place
+    webmention_replies = []
+    for wm in [
+        wm for wm in webmentions if wm.webmention_type == models.WebmentionType.REPLY
+    ]:
+        if rep := WebmentionReply.from_webmention(wm):
+            webmention_replies.append(
+                boxes.ReplyTreeNode(
+                    ap_object=None,
+                    wm_reply=rep,
+                    is_requested=False,
+                    children=[],
+                )
+            )
+
+    reply_tree_node.children = sorted(
+        reply_tree_node.children + webmention_replies,
+        key=lambda node: node.published_at,
+        reverse=True,
+    )
+    return reply_tree_node
+
+
+@app.get("/articles/{short_id}/{slug}")
+async def article_by_slug(
+    short_id: str,
+    slug: str,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    httpsig_info: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
+) -> ActivityPubResponse | templates.TemplateResponse | RedirectResponse:
+    maybe_object = await boxes.get_outbox_object_by_slug_and_short_id(
+        db_session, slug, short_id
+    )
+    if not maybe_object:
+        raise HTTPException(status_code=404)
+
+    await _check_outbox_object_acl(request, db_session, maybe_object, httpsig_info)
+
+    if is_activitypub_requested(request):
+        return ActivityPubResponse(maybe_object.ap_object)
+
+    replies_tree = await boxes.get_replies_tree(
+        db_session,
+        maybe_object,
+        is_current_user_admin=is_current_user_admin(request),
+    )
+
+    likes = await _fetch_likes(db_session, maybe_object)
+    shares = await _fetch_shares(db_session, maybe_object)
+    webmentions = await _fetch_webmentions(db_session, maybe_object)
+    return await templates.render_template(
+        db_session,
+        request,
+        "object.html",
+        {
+            "replies_tree": _merge_replies(replies_tree, webmentions),
+            "outbox_object": maybe_object,
+            "likes": _merge_faces_from_inbox_object_and_webmentions(
+                likes,
+                webmentions,
+                models.WebmentionType.LIKE,
+            ),
+            "shares": _merge_faces_from_inbox_object_and_webmentions(
+                shares,
+                webmentions,
+                models.WebmentionType.REPOST,
+            ),
+            "webmentions": _filter_webmentions(webmentions),
         },
     )
 
@@ -676,7 +942,7 @@ async def tag_by_name(
     _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse:
     where = [
-        models.TaggedOutboxObject.tag == tag,
+        models.TaggedOutboxObject.tag == tag.lower(),
         models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC,
         models.OutboxObject.is_deleted.is_(False),
     ]
@@ -685,10 +951,10 @@ async def tag_by_name(
         .join(models.TaggedOutboxObject)
         .where(*where)
     )
-    if not tagged_count:
-        raise HTTPException(status_code=404)
-
     if is_activitypub_requested(request):
+        if not tagged_count:
+            raise HTTPException(status_code=404)
+
         outbox_object_ids = await db_session.execute(
             select(models.OutboxObject.ap_id)
             .join(
@@ -702,7 +968,7 @@ async def tag_by_name(
         return ActivityPubResponse(
             {
                 "@context": ap.AS_CTX,
-                "id": BASE_URL + f"/t/{tag}",
+                "id": BASE_URL + f"/t/{tag.lower()}",
                 "type": "OrderedCollection",
                 "totalItems": tagged_count,
                 "orderedItems": [
@@ -736,6 +1002,7 @@ async def tag_by_name(
             "request": request,
             "objects": outbox_objects,
         },
+        status_code=200 if len(outbox_objects) else 404,
     )
 
 
@@ -778,9 +1045,10 @@ async def get_remote_follow(
 @app.post("/remote_follow")
 async def post_remote_follow(
     request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
     csrf_check: None = Depends(verify_csrf_token),
     profile: str = Form(),
-) -> RedirectResponse:
+) -> templates.TemplateResponse:
     if not profile.startswith("@"):
         profile = f"@{profile}"
 
@@ -789,9 +1057,54 @@ async def post_remote_follow(
         # TODO(ts): error message to user
         raise HTTPException(status_code=404)
 
-    return RedirectResponse(
+    return await redirect_to_remote_instance(
+        request,
+        db_session,
+        remote_follow_template.format(uri=ID),
+    )
+
+
+@app.get("/remote_interaction")
+async def remote_interaction(
+    request: Request,
+    ap_id: str,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> templates.TemplateResponse:
+    outbox_object = await boxes.get_outbox_object_by_ap_id(
+        db_session,
+        ap_id,
+    )
+    if not outbox_object:
+        raise HTTPException(status_code=404)
+
+    return await templates.render_template(
+        db_session,
+        request,
+        "remote_interact.html",
+        {"outbox_object": outbox_object},
+    )
+
+
+@app.post("/remote_interaction")
+async def post_remote_interaction(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    csrf_check: None = Depends(verify_csrf_token),
+    profile: str = Form(),
+    ap_id: str = Form(),
+) -> templates.TemplateResponse:
+    if not profile.startswith("@"):
+        profile = f"@{profile}"
+
+    remote_follow_template = await get_remote_follow_template(profile)
+    if not remote_follow_template:
+        # TODO(ts): error message to user
+        raise HTTPException(status_code=404)
+
+    return await redirect_to_remote_instance(
+        request,
+        db_session,
         remote_follow_template.format(uri=ID),
-        status_code=302,
     )
 
 
@@ -867,7 +1180,11 @@ async def nodeinfo(
     )
 
 
-proxy_client = httpx.AsyncClient(follow_redirects=True, http2=True)
+proxy_client = httpx.AsyncClient(
+    http2=True,
+    follow_redirects=True,
+    timeout=httpx.Timeout(timeout=10.0),
+)
 
 
 async def _proxy_get(
@@ -904,37 +1221,62 @@ def _filter_proxy_resp_headers(
     }
 
 
-@app.get("/proxy/media/{encoded_url}")
+def _strip_content_type(headers: dict[str, str]) -> dict[str, str]:
-async def serve_proxy_media(request: Request, encoded_url: str) -> StreamingResponse:
+    return {k: v for k, v in headers.items() if k.lower() != "content-type"}
+
+
+def _add_cache_control(headers: dict[str, str]) -> dict[str, str]:
+    return {**headers, "Cache-Control": "max-age=31536000"}
+
+
+@app.get("/proxy/media/{exp}/{sig}/{encoded_url}")
+async def serve_proxy_media(
+    request: Request,
+    exp: int,
+    sig: str,
+    encoded_url: str,
+) -> StreamingResponse | PlainTextResponse:
     # Decode the base64-encoded URL
     url = base64.urlsafe_b64decode(encoded_url).decode()
     check_url(url)
+    media.verify_proxied_media_sig(exp, url, sig)
 
     proxy_resp = await _proxy_get(request, url, stream=True)
 
+    if proxy_resp.status_code >= 300:
+        logger.info(f"failed to proxy {url}, got {proxy_resp.status_code}")
+        return PlainTextResponse(
+            "proxy error",
+            status_code=proxy_resp.status_code,
+        )
+
     return StreamingResponse(
         proxy_resp.aiter_raw(),
         status_code=proxy_resp.status_code,
-        headers=_filter_proxy_resp_headers(
+        headers=_add_cache_control(
-            proxy_resp,
+            _filter_proxy_resp_headers(
-            [
+                proxy_resp,
-                "content-length",
+                [
-                "content-type",
+                    "content-length",
-                "content-range",
+                    "content-type",
-                "accept-ranges" "etag",
+                    "content-range",
-                "cache-control",
+                    "accept-ranges",
-                "expires",
+                    "etag",
-                "date",
+                    "expires",
-                "last-modified",
+                    "date",
-            ],
+                    "last-modified",
+                ],
+            )
         ),
         background=BackgroundTask(proxy_resp.aclose),
     )
 
 
-@app.get("/proxy/media/{encoded_url}/{size}")
+@app.get("/proxy/media/{exp}/{sig}/{encoded_url}/{size}")
 async def serve_proxy_media_resized(
     request: Request,
+    exp: int,
+    sig: str,
     encoded_url: str,
     size: int,
 ) -> PlainTextResponse:
@@ -944,6 +1286,7 @@ async def serve_proxy_media_resized(
     # Decode the base64-encoded URL
     url = base64.urlsafe_b64decode(encoded_url).decode()
     check_url(url)
+    media.verify_proxied_media_sig(exp, url, sig)
 
     if cached_resp := _RESIZED_CACHE.get((url, size)):
         resized_content, resized_mimetype, resp_headers = cached_resp
@@ -954,22 +1297,24 @@ async def serve_proxy_media_resized(
         )
 
     proxy_resp = await _proxy_get(request, url, stream=False)
-    if proxy_resp.status_code != 200:
+    if proxy_resp.status_code >= 300:
+        logger.info(f"failed to proxy {url}, got {proxy_resp.status_code}")
         return PlainTextResponse(
-            proxy_resp.content,
+            "proxy error",
             status_code=proxy_resp.status_code,
         )
 
     # Filter the headers
-    proxy_resp_headers = _filter_proxy_resp_headers(
+    proxy_resp_headers = _add_cache_control(
-        proxy_resp,
+        _filter_proxy_resp_headers(
-        [
+            proxy_resp,
-            "content-type",
+            [
-            "etag",
+                "content-type",
-            "cache-control",
+                "etag",
-            "expires",
+                "expires",
-            "last-modified",
+                "last-modified",
-        ],
+            ],
+        )
     )
 
     try:
@@ -978,23 +1323,31 @@ async def serve_proxy_media_resized(
         if getattr(i, "is_animated", False):
             raise ValueError
         i.thumbnail((size, size))
-        resized_buf = BytesIO()
+        is_webp = False
-        i.save(resized_buf, format=i.format)
+        try:
+            resized_buf = BytesIO()
+            i.save(resized_buf, format="webp")
+            is_webp = True
+        except Exception:
+            logger.exception("Failed to convert to webp")
+            resized_buf = BytesIO()
+            i.save(resized_buf, format=i.format)
         resized_buf.seek(0)
         resized_content = resized_buf.read()
-        resized_mimetype = i.get_format_mimetype()  # type: ignore
+        resized_mimetype = (
-
+            "image/webp" if is_webp else i.get_format_mimetype()  # type: ignore
+        )
         # Only cache images < 1MB
         if len(resized_content) < 2**20:
             _RESIZED_CACHE[(url, size)] = (
                 resized_content,
                 resized_mimetype,
-                proxy_resp_headers,
+                _strip_content_type(proxy_resp_headers),
             )
         return PlainTextResponse(
             resized_content,
             media_type=resized_mimetype,
-            headers=proxy_resp_headers,
+            headers=_strip_content_type(proxy_resp_headers),
         )
     except ValueError:
         return PlainTextResponse(
@@ -1028,6 +1381,7 @@ async def serve_attachment(
     return FileResponse(
         UPLOAD_DIR / content_hash,
         media_type=upload.content_type,
+        headers={"Cache-Control": "max-age=31536000"},
     )
 
 
@@ -1049,7 +1403,8 @@ async def serve_attachment_thumbnail(
 
     return FileResponse(
         UPLOAD_DIR / (content_hash + "_resized"),
-        media_type=upload.content_type,
+        media_type="image/webp",
+        headers={"Cache-Control": "max-age=31536000"},
     )
 
 
@@ -1059,6 +1414,7 @@ async def robots_file():
 Disallow: /followers
 Disallow: /following
 Disallow: /admin
+Disallow: /remote_interaction
 Disallow: /remote_follow"""
 
 
@@ -1108,7 +1464,7 @@ async def json_feed(
                 ],
             }
         )
-    return {
+    result = {
         "version": "https://jsonfeed.org/version/1",
         "title": f"{LOCAL_ACTOR.display_name}'s microblog'",
         "home_page_url": LOCAL_ACTOR.url,
@@ -1116,10 +1472,12 @@ async def json_feed(
         "author": {
             "name": LOCAL_ACTOR.display_name,
             "url": LOCAL_ACTOR.url,
-            "avatar": LOCAL_ACTOR.icon_url,
         },
         "items": data,
     }
+    if LOCAL_ACTOR.icon_url:
+        result["author"]["avatar"] = LOCAL_ACTOR.icon_url  # type: ignore
+    return result
 
 
 async def _gen_rss_feed(
@@ -1131,7 +1489,8 @@ async def _gen_rss_feed(
     fg.description(f"{LOCAL_ACTOR.display_name}'s microblog")
     fg.author({"name": LOCAL_ACTOR.display_name})
     fg.link(href=LOCAL_ACTOR.url, rel="alternate")
-    fg.logo(LOCAL_ACTOR.icon_url)
+    if LOCAL_ACTOR.icon_url:
+        fg.logo(LOCAL_ACTOR.icon_url)
     fg.language("en")
 
     outbox_objects = await _get_outbox_for_feed(db_session)

app/media.py

@@ -1,15 +1,44 @@
 import base64
+import time
 
 from app.config import BASE_URL
+from app.config import hmac_sha256
 
 SUPPORTED_RESIZE = [50, 740]
+EXPIRY_PERIOD = 86400
+EXPIRY_LENGTH = 7
+
+
+class InvalidProxySignatureError(Exception):
+    pass
+
+
+def proxied_media_sig(expires: int, url: str) -> str:
+    hm = hmac_sha256()
+    hm.update(f"{expires}".encode())
+    hm.update(b"|")
+    hm.update(url.encode())
+    return base64.urlsafe_b64encode(hm.digest()).decode()
+
+
+def verify_proxied_media_sig(expires: int, url: str, sig: str) -> None:
+    now = int(time.time() / EXPIRY_PERIOD)
+    expected = proxied_media_sig(expires, url)
+    if now > expires or sig != expected:
+        raise InvalidProxySignatureError("invalid or expired media")
 
 
 def proxied_media_url(url: str) -> str:
     if url.startswith(BASE_URL):
         return url
+    expires = int(time.time() / EXPIRY_PERIOD) + EXPIRY_LENGTH
+    sig = proxied_media_sig(expires, url)
 
-    return "/proxy/media/" + base64.urlsafe_b64encode(url.encode()).decode()
+    return (
+        BASE_URL
+        + f"/proxy/media/{expires}/{sig}/"
+        + base64.urlsafe_b64encode(url.encode()).decode()
+    )
 
 
 def resized_media_url(url: str, size: int) -> str:

app/models.py

@@ -45,7 +45,7 @@ class Actor(Base, BaseActor):
     created_at = Column(DateTime(timezone=True), nullable=False, default=now)
     updated_at = Column(DateTime(timezone=True), nullable=False, default=now)
 
-    ap_id = Column(String, unique=True, nullable=False, index=True)
+    ap_id: Mapped[str] = Column(String, unique=True, nullable=False, index=True)
     ap_actor: Mapped[ap.RawObject] = Column(JSON, nullable=False)
     ap_type = Column(String, nullable=False)
 
@@ -75,7 +75,7 @@ class InboxObject(Base, BaseObject):
 
     ap_actor_id = Column(String, nullable=False)
     ap_type = Column(String, nullable=False, index=True)
-    ap_id = Column(String, nullable=False, unique=True, index=True)
+    ap_id: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
     ap_context = Column(String, nullable=True)
     ap_published_at = Column(DateTime(timezone=True), nullable=False)
     ap_object: Mapped[ap.RawObject] = Column(JSON, nullable=False)
@@ -126,7 +126,7 @@ class InboxObject(Base, BaseObject):
     is_deleted = Column(Boolean, nullable=False, default=False)
     is_transient = Column(Boolean, nullable=False, default=False, server_default="0")
 
-    replies_count = Column(Integer, nullable=False, default=0)
+    replies_count: Mapped[int] = Column(Integer, nullable=False, default=0)
 
     og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
 
@@ -158,9 +158,10 @@ class OutboxObject(Base, BaseObject):
     is_hidden_from_homepage = Column(Boolean, nullable=False, default=False)
 
     public_id = Column(String, nullable=False, index=True)
+    slug = Column(String, nullable=True, index=True)
 
     ap_type = Column(String, nullable=False, index=True)
-    ap_id = Column(String, nullable=False, unique=True, index=True)
+    ap_id: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
     ap_context = Column(String, nullable=True)
     ap_object: Mapped[ap.RawObject] = Column(JSON, nullable=False)
 
@@ -176,7 +177,7 @@ class OutboxObject(Base, BaseObject):
 
     likes_count = Column(Integer, nullable=False, default=0)
     announces_count = Column(Integer, nullable=False, default=0)
-    replies_count = Column(Integer, nullable=False, default=0)
+    replies_count: Mapped[int] = Column(Integer, nullable=False, default=0)
     webmentions_count: Mapped[int] = Column(
         Integer, nullable=False, default=0, server_default="0"
     )
@@ -250,6 +251,8 @@ class OutboxObject(Base, BaseObject):
                         "mediaType": attachment.upload.content_type,
                         "name": attachment.alt or attachment.filename,
                         "url": url,
+                        "width": attachment.upload.width,
+                        "height": attachment.upload.height,
                         "proxiedUrl": url,
                         "resizedUrl": BASE_URL
                         + (
@@ -281,6 +284,13 @@ class OutboxObject(Base, BaseObject):
     def is_from_outbox(self) -> bool:
         return True
 
+    @property
+    def url(self) -> str | None:
+        # XXX: rewrite old URL here for compat
+        if self.ap_type == "Article" and self.slug and self.public_id:
+            return f"{BASE_URL}/articles/{self.public_id[:7]}/{self.slug}"
+        return super().url
+
 
 class Follower(Base):
     __tablename__ = "follower"
@@ -458,6 +468,14 @@ class IndieAuthAccessToken(Base):
     is_revoked = Column(Boolean, nullable=False, default=False)
 
 
+@enum.unique
+class WebmentionType(str, enum.Enum):
+    UNKNOWN = "unknown"
+    LIKE = "like"
+    REPLY = "reply"
+    REPOST = "repost"
+
+
 class Webmention(Base):
     __tablename__ = "webmention"
     __table_args__ = (UniqueConstraint("source", "target", name="uix_source_target"),)
@@ -474,6 +492,8 @@ class Webmention(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
     outbox_object = relationship(OutboxObject, uselist=False)
 
+    webmention_type = Column(Enum(WebmentionType), nullable=True)
+
     @property
     def as_facepile_item(self) -> webmentions.Webmention | None:
         if not self.source_microformats:
@@ -483,6 +503,7 @@ class Webmention(Base):
                 self.source_microformats["items"], self.source
             )
         except Exception:
+            # TODO: return a facepile with the unknown image
             logger.warning(
                 f"Failed to generate facefile item for Webmention id={self.id}"
             )
@@ -537,6 +558,8 @@ class NotificationType(str, enum.Enum):
     FOLLOW_REQUEST_ACCEPTED = "follow_request_accepted"
     FOLLOW_REQUEST_REJECTED = "follow_request_rejected"
 
+    MOVE = "move"
+
     LIKE = "like"
     UNDO_LIKE = "undo_like"
 
@@ -549,6 +572,14 @@ class NotificationType(str, enum.Enum):
     UPDATED_WEBMENTION = "updated_webmention"
     DELETED_WEBMENTION = "deleted_webmention"
 
+    # incoming
+    BLOCKED = "blocked"
+    UNBLOCKED = "unblocked"
+
+    # outgoing
+    BLOCK = "block"
+    UNBLOCK = "unblock"
+
 
 class Notification(Base):
     __tablename__ = "notifications"

app/outgoing_activities.py

@@ -67,6 +67,7 @@ async def _send_actor_update_if_needed(
     logger.info("Will send an Update for the local actor")
 
     from app.boxes import allocate_outbox_id
+    from app.boxes import compute_all_known_recipients
     from app.boxes import outbox_object_id
     from app.boxes import save_outbox_object
 
@@ -85,24 +86,8 @@ async def _send_actor_update_if_needed(
 
     # Send the update to the followers collection and all the actor we have ever
     # contacted
-    followers = (
+    recipients = await compute_all_known_recipients(db_session)
-        (
+    for rcp in recipients:
-            await db_session.scalars(
-                select(models.Follower).options(joinedload(models.Follower.actor))
-            )
-        )
-        .unique()
-        .all()
-    )
-    for rcp in {
-        follower.actor.shared_inbox_url or follower.actor.inbox_url
-        for follower in followers
-    } | {
-        row.recipient
-        for row in await db_session.execute(
-            select(func.distinct(models.OutgoingActivity.recipient).label("recipient"))
-        )
-    }:  # type: ignore
         await new_outgoing_activity(
             db_session,
             recipient=rcp,

app/scss/main.scss

@@ -13,6 +13,40 @@ $code-highlight-background: #f0f0f0;
 // Load custom theme
 @import "theme.scss";
 
+.primary-color {
+  color: $primary-color;
+}
+
+#admin {
+  .admin-menu {
+    margin-bottom: 30px;
+    padding: 0 20px;
+  }
+}
+
+.empty-state {
+  padding: 20px;
+}
+
+.public-top-menu {
+  margin: 30px 0 0 0;
+}
+
+.width-95 {
+  width: 95%;
+}
+
+.bold {
+  font-weight: bold;
+}
+
+.admin-new {
+  textarea {
+    font-size: 1.2em;
+    width: 95%;
+  }
+}
+
 .show-more-wrapper {
   .p-summary {
     display: inline-block;
@@ -61,13 +95,6 @@ blockquote {
     color: $muted-color;
 }
 
-.poll-bar {
-   width:100%;height:20px;
-   line {
-        stroke: $secondary-color;
-   }
-}
-
 .light-background {
     background: $light-background;
 }
@@ -116,6 +143,9 @@ dl {
     strong {
         color: $primary-color;
     }
+    span {
+      color: $muted-color;
+    }
 }
 
 div.highlight {
@@ -182,6 +212,7 @@ a {
   }
 }
 #main {
+    display: flex;
     flex: 1; 
 }
 main {
@@ -189,11 +220,36 @@ main {
     max-width: 1000px;
     margin: 30px auto;
 }
+
+.main-flex {
+    display: flex;
+    flex: 1;
+}
+
+.centered {
+  display: flex;
+  flex: 1;
+  justify-content: center;
+  align-items: center;
+  div {
+    display: block;
+  }
+}
+
 footer {
-    width: 100%;
+  width: 100%;
-    max-width: 1000px;
+  max-width: 1000px;
-    margin: 20px auto;
+  margin: 20px auto;
-    color: $muted-color;
+  color: $muted-color;
+  p {
+    margin: 0;    
+  }
+}
+.tiny-actor-icon {
+  max-width: 24px;
+  max-height: 24px;
+  position: relative;
+  top: 5px;
 }
 .actor-box {
   display: flex;
@@ -217,6 +273,9 @@ footer {
     padding: 0 20px;
     li {
         display: block;
+        span {
+          padding-right:10px;
+        }
     }
 }
 
@@ -251,6 +310,57 @@ footer {
     margin: 20px 0;
 }
 
+.show-hide-sensitive-btn {
+  display:inline-block;
+}
+
+.no-margin-top {
+  margin-top: 0;
+}
+
+.float-right {
+  float: right;
+}
+
+ul.poll-items {
+  list-style-type: none;
+  padding: 0;
+  li {
+    display: block;
+    p {
+      margin: 20px 0 10px 0;
+      .poll-vote {
+        padding-left: 20px;
+      }
+    }
+
+    .poll-bar {
+       width:100%;height:20px;
+       line {
+            stroke: $secondary-color;
+            stroke-width: 20px;
+       }
+    }
+
+  }
+}
+
+.attachment-wrapper {
+  .attachment-item {
+    margin-top: 20px;
+  }
+  img.attachment {
+    margin: 0;
+  }
+  a.attachment {
+    display: inline-block;
+    margin-bottom: 15px;
+  }
+  audio.attachment {
+    width: 480px;
+  }
+}
+
 nav {
     form {
         margin: 15px 0;
@@ -278,7 +388,7 @@ nav.flexbox {
             margin-right: 0px;
         }
     }
-    a {
+    a:not(.label-btn) {
         color: $primary-color;
         text-decoration: none;
         &:hover, &:active {
@@ -286,32 +396,38 @@ nav.flexbox {
             text-decoration: underline;
         }
     }
-    a.active {
+    a.active:not(.label-btn) {
         color: $secondary-color;
         font-weight: bold;
     }
 }
 
+// after nav.flexbox to override default behavior
+a.label-btn {
+  color: $form-text-color;
+  &:hover {
+    text-decoration: none;
+    color: $form-text-color;
+  }
+}
+
 .ap-object {
     margin: 15px 0;
     padding: 20px;
-    .in-reply-to {
-        color: $muted-color;
-        &:hover {
-            color: $secondary-color;
-            text-decoration: underline;
-        }
-    }
     nav {
         color: $muted-color;
     }
+    .in-reply-to {
+        display: inline;
+        color: $muted-color;
+    }
     .e-content, .activity-og-meta {
         a:hover {
             text-decoration: underline;
         }
     }
     .activity-attachment {
-        margin: 30px 0;
+        margin: 30px 0 20px 0;
         img, audio, video {
             width: 100%;
             max-width: 740px;
@@ -322,6 +438,20 @@ nav.flexbox {
         max-width: 740px;
     }
 }
+
+.activity-og-meta {
+  display: flex;
+  column-gap: 20px;
+  margin: 20px 0;
+  img {
+    max-width: 200px;
+    max-height: 100px;
+  }
+  small {
+    display: block;
+  }
+}
+
 .ap-object-expanded {
     border: 2px dashed $secondary-color;
 }
@@ -337,6 +467,9 @@ nav.flexbox {
   span {
     color: $muted-color;
   }
+  span.new {
+    color: $secondary-color;
+  }
 }
 .actor-metadata {
     color: $muted-color;
@@ -344,3 +477,74 @@ nav.flexbox {
 .emoji, .custom-emoji {
   max-width: 25px;
 }
+
+.indieauth-box {
+  display: flex;
+  column-gap: 20px;
+
+  .indieauth-logo {
+    flex: initial;
+    width: 100px;
+    img {
+      max-width: 100px;
+    }
+  }
+  .indieauth-details {
+    flex: 1;
+    div {
+      padding-left: 20px;
+      a {
+        font-size: 1.2em;
+        font-weight: 600;
+      }
+    }
+  }
+}
+
+.public-interactions {
+  display: flex;
+  column-gap: 20px;
+  flex-wrap: wrap;
+  margin-top: 20px;
+  .interactions-block {
+    flex: 0 1 30%;
+    max-width: 50%;
+    .facepile-wrapper {
+      display: flex;
+      column-gap: 20px;
+      row-gap: 20px;
+      flex-wrap: wrap;
+      margin-top: 20px;
+      a {
+        height: 50px;
+        img {
+          max-width: 50px;
+        }
+      }
+      .and-x-more {
+        display: inline-block;
+        align-self: center;
+      }
+    }
+  }
+}
+
+.error-title {
+  a {
+    text-decoration: underline;
+  }
+}
+
+.ap-place {
+  h3 {
+    display: inline;
+    font-weight: normal;
+  }
+  h3::after {
+    content: ': ';
+  }
+}
+
+.margin-top-20 {
+  margin-top: 20px;
+}

app/source.py

@@ -1,93 +1,212 @@
 import re
+import typing
 
-from markdown import markdown
+from loguru import logger
+from mistletoe import Document  # type: ignore
+from mistletoe.html_renderer import HTMLRenderer  # type: ignore
+from mistletoe.span_token import SpanToken  # type: ignore
+from pygments import highlight  # type: ignore
+from pygments.formatters import HtmlFormatter  # type: ignore
+from pygments.lexers import get_lexer_by_name as get_lexer  # type: ignore
+from pygments.lexers import guess_lexer  # type: ignore
 from sqlalchemy import select
 
-from app import models
 from app import webfinger
-from app.actor import Actor
-from app.actor import fetch_actor
 from app.config import BASE_URL
+from app.config import CODE_HIGHLIGHTING_THEME
 from app.database import AsyncSession
 from app.utils import emoji
 
+if typing.TYPE_CHECKING:
+    from app.actor import Actor
 
-def _set_a_attrs(attrs, new=False):
+_FORMATTER = HtmlFormatter(style=CODE_HIGHLIGHTING_THEME)
-    attrs[(None, "target")] = "_blank"
-    attrs[(None, "class")] = "external"
-    attrs[(None, "rel")] = "noopener"
-    attrs[(None, "title")] = attrs[(None, "href")]
-    return attrs
-
-
 _HASHTAG_REGEX = re.compile(r"(#[\d\w]+)")
-_MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+")
+_MENTION_REGEX = re.compile(r"(@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+)")
+_URL_REGEX = re.compile(
+    "(https?:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*))"  # noqa: E501
+)
 
 
-async def _hashtagify(
+class AutoLink(SpanToken):
-    db_session: AsyncSession, content: str
+    parse_inner = False
-) -> tuple[str, list[dict[str, str]]]:
+    precedence = 1
-    tags = []
+    pattern = _URL_REGEX
-    hashtags = re.findall(_HASHTAG_REGEX, content)
+
-    hashtags = sorted(set(hashtags), reverse=True)  # unique tags, longest first
+    def __init__(self, match_obj: re.Match) -> None:
-    for hashtag in hashtags:
+        self.target = match_obj.group()
-        tag = hashtag[1:]
-        link = f'<a href="{BASE_URL}/t/{tag}" class="mention hashtag" rel="tag">#<span>{tag}</span></a>'  # noqa: E501
-        tags.append(dict(href=f"{BASE_URL}/t/{tag}", name=hashtag, type="Hashtag"))
-        content = content.replace(hashtag, link)
-    return content, tags
 
 
-async def _mentionify(
+class Mention(SpanToken):
+    parse_inner = False
+    precedence = 10
+    pattern = _MENTION_REGEX
+
+    def __init__(self, match_obj: re.Match) -> None:
+        self.target = match_obj.group()
+
+
+class Hashtag(SpanToken):
+    parse_inner = False
+    precedence = 10
+    pattern = _HASHTAG_REGEX
+
+    def __init__(self, match_obj: re.Match) -> None:
+        self.target = match_obj.group()
+
+
+class CustomRenderer(HTMLRenderer):
+    def __init__(
+        self,
+        mentioned_actors: dict[str, "Actor"] = {},
+        enable_mentionify: bool = True,
+        enable_hashtagify: bool = True,
+    ) -> None:
+        extra_tokens = []
+        if enable_mentionify:
+            extra_tokens.append(Mention)
+        if enable_hashtagify:
+            extra_tokens.append(Hashtag)
+        super().__init__(AutoLink, *extra_tokens)
+
+        self.tags: list[dict[str, str]] = []
+        self.mentioned_actors = mentioned_actors
+
+    def render_auto_link(self, token: AutoLink) -> str:
+        template = '<a href="{target}" rel="noopener">{inner}</a>'
+        target = self.escape_url(token.target)
+        return template.format(target=target, inner=target)
+
+    def render_mention(self, token: Mention) -> str:
+        mention = token.target
+        suffix = ""
+        if mention.endswith("."):
+            mention = mention[:-1]
+            suffix = "."
+        actor = self.mentioned_actors.get(mention)
+        if not actor:
+            return mention
+
+        self.tags.append(dict(type="Mention", href=actor.ap_id, name=mention))
+
+        link = f'<span class="h-card"><a href="{actor.url}" class="u-url mention">{actor.handle}</a></span>{suffix}'  # noqa: E501
+        return link
+
+    def render_hashtag(self, token: Hashtag) -> str:
+        tag = token.target[1:]
+        link = f'<a href="{BASE_URL}/t/{tag.lower()}" class="mention hashtag" rel="tag">#<span>{tag}</span></a>'  # noqa: E501
+        self.tags.append(
+            dict(
+                href=f"{BASE_URL}/t/{tag.lower()}",
+                name=token.target.lower(),
+                type="Hashtag",
+            )
+        )
+        return link
+
+    def render_block_code(self, token: typing.Any) -> str:
+        code = token.children[0].content
+        lexer = get_lexer(token.language) if token.language else guess_lexer(code)
+        return highlight(code, lexer, _FORMATTER)
+
+
+async def _prefetch_mentioned_actors(
     db_session: AsyncSession,
     content: str,
-) -> tuple[str, list[dict[str, str]], list[Actor]]:
+) -> dict[str, "Actor"]:
-    tags = []
+    from app import models
-    mentioned_actors = []
+    from app.actor import fetch_actor
+
+    actors = {}
+
     for mention in re.findall(_MENTION_REGEX, content):
-        _, username, domain = mention.split("@")
+        if mention in actors:
-        actor = (
+            continue
-            await db_session.execute(
-                select(models.Actor).where(models.Actor.handle == mention)
-            )
-        ).scalar_one_or_none()
-        if not actor:
-            actor_url = await webfinger.get_actor_url(mention)
-            if not actor_url:
-                # FIXME(ts): raise an error?
-                continue
-            actor = await fetch_actor(db_session, actor_url)
 
-        mentioned_actors.append(actor)
+        # XXX: the regex catches stuff like `@toto@example.com.`
-        tags.append(dict(type="Mention", href=actor.ap_id, name=mention))
+        if mention.endswith("."):
+            mention = mention[:-1]
 
-        link = f'<span class="h-card"><a href="{actor.url}" class="u-url mention">{actor.handle}</a></span>'  # noqa: E501
+        try:
-        content = content.replace(mention, link)
+            _, username, domain = mention.split("@")
-    return content, tags, mentioned_actors
+            actor = (
+                await db_session.execute(
+                    select(models.Actor).where(
+                        models.Actor.handle == mention,
+                        models.Actor.is_deleted.is_(False),
+                    )
+                )
+            ).scalar_one_or_none()
+            if not actor:
+                actor_url = await webfinger.get_actor_url(mention)
+                if not actor_url:
+                    # FIXME(ts): raise an error?
+                    continue
+                actor = await fetch_actor(db_session, actor_url)
+
+            actors[mention] = actor
+        except Exception:
+            logger.exception(f"Failed to prefetch {mention}")
+
+    return actors
+
+
+def hashtagify(
+    content: str,
+) -> tuple[str, list[dict[str, str]]]:
+    tags = []
+    with CustomRenderer(
+        mentioned_actors={},
+        enable_mentionify=False,
+        enable_hashtagify=True,
+    ) as renderer:
+        rendered_content = renderer.render(Document(content))
+        tags.extend(renderer.tags)
+
+    # Handle custom emoji
+    tags.extend(emoji.tags(content))
+
+    return rendered_content, tags
 
 
 async def markdownify(
     db_session: AsyncSession,
     content: str,
-    mentionify: bool = True,
+    enable_mentionify: bool = True,
-    hashtagify: bool = True,
+    enable_hashtagify: bool = True,
-) -> tuple[str, list[dict[str, str]], list[Actor]]:
+) -> tuple[str, list[dict[str, str]], list["Actor"]]:
     """
     >>> content, tags = markdownify("Hello")
 
     """
     tags = []
-    mentioned_actors: list[Actor] = []
+    mentioned_actors: dict[str, "Actor"] = {}
-    if hashtagify:
+    if enable_mentionify:
-        content, hashtag_tags = await _hashtagify(db_session, content)
+        mentioned_actors = await _prefetch_mentioned_actors(db_session, content)
-        tags.extend(hashtag_tags)
+
-    if mentionify:
+    with CustomRenderer(
-        content, mention_tags, mentioned_actors = await _mentionify(db_session, content)
+        mentioned_actors=mentioned_actors,
-        tags.extend(mention_tags)
+        enable_mentionify=enable_mentionify,
+        enable_hashtagify=enable_hashtagify,
+    ) as renderer:
+        rendered_content = renderer.render(Document(content))
+        tags.extend(renderer.tags)
 
     # Handle custom emoji
     tags.extend(emoji.tags(content))
 
-    content = markdown(content, extensions=["mdx_linkify", "fenced_code"])
+    return rendered_content, dedup_tags(tags), list(mentioned_actors.values())
 
-    return content, tags, mentioned_actors
+
+def dedup_tags(tags: list[dict[str, str]]) -> list[dict[str, str]]:
+    idx = set()
+    deduped_tags = []
+    for tag in tags:
+        tag_idx = (tag["type"], tag["name"])
+        if tag_idx in idx:
+            continue
+
+        idx.add(tag_idx)
+        deduped_tags.append(tag)
+
+    return deduped_tags

app/static/common-admin.js

@@ -0,0 +1,11 @@
+document.addEventListener('DOMContentLoaded', (ev) => {
+  // Add confirm to "delete" button next to outbox objects
+  var forms = document.getElementsByClassName("object-delete-form")
+  for (var i = 0; i < forms.length; i++) {
+    forms[i].addEventListener('submit', (ev) => {
+      if (!confirm('Do you really want to delete this object?')) {
+        ev.preventDefault();
+      };
+    });
+  }
+});

app/templates.py

@@ -1,4 +1,3 @@
-import base64
 from datetime import datetime
 from datetime import timezone
 from functools import lru_cache
@@ -26,8 +25,9 @@ from app.actor import LOCAL_ACTOR
 from app.ap_object import Attachment
 from app.ap_object import Object
 from app.config import BASE_URL
-from app.config import CSS_HASH
+from app.config import CUSTOM_FOOTER
 from app.config import DEBUG
+from app.config import SESSION_TIMEOUT
 from app.config import VERSION
 from app.config import generate_csrf_token
 from app.config import session_serializer
@@ -39,7 +39,7 @@ from app.utils.highlight import HIGHLIGHT_CSS
 from app.utils.highlight import highlight
 
 _templates = Jinja2Templates(
-    directory="app/templates",
+    directory=["data/templates", "app/templates"],  # type: ignore  # bad typing
     trim_blocks=True,
     lstrip_blocks=True,
 )
@@ -59,13 +59,8 @@ def _filter_domain(text: str) -> str:
 
 def _media_proxy_url(url: str | None) -> str:
     if not url:
-        return "/static/nopic.png"
+        return BASE_URL + "/static/nopic.png"
-
+    return proxied_media_url(url)
-    if url.startswith(BASE_URL):
-        return url
-
-    encoded_url = base64.urlsafe_b64encode(url.encode()).decode()
-    return f"/proxy/media/{encoded_url}"
 
 
 def is_current_user_admin(request: Request) -> bool:
@@ -75,10 +70,10 @@ def is_current_user_admin(request: Request) -> bool:
         try:
             loaded_session = session_serializer.loads(
                 session_cookie,
-                max_age=3600 * 12,
+                max_age=SESSION_TIMEOUT,
             )
         except Exception:
-            pass
+            logger.exception("Failed to validate session timeout")
         else:
             is_admin = loaded_session.get("is_logged_in")
 
@@ -90,6 +85,8 @@ async def render_template(
     request: Request,
     template: str,
     template_args: dict[str, Any] | None = None,
+    status_code: int = 200,
+    headers: dict[str, str] | None = None,
 ) -> TemplateResponse:
     if template_args is None:
         template_args = {}
@@ -103,7 +100,6 @@ async def render_template(
             "request": request,
             "debug": DEBUG,
             "microblogpub_version": VERSION,
-            "css_hash": CSS_HASH,
             "is_admin": is_admin,
             "csrf_token": generate_csrf_token(),
             "highlight_css": HIGHLIGHT_CSS,
@@ -131,8 +127,11 @@ async def render_template(
                 select(func.count(models.Following.id))
             ),
             "actor_types": ap.ACTOR_TYPES,
+            "custom_footer": CUSTOM_FOOTER,
             **template_args,
         },
+        status_code=status_code,
+        headers=headers,
     )
 
 
@@ -289,6 +288,10 @@ ALLOWED_ATTRIBUTES: dict[str, list[str] | Callable[[str, str, str], bool]] = {
 }
 
 
+def _allow_all_attributes(tag: Any, name: Any, value: Any) -> bool:
+    return True
+
+
 @lru_cache(maxsize=256)
 def _update_inline_imgs(content):
     soup = BeautifulSoup(content, "html5lib")
@@ -318,7 +321,11 @@ def _clean_html(html: str, note: Object) -> str:
                         _update_inline_imgs(highlight(html))
                     ),
                     tags=ALLOWED_TAGS,
-                    attributes=ALLOWED_ATTRIBUTES,
+                    attributes=(
+                        _allow_all_attributes
+                        if note.ap_id.startswith(config.ID)
+                        else ALLOWED_ATTRIBUTES
+                    ),
                     strip=True,
                 ),
                 note,
@@ -329,6 +336,14 @@ def _clean_html(html: str, note: Object) -> str:
         raise
 
 
+def _clean_html_wm(html: str) -> str:
+    return bleach.clean(
+        html,
+        attributes=ALLOWED_ATTRIBUTES,
+        strip=True,
+    )
+
+
 def _timeago(original_dt: datetime) -> str:
     dt = original_dt
     if dt.tzinfo:
@@ -377,8 +392,8 @@ def _html2text(content: str) -> str:
 
 
 def _replace_emoji(u: str, _) -> str:
-    filename = hex(ord(u))[2:]
+    filename = "-".join(hex(ord(c))[2:] for c in u)
-    return config.EMOJI_TPL.format(filename=filename, raw=u)
+    return config.EMOJI_TPL.format(base_url=BASE_URL, filename=filename, raw=u)
 
 
 def _emojify(text: str, is_local: bool) -> str:
@@ -405,6 +420,7 @@ def _poll_item_pct(item: ap.RawObject, voters_count: int) -> int:
 _templates.env.filters["domain"] = _filter_domain
 _templates.env.filters["media_proxy_url"] = _media_proxy_url
 _templates.env.filters["clean_html"] = _clean_html
+_templates.env.filters["clean_html_wm"] = _clean_html_wm
 _templates.env.filters["timeago"] = _timeago
 _templates.env.filters["format_date"] = _format_date
 _templates.env.filters["has_media_type"] = _has_media_type
@@ -414,3 +430,10 @@ _templates.env.filters["pluralize"] = _pluralize
 _templates.env.filters["parse_datetime"] = _parse_datetime
 _templates.env.filters["poll_item_pct"] = _poll_item_pct
 _templates.env.filters["privacy_replace_url"] = privacy_replace.replace_url
+_templates.env.globals["JS_HASH"] = config.JS_HASH
+_templates.env.globals["CSS_HASH"] = config.CSS_HASH
+_templates.env.globals["BASE_URL"] = config.BASE_URL
+_templates.env.globals["HIDES_FOLLOWERS"] = config.HIDES_FOLLOWERS
+_templates.env.globals["HIDES_FOLLOWING"] = config.HIDES_FOLLOWING
+_templates.env.globals["NAVBAR_ITEMS"] = config.NavBarItems
+_templates.env.globals["ICON_URL"] = config.CONFIG.icon_url

app/templates/admin_direct_messages.html

@@ -10,7 +10,9 @@
 {% for anybox_object, convo, actors in threads %}
     <div class="actor-action">
     With {% for actor in actors %}
-        <a href="">{{ actor.handle }}</a>
+    <a href="{{ url_for("admin_profile") }}?actor_id={{ actor.ap_id }}">
+        {{ actor.handle }}
+    </a>
     {% endfor %}
     </div>
     {{ utils.display_object(anybox_object) }}

app/templates/admin_inbox.html

@@ -19,7 +19,7 @@
 
 {% for inbox_object in inbox %}
 {% if inbox_object.ap_type == "Announce" %}
-    {{ utils.actor_action(inbox_object, "shared") }}
+    {{ utils.actor_action(inbox_object, "shared", with_icon=True) }}
     {{ utils.display_object(inbox_object.relates_to_anybox_object) }}
 {% elif inbox_object.ap_type in ["Article", "Note", "Video", "Page", "Question"] %}
 {{ utils.display_object(inbox_object) }}
@@ -27,7 +27,7 @@
     {{ utils.actor_action(inbox_object, "followed you") }}
     {{ utils.display_actor(inbox_object.actor, actors_metadata) }}
 {% elif inbox_object.ap_type == "Like" %}
-    {{ utils.actor_action(inbox_object, "liked one of your post") }}
+    {{ utils.actor_action(inbox_object, "liked one of your posts", with_icon=True) }}
     {{ utils.display_object(inbox_object.relates_to_anybox_object) }}
 {% else %}
     <p>

app/templates/admin_new.html

@@ -25,7 +25,7 @@
 </nav>
 
 
-<form class="form" action="{{ request.url_for("admin_actions_new") }}" enctype="multipart/form-data" method="POST">
+<form class="form admin-new" action="{{ request.url_for("admin_actions_new") }}" enctype="multipart/form-data" method="POST">
     {{ utils.embed_csrf_token() }}
     {{ utils.embed_redirect_url() }}
     <p>
@@ -38,7 +38,7 @@
 
     {% if request.query_params.type == "Article" %}
     <p>
-        <input type="text" style="width:95%" name="name" placeholder="Title">     
+        <input type="text" class="width-95" name="name" placeholder="Title">
     </p>
     {% endif %}
 
@@ -49,7 +49,7 @@
     <span class="ji"><img src="{{ emoji.icon.url }}" alt="{{ emoji.name }}" title="{{ emoji.name }}" class="custom-emoji"></span>
     {% endfor %}
 
-    <textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" placeholder="Hey!" style="font-size:1.2em;width:95%;">{{ content }}</textarea>
+    <textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" placeholder="Hey!">{{ content }}</textarea>
 
     {% if request.query_params.type == "Question" %}
         <p>
@@ -69,20 +69,20 @@
         </p>
         {% for i in ["1", "2", "3", "4"] %}
             <p>
-            <input type="text" name="poll_answer_{{ i }}" style="width:95%;" placeholder="Option {{ i }}, leave empty to disable">
+            <input type="text" name="poll_answer_{{ i }}" class="width-95" placeholder="Option {{ i }}, leave empty to disable">
             </p>
         {% endfor %}
     {% endif %}
 
     <p>
-    <input type="text" name="content_warning" placeholder="content warning (will mark the post as sensitive)"{% if content_warning %} value="{{ content_warning }}"{% endif %} style="width:95%;">
+    <input type="text" name="content_warning" placeholder="content warning (will mark the post as sensitive)"{% if content_warning %} value="{{ content_warning }}"{% endif %} class="width-95">
     </p>
     <p>
         <input type="checkbox" name="is_sensitive" id="is_sensitive"> <label for="is_sensitive">Mark attachment(s) as sensitive</label>
     </p>
     <input type="hidden" name="in_reply_to" value="{{ request.query_params.in_reply_to }}">
     <p>
-        <input id="files" name="files" type="file" multiple style="width:95%;">
+        <input id="files" name="files" type="file" class="width-95" multiple>
     </p>
     <div id="alts"></div>
     <p>
@@ -90,5 +90,5 @@
     </p>
 </form>
 </div>
-<script src="/static/new.js"></script>
+<script src="{{ BASE_URL }}/static/new.js?v={{ JS_HASH }}"></script>
 {% endblock %}

app/templates/admin_outbox.html

@@ -12,18 +12,16 @@
 {% for outbox_object in outbox %}
 
     {% if outbox_object.ap_type == "Announce" %}
-        <div class="actor-action">You shared</div>
+        <div class="actor-action">You shared <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
         {{ utils.display_object(outbox_object.relates_to_anybox_object) }}
     {% elif outbox_object.ap_type == "Like" %}
-        <div class="actor-action">You liked</div>
+        <div class="actor-action">You liked <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
         {{ utils.display_object(outbox_object.relates_to_anybox_object) }}
     {% elif outbox_object.ap_type == "Follow" %}
-        <div class="actor-action">You followed</div>
+        <div class="actor-action">You followed <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
         {{ utils.display_actor(outbox_object.relates_to_actor, actors_metadata) }}
     {% elif outbox_object.ap_type in ["Article", "Note", "Video", "Question"] %}
         {{ utils.display_object(outbox_object) }}
-    {% else %}
-        Implement {{ outbox_object.ap_type }}
     {% endif %}
 
 {% endfor %}

app/templates/admin_profile.html

@@ -9,10 +9,21 @@
     {{ utils.display_actor(actor, actors_metadata, with_details=True) }}
     {% for inbox_object in inbox_objects %}
         {% if inbox_object.ap_type == "Announce" %}
-            {{ utils.actor_action(inbox_object, "shared") }}
+            {{ utils.actor_action(inbox_object, "shared", with_icon=True) }}
             {{ utils.display_object(inbox_object.relates_to_anybox_object) }}
         {% else %}
                 {{ utils.display_object(inbox_object) }}
         {% endif %}
     {% endfor %}
+
+{% if next_cursor %}
+<div class="box">
+    <p>
+    <a href="{{ request.url._path }}?actor_id={{ request.query_params.actor_id }}&cursor={{ next_cursor }}">
+            See more
+        </a>
+    </p>
+</div>
+{% endif %}
+
 {% endblock %}

app/templates/articles.html

@@ -12,7 +12,7 @@
 <data class="p-name" value="{{ local_actor.display_name}}'s articles"></data>
 {% for outbox_object in objects %}
     <li>
-        <span class="muted" style="padding-right:10px;">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</span> <a href="{{ outbox_object.url }}">{{ outbox_object.name }}</a>
+        <span class="muted">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</span> <a href="{{ outbox_object.url }}">{{ outbox_object.name }}</a>
     </li>
 {% endfor %}
 </ul>

app/templates/custom_page.html

@@ -0,0 +1,30 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>{{ title }}</title>
+{% if request.url.path == "/" %}
+<link rel="indieauth-metadata" href="{{ url_for("well_known_authorization_server") }}">
+<link rel="authorization_endpoint" href="{{ url_for("indieauth_authorization_endpoint") }}">
+<link rel="token_endpoint" href="{{ url_for("indieauth_token_endpoint") }}">
+<link rel="micropub" href="{{ url_for("micropub_endpoint") }}">
+<link rel="alternate" href="{{ local_actor.url }}" title="ActivityPub profile"  type="application/activity+json">
+<meta content="profile" property="og:type" />
+<meta content="{{ local_actor.url }}" property="og:url" />
+<meta content="{{ local_actor.display_name }}'s microblog" property="og:site_name" />
+<meta content="Homepage" property="og:title" />
+<meta content="{{ local_actor.summary | html2text | trim }}" property="og:description" />
+<meta content="{{ ICON_URL }}" property="og:image" />
+<meta content="summary" property="twitter:card" />
+<meta content="{{ local_actor.handle }}" property="profile:username" />
+{% endif %}
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    {{ page_content | safe }}
+</div>
+
+{% endblock %}

app/templates/error.html

@@ -0,0 +1,12 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+{% block main_tag %} class="main-flex"{% endblock %}
+{% block head %}
+<title>{{ title }}</title>
+{% endblock %}
+
+{% block content %}
+<div class="centered primary-color box">
+    <h1 class="error-title">{{ title | safe }}</h1>
+</div>
+{% endblock %}

app/templates/followers.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>{{ local_actor.display_name }}'s followers</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/following.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>{{ local_actor.display_name }}'s follows</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/header.html

@@ -25,20 +25,35 @@
 </div>
 
 {%- macro header_link(url, text) -%}
-{% set url_for = request.app.router.url_path_for(url) %}
+{% set url_for = BASE_URL + request.app.router.url_path_for(url) %}
-<a href="{{ url_for }}" {% if request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
+<a href="{{ url_for }}" {% if BASE_URL + request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
 {% endmacro %}
 
-<div style="margin:30px 0 0 0;">
+{%- macro navbar_item_link(navbar_item) -%}
+{% set url_for = BASE_URL + navbar_item[0] %}
+<a href="{{ navbar_item[0] }}" {% if BASE_URL + request.url.path == url_for %}class="active"{% endif %}>{{ navbar_item[1] }}</a>
+{% endmacro %}
+
+<div class="public-top-menu">
 <nav class="flexbox">
     <ul>
+        {% if NAVBAR_ITEMS.INDEX_NAVBAR_ITEM %}
+        <li>{{ navbar_item_link(NAVBAR_ITEMS.INDEX_NAVBAR_ITEM) }}</li>
+        {% endif %}
         <li>{{ header_link("index", "Notes") }}</li>
         {% if articles_count %}
             <li>{{ header_link("articles", "Articles") }}</li>
         {% endif %}
+        {% if not HIDES_FOLLOWERS or is_admin %}
         <li>{{ header_link("followers", "Followers") }} <span class="counter">{{ followers_count }}</span></li>
+        {% endif %}
+        {% if not HIDES_FOLLOWING or is_admin %}
         <li>{{ header_link("following", "Following") }} <span class="counter">{{ following_count }}</span></li>
+        {% endif %}
         <li>{{ header_link("get_remote_follow", "Remote follow") }}</li>
+        {% for navbar_item in NAVBAR_ITEMS.EXTRA_NAVBAR_ITEMS %}
+            {{ navbar_item_link(navbar_item) }}
+        {% endfor %}
     </ul>
 </nav>
 </div>

app/templates/index.html

@@ -13,7 +13,7 @@
 <meta content="{{ local_actor.display_name }}'s microblog" property="og:site_name" />
 <meta content="Homepage" property="og:title" />
 <meta content="{{ local_actor.summary | html2text | trim }}" property="og:description" />
-<meta content="{{ local_actor.url }}" property="og:image" />
+<meta content="{{ ICON_URL }}" property="og:image" />
 <meta content="summary" property="twitter:card" />
 <meta content="{{ local_actor.handle }}" property="profile:username" />
 {% endblock %}
@@ -21,26 +21,34 @@
 {% block content %}
 {% include "header.html" %}
 
-<div class="h-feed">
+{% if objects %}
-<data class="p-name" value="{{ local_actor.display_name}}'s notes"></data>
+
-{% for outbox_object in objects %}
+    <div class="h-feed">
-{% if outbox_object.ap_type in ["Note", "Article", "Video", "Question"] %}
+    <data class="p-name" value="{{ local_actor.display_name}}'s notes"></data>
-{{ utils.display_object(outbox_object) }}
+    {% for outbox_object in objects %}
-{% elif outbox_object.ap_type == "Announce" %}
+    {% if outbox_object.ap_type in ["Note", "Article", "Video", "Question"] %}
-<div class="shared-header"><strong>{{ local_actor.display_name }}</strong> shared</div>
+    {{ utils.display_object(outbox_object) }}
-{{ utils.display_object(outbox_object.relates_to_anybox_object) }}
+    {% elif outbox_object.ap_type == "Announce" %}
+    <div class="shared-header"><strong>{{ utils.display_tiny_actor_icon(local_actor) }} {{ local_actor.display_name | clean_html(local_actor) | safe  }}</strong> shared <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
+    {{ utils.display_object(outbox_object.relates_to_anybox_object) }}
+    {% endif %}
+    {% endfor %}
+    </div>
+
+    <div class="box">
+        {% if has_previous_page %}
+        <a href="{{ url_for("index") }}?page={{ current_page - 1 }}">Previous</a>
+        {% endif %}
+
+        {% if has_next_page %}
+        <a href="{{ url_for("index") }}?page={{ current_page + 1 }}">Next</a>
+        {% endif %}
+    </div>
+
+{% else %}
+    <div class="empty-state">
+        <p>Nothing to see here yet!</p>
+    </div>
 {% endif %}
-{% endfor %}
-</div>
-
-<div class="box">
-    {% if has_previous_page %}
-    <a href="{{ url_for("index") }}?page={{ current_page - 1 }}">Previous</a>
-    {% endif %}
-
-    {% if has_next_page %}
-    <a href="{{ url_for("index") }}?page={{ current_page + 1 }}">Next</a>
-    {% endif %}
-</div>
 
 {% endblock %}

app/templates/indieauth_flow.html

@@ -2,15 +2,15 @@
 {% extends "layout.html" %}
 {% block content %}
 <div class="box">
-    <div style="display:flex;column-gap: 20px;">
+    <div class"indieauth-box">
         {% if client.logo %}
-        <div style="flex:initial;width:100px;">
+        <div class="indieauth-logo">
-            <img src="{{client.logo | media_proxy_url }}" style="max-width:100px;" alt="{{ client.name }} logo">
+            <img src="{{client.logo | media_proxy_url }}" alt="{{ client.name }} logo">
         </div>
         {% endif %}
-        <div style="flex:1;">
+        <div class="indieauth-details">
-            <div style="padding-left: 20px;">
+            <div>
-                <a class="lcolor" style="font-size:1.2em;font-weight:600;" href="{{ client.url }}">{{ client.name }}</a>
+                <a class="lcolor" href="{{ client.url }}">{{ client.name }}</a>
                 <p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
 
 

app/templates/layout.html

@@ -4,26 +4,24 @@
 <meta charset="utf-8">
 <meta http-equiv="x-ua-compatible" content="ie=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-<link rel="stylesheet" href="/static/css/main.css?v={{ css_hash }}">
+<link rel="stylesheet" href="{{ BASE_URL }}/static/css/main.css?v={{ CSS_HASH }}">
 <link rel="alternate" title="{{ local_actor.display_name}}'s microblog" type="application/json" href="{{ url_for("json_feed") }}" />
 <link rel="alternate" href="{{ url_for("rss_feed") }}" type="application/rss+xml" title="{{ local_actor.display_name}}'s microblog">
 <link rel="alternate" href="{{ url_for("atom_feed") }}" type="application/atom+xml" title="{{ local_actor.display_name}}'s microblog">
-<link rel="icon" type="image/x-icon" href="/static/favicon.ico">
+<link rel="icon" type="image/x-icon" href="{{ BASE_URL }}/static/favicon.ico">
-<style>
+<style>{{ highlight_css }}</style>
-{{ highlight_css }}
-</style>
 {% block head %}{% endblock %}
 </head>
 <body>
 <div id="main">
-<main>
+<main{%- block main_tag %}{%- endblock %}>
 {% if is_admin %}
 <div id="admin">
 {% macro admin_link(url, text) %}
-{% set url_for = request.app.router.url_path_for(url) %}
+{% set url_for = BASE_URL + request.app.router.url_path_for(url) %}
-<a href="{{ url_for }}" {% if request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
+<a href="{{ url_for }}" {% if BASE_URL + request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
 {% endmacro %}
-<div style="margin-bottom:30px;padding: 0 20px;">
+<div class="admin-menu">
 <nav class="flexbox">
     <ul>
         <li>{{ admin_link("index", "Public") }}</li>
@@ -47,8 +45,15 @@
 
 <footer class="footer">
     <div class="box">
-    Powered by <a href="https://docs.microblog.pub">microblog.pub</a> <small class="microblogpub-version"><code>{{ microblogpub_version }}</code></small> and the <a href="https://activitypub.rocks/">ActivityPub</a> protocol. <a href="{{ url_for("login") }}">Admin</a>.
+    {% if custom_footer %}
+        {{ custom_footer | safe }}
+    {% else %}
+        Powered by <a href="https://docs.microblog.pub">microblog.pub</a> <small class="microblogpub-version"><code>{{ microblogpub_version }}</code></small> and the <a href="https://activitypub.rocks/">ActivityPub</a> protocol. <a href="{{ url_for("login") }}">Admin</a>.
+    {% endif %}
     </div>
 </footer>
+{% if is_admin %}
+<script src="{{ BASE_URL }}/static/common-admin.js?v={{ JS_HASH }}"></script>
+{% endif %}
 </body>
 </html>

app/templates/login.html

@@ -1,14 +1,21 @@
 {%- import "utils.html" as utils with context -%}
 {% extends "layout.html" %}
+{% block head %}
+<meta name="robots" content="noindex, nofollow">
+{% endblock %}
+{% block main_tag %} class="main-flex"{% endblock %}
 {% block content %}
-<div style="display:grid;height:80%;">
+<div class="centered">
-<div style="margin:auto;">
+    <div>
-<form class="form" action="/admin/login" method="POST">
+        {% if error %}
-<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+            <p class="primary-color">Invalid password.</p>
-<input type="hidden" name="redirect" value="{{ redirect }}">
+        {% endif %}
-<input type="password" placeholder="password" name="password" autofocus>
+        <form class="form" action="{{ BASE_URL }}/admin/login" method="POST">
-<input type="submit" value="login">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-</form>
+            <input type="hidden" name="redirect" value="{{ redirect }}">
-</div>
+            <input type="password" placeholder="password" name="password" autofocus>
+            <input type="submit" value="login">
+        </form>
+    </div>
 </div>
 {% endblock %}

app/templates/lookup.html

@@ -19,7 +19,9 @@
     {% if error %}
     <div class="box error-box">
         {% if error.value == "NOT_FOUND" %}
-        <p>The remote object was deleted.</p>
+        <p>The remote object is unavailable.</p>
+        {% elif error.value == "UNAUTHORIZED" %}
+        <p>Missing permissions to fetch the remote object.</p>
         {% elif error.value == "TIMEOUT" %}
         <p>Lookup timed out, please try refreshing the page.</p>
         {% else %}

app/templates/notifications.html

@@ -5,10 +5,14 @@
 <title>{{ local_actor.display_name }} - Notifications</title>
 {% endblock %}
 
-{% macro notif_actor_action(notif, text) %}
+{% macro notif_actor_action(notif, text, with_icon=False) %}
     <div class="actor-action">
-        <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.actor.ap_id }}">{{ notif.actor.display_name | clean_html(notif.actor) | safe }}</a> {{ text }}
+        <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.actor.ap_id }}">
+            {% if with_icon %}{{ utils.display_tiny_actor_icon(notif.actor) }}{% endif %} {{ notif.actor.display_name | clean_html(notif.actor) | safe }}</a> {{ text }}
         <span title="{{ notif.created_at.isoformat() }}">{{ notif.created_at | timeago }}</span>
+        {% if notif.is_new %}
+            <span class="new">new</span>
+        {% endif %}
     </div>
 {% endmacro %}
 
@@ -35,17 +39,37 @@
             {%- elif notif.notification_type.value == "follow_request_rejected" %}
                 {{ notif_actor_action(notif, "rejected your follow request") }}
                 {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "blocked" %}
+                {{ notif_actor_action(notif, "blocked you") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "unblocked" %}
+                {{ notif_actor_action(notif, "unblocked you") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "block" %}
+                {{ notif_actor_action(notif, "was blocked") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "unblock" %}
+                {{ notif_actor_action(notif, "was unblocked") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {%- elif notif.notification_type.value == "move" %}
+                {# for move notif, the actor is the target and the inbox object the Move activity #}
+                <div class="actor-action">
+                    <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.inbox_object.actor.ap_id }}">
+                        {{ utils.display_tiny_actor_icon(notif.inbox_object.actor) }} {{ notif.inbox_object.actor.display_name | clean_html(notif.inbox_object.actor) | safe }}</a> has moved to
+                    <span title="{{ notif.created_at.isoformat() }}">{{ notif.created_at | timeago }}</span>
+                </div>
+                {{ utils.display_actor(notif.actor) }}
             {% elif notif.notification_type.value == "like" %}
-                {{ notif_actor_action(notif, "liked a post") }}
+                {{ notif_actor_action(notif, "liked a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
            {% elif notif.notification_type.value == "undo_like" %}
-                {{ notif_actor_action(notif, "unliked a post") }}
+                {{ notif_actor_action(notif, "unliked a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "announce" %}
-                {{ notif_actor_action(notif, "shared a post") }}
+                {{ notif_actor_action(notif, "shared a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
            {% elif notif.notification_type.value == "undo_announce" %}
-                {{ notif_actor_action(notif, "unshared a post") }}
+                {{ notif_actor_action(notif, "unshared a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "mention" %}
                 {{ notif_actor_action(notif, "mentioned you") }}
@@ -57,7 +81,7 @@
                     {% if facepile_item %}
                         <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
                     {% endif %}
-                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                    <a class="bold" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
                 </div>
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "updated_webmention" %}
@@ -67,7 +91,7 @@
                     {% if facepile_item %}
                         <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
                     {% endif %}
-                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                    <a class="bold" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
                 </div>
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "deleted_webmention" %}
@@ -77,7 +101,7 @@
                     {% if facepile_item %}
                         <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
                     {% endif %}
-                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                    <a class="bold" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
                 </div>
                 {{ utils.display_object(notif.outbox_object) }}
             {% else %}
@@ -88,4 +112,15 @@
     </div>
     {%- endfor %}
     </div>
+
+{% if next_cursor %}
+<div class="box">
+    <p>
+        <a href="{{ request.url._path }}?cursor={{ next_cursor }}">
+            See more{% if more_unread_count %} ({{ more_unread_count }} unread left){% endif %}
+        </a>
+    </p>
+</div>
+{% endif %}
+
 {% endblock %}

app/templates/object.html

@@ -3,7 +3,11 @@
 
 {% block head %}
 {% if outbox_object %}
-{% set excerpt = outbox_object.content | html2text | trim | truncate(50) %}
+{% if outbox_object.content %}
+    {% set excerpt = outbox_object.content | html2text | trim | truncate(50) %}
+{% else %}
+    {% set excerpt = outbox_object.summary | html2text | trim | truncate(50) %}
+{% endif %}
 <title>{% if outbox_object.name %}{{ outbox_object.name }}{% else %}{{ local_actor.display_name }}: "{{ excerpt }}"{% endif %}</title>
 <link rel="webmention" href="{{ url_for("webmention_endpoint") }}">
 <link rel="alternate" href="{{ request.url }}" type="application/activity+json">
@@ -27,9 +31,16 @@
 {% macro display_replies_tree(replies_tree_node) %}
 
 {% if replies_tree_node.is_requested %}
-{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root, is_object_page=True) }}
+{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root, is_object_page=True, is_h_entry=False) }}
 {% else %}
-    {{ utils.display_object(replies_tree_node.ap_object) }}
+    {% if replies_tree_node.wm_reply %}
+        {# u-comment h-cite is displayed by default for webmention #}
+        {{ utils.display_webmention_reply(replies_tree_node.wm_reply) }}
+    {% else %}
+        <div class="u-comment h-cite">
+            {{ utils.display_object(replies_tree_node.ap_object, is_h_entry=False) }}
+        </div>
+    {% endif %}
 {% endif %}
 
 {% for child in replies_tree_node.children %}
@@ -38,6 +49,8 @@
 
 {% endmacro %}
 
+<div class="h-entry">
 {{ display_replies_tree(replies_tree) }}
+</div>
 
 {% endblock %}

app/templates/redirect_to_remote_instance.html

@@ -0,0 +1,15 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>{{ local_actor.display_name }}'s microblog - Redirect</title>
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    <p>You are being redirected to your instance: <a href="{{ url }}">{{ url }}</a></p>
+</div>
+
+{% endblock %}

app/templates/remote_follow.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>Remote follow {{ local_actor.display_name }}</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/remote_interact.html

@@ -0,0 +1,27 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>Interact from your instance</title>
+<meta name="robots" content="noindex, nofollow">
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    <h2>Interact with this object</h2>
+</div>
+
+{{ utils.display_object(outbox_object) }}
+
+<div class="box">
+    <form class="form" action="{{ url_for("post_remote_interaction") }}" method="POST">
+        {{ utils.embed_csrf_token() }}
+        <input type="text" name="profile" placeholder="you@instance.tld" autofocus>
+        <input type="hidden" name="ap_id" value="{{ outbox_object.ap_id }}">
+        <input type="submit" value="interact from your instance">
+    </form>
+</div>
+
+{% endblock %}

app/templates/utils.html

@@ -1,167 +1,231 @@
 {% macro embed_csrf_token() %}
+{% block embed_csrf_token scoped %}
 <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+{% endblock %}
 {% endmacro %}
 
 {% macro embed_redirect_url(permalink_id=None) %}
+{% block embed_redirect_url scoped %}
 <input type="hidden" name="redirect_url" value="{{ request.url }}{% if permalink_id %}#{{ permalink_id }}{% endif %}">
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_block_button(actor) %}
+{% block admin_block_button scoped %}
 <form action="{{ request.url_for("admin_actions_block") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
     <input type="submit" value="block">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_unblock_button(actor) %}
+{% block admin_unblock_button scoped %}
 <form action="{{ request.url_for("admin_actions_unblock") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
     <input type="submit" value="unblock">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_follow_button(actor) %}
+{% block admin_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_follow") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
     <input type="submit" value="follow">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_accept_incoming_follow_button(notif) %}
+{% block admin_accept_incoming_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_accept_incoming_follow") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="notification_id" value="{{ notif.id }}">
     <input type="submit" value="accept follow">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_reject_incoming_follow_button(notif) %}
+{% block admin_reject_incoming_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_reject_incoming_follow") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="notification_id" value="{{ notif.id }}">
     <input type="submit" value="reject follow">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_like_button(ap_object_id, permalink_id) %}
+{% block admin_like_button scoped %}
 <form action="{{ request.url_for("admin_actions_like") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="like">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_bookmark_button(ap_object_id, permalink_id) %}
+{% block admin_bookmark_button scoped %}
 <form action="{{ request.url_for("admin_actions_bookmark") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="bookmark">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_unbookmark_button(ap_object_id, permalink_id) %}
+{% block admin_unbookmark_button scoped %}
 <form action="{{ request.url_for("admin_actions_unbookmark") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="unbookmark">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_pin_button(ap_object_id, permalink_id) %}
+{% block admin_pin_button scoped %}
 <form action="{{ request.url_for("admin_actions_pin") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="pin">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_unpin_button(ap_object_id, permalink_id) %}
+{% block admin_unpin_button scoped %}
 <form action="{{ request.url_for("admin_actions_unpin") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="unpin">
 </form>
+{% endblock %}
 {% endmacro %}
 
-{% macro admin_delete_button(ap_object_id) %}
+{% macro admin_delete_button(ap_object) %}
-<form action="{{ request.url_for("admin_actions_delete") }}" method="POST" onsubmit="return confirm('Do you really want to delete this object?');">
+{% block admin_delete_button scoped %}
+<form action="{{ request.url_for("admin_actions_delete") }}" class="object-delete-form" method="POST">
     {{ embed_csrf_token() }}
-    {{ embed_redirect_url() }}
+    <input type="hidden" name="redirect_url" value="{% if request.url.path.endswith("/" + ap_object.public_id) or (request.url.path == "/admin/object" and request.query_params.ap_id.endswith("/" + ap_object.public_id)) %}{{ request.base_url}}{% else %}{{ request.url }}{% endif %}">
-    <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
+    <input type="hidden" name="ap_object_id" value="{{ ap_object.ap_id }}">
     <input type="submit" value="delete">
 </form>
+{% endblock %}
+{% endmacro %}
+
+{% macro admin_force_delete_button(ap_object_id, permalink_id=None) %}
+{% block admin_force_delete_button scoped %}
+<form action="{{ request.url_for("admin_actions_force_delete") }}" class="object-delete-form" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url(permalink_id) }}
+    <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
+    <input type="submit" value="local delete">
+</form>
+{% endblock %}
+{% endmacro %}
+
+{% macro admin_force_delete_webmention_button(webmention_id, permalink_id=None) %}
+{% block admin_force_delete_webmention_button scoped %}
+<form action="{{ request.url_for("admin_actions_force_delete_webmention") }}" class="object-delete-form" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url(permalink_id) }}
+    <input type="hidden" name="webmention_id" value="{{ webmention_id }}">
+    <input type="submit" value="local delete">
+</form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_announce_button(ap_object_id, permalink_id=None) %}
+{% block admin_announce_button scoped %}
 <form action="{{ request.url_for("admin_actions_announce") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="share">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_undo_button(ap_object_id, action="undo", permalink_id=None) %}
+{% block admin_undo_button scoped %}
 <form action="{{ request.url_for("admin_actions_undo") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="{{ action }}">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_reply_button(ap_object_id) %}
-<form action="/admin/new"  method="GET">
+{% block admin_reply_button scoped %}
+<form action="{{ BASE_URL }}/admin/new"  method="GET">
 <input type="hidden" name="in_reply_to" value="{{ ap_object_id }}">
 <button type="submit">reply</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_dm_button(actor_handle) %}
-<form action="/admin/new"  method="GET">
+{% block admin_dm_button scoped %}
+<form action="{{ BASE_URL }}/admin/new"  method="GET">
 <input type="hidden" name="with_content" value="{{ actor_handle }}">
 <input type="hidden" name="with_visibility" value="DIRECT">
 <button type="submit">direct message</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_mention_button(actor_handle) %}
-<form action="/admin/new"  method="GET">
+{% block admin_mention_button scoped %}
+<form action="{{ BASE_URL }}/admin/new"  method="GET">
 <input type="hidden" name="with_content" value="{{ actor_handle }}">
 <button type="submit">mention</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 
 
 {% macro admin_profile_button(ap_actor_id) %}
+{% block admin_profile_button scoped %}
 <form action="{{ url_for("admin_profile") }}"  method="GET">
 <input type="hidden" name="actor_id" value="{{ ap_actor_id }}">
 <button type="submit">profile</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
-{% macro admin_expand_button(ap_object_id) %}
+{% macro admin_expand_button(ap_object) %}
+{% block admin_expand_button scoped %}
+{# TODO turn these into a regular link and append permalink ID if it's a reply #}
 <form action="{{ url_for("admin_object") }}"  method="GET">
-<input type="hidden" name="ap_id" value="{{ ap_object_id }}">
+<input type="hidden" name="ap_id" value="{{ ap_object.ap_id }}">
 <button type="submit">expand</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 {% macro display_box_filters(route) %}
+{% block display_box_filters scoped %}
 <nav class="flexbox box">
 <ul>
     <li>Filter by</li>
@@ -178,17 +242,29 @@
 {% endif %}
 </ul>
 </nav>
+{% endblock %}
 {% endmacro %}
 
-{% macro actor_action(inbox_object, text) %}
+{% macro display_tiny_actor_icon(actor) %}
+{% block display_tiny_actor_icon scoped %}
+    <img class="tiny-actor-icon" src="{{ actor.resized_icon_url }}" alt="{{ actor.display_name }}'s avatar">
+{% endblock %}
+{% endmacro %}
+
+{% macro actor_action(inbox_object, text, with_icon=False) %}
+{% block actor_action scoped %}
     <div class="actor-action">
-        <a href="{{ url_for("admin_profile") }}?actor_id={{ inbox_object.actor.ap_id }}">{{ inbox_object.actor.display_name | clean_html(inbox_object.actor) | safe }}</a> {{ text }}
+        <a href="{{ url_for("admin_profile") }}?actor_id={{ inbox_object.actor.ap_id }}">
+            {% if with_icon %}{{ display_tiny_actor_icon(inbox_object.actor) }}{% endif %} {{ inbox_object.actor.display_name | clean_html(inbox_object.actor) | safe }}
+        </a> {{ text }}
         <span title="{{ inbox_object.ap_published_at.isoformat() }}">{{ inbox_object.ap_published_at | timeago }}</span>
     </div>
 
+{% endblock %}
 {% endmacro %}
 
 {% macro display_actor(actor, actors_metadata={}, embedded=False, with_details=False, pending_incoming_follow_notif=None) %}
+{% block display_actor scoped %}
 {% set metadata = actors_metadata.get(actor.ap_id) %}
 
 {% if not embedded %}
@@ -199,7 +275,7 @@
     <div class="icon-box">
         <img src="{{ actor.resized_icon_url }}" alt="{{ actor.display_name }}'s avatar" class="actor-icon u-photo">
     </div>
-    <a href="{{ actor.url }}" class="u-url" style="">
+    <a href="{{ actor.url }}" class="u-url">
         <div><strong>{{ actor.display_name | clean_html(actor) | safe  }}</strong></div>
         <div class="actor-handle p-name">{{ actor.handle }}</div>
     </a>
@@ -209,14 +285,26 @@
 <div>
     <nav class="flexbox actor-metadata">
         <ul>
+            {% if metadata.has_blocked_local_actor %}
+                <li>blocked you</li>
+            {% endif %}
             {% if metadata.is_following %}
                 <li>already following</li>
                 <li>{{ admin_undo_button(metadata.outbox_follow_ap_id, "unfollow")}}</li>
-                <li>{{ admin_profile_button(actor.ap_id) }}</li>
+                {% if not with_details %}
+                    <li>{{ admin_profile_button(actor.ap_id) }}</li>
+                {% endif %}
             {% elif metadata.is_follow_request_sent %}
-                <li>follow request sent</li>
+                {% if metadata.is_follow_request_rejected %}
-                <li>{{ admin_undo_button(metadata.outbox_follow_ap_id, "undo follow") }}</li>
+                    <li>follow request rejected</li>
-            {% else %}
+                    {% if not metadata.has_blocked_local_actor %}
+                        <li>{{ admin_follow_button(actor) }}</li>
+                    {% endif %}
+                {% else %}
+                    <li>follow request sent</li>
+                    <li>{{ admin_undo_button(metadata.outbox_follow_ap_id, "undo follow") }}</li>
+                {% endif %}
+            {% elif not actor.moved_to %}
                 <li>{{ admin_follow_button(actor) }}</li>
             {% endif %}
             {% if metadata.is_follower %}
@@ -224,7 +312,11 @@
                 {% if not metadata.is_following and not with_details %}
                     <li>{{ admin_profile_button(actor.ap_id) }}</li>
                 {% endif %}
-                </li>
+            {% elif actor.is_from_db and not with_details and not metadata.is_following %}
+                <li>{{ admin_profile_button(actor.ap_id) }}</li>
+            {% endif %}
+            {% if actor.moved_to %}
+            <li>has moved to {% if metadata.moved_to %}<a href="{{ url_for("admin_profile") }}?actor_id={{ actor.moved_to }}">{{ metadata.moved_to.handle }}</a>{% else %}<a href="{{ url_for("get_lookup") }}?query={{ actor.moved_to }}">{{ actor.moved_to }}</a>{% endif %}</li>
             {% endif %}
             {% if actor.is_from_db %}
                 {% if actor.is_blocked %}
@@ -250,6 +342,9 @@
                     <li>rejected</li>
                 {% endif %}
             {% endif %}
+            {% if with_details %}
+                <li><a href="{{ actor.url }}" class="label-btn">remote profile</a></li>
+            {% endif %}
         </ul>
     </nav>
 </div>
@@ -280,58 +375,71 @@
 </div>
 {% endif %}
 
+{% endblock %}
 {% endmacro %}
 
 {% macro display_og_meta(object) %}
+{% block display_og_meta scoped %}
 {% if object.og_meta %}
-{% for og_meta in object.og_meta %}
+{% for og_meta in object.og_meta[:1] %}
-<div class="activity-og-meta" style="display:flex;column-gap: 20px;margin:20px 0;">
+<div class="activity-og-meta">
     {% if og_meta.image %}
     <div>
-        <img src="{{ og_meta.image | media_proxy_url }}" style="max-width:200px;max-height:100px;">
+        <img src="{{ og_meta.image | media_proxy_url }}">
     </div>
     {% endif %}
     <div>
         <a href="{{ og_meta.url | privacy_replace_url }}">{{ og_meta.title }}</a>
         {% if og_meta.site_name %}
-            <small style="display:block;">{{ og_meta.site_name }}</small>
+            <small>{{ og_meta.site_name }}</small>
         {% endif %}
     </div>
 </div>
 {% endfor %}
 {% endif %}
+{% endblock %}
 {% endmacro %}
 
 
 {% macro display_attachments(object) %}
+{% block display_attachments scoped %}
 
     {% for attachment in object.attachments %}
+    {% if attachment.type != "PropertyValue" %}
+    {% set orientation = "unknown" %}
+    {% if attachment.width %}
+        {% set orientation = "portrait" if attachment.width < attachment.height else "landscape" %}
+    {% endif %}
     {% if object.sensitive and (attachment.type == "Image" or (attachment | has_media_type("image")) or attachment.type == "Video" or (attachment | has_media_type("video"))) %}
-    <div>
+    <div class="attachment-wrapper">
-        <label for="{{attachment.proxied_url}}" class="label-btn" style="display:inline-block;">show/hide sensitive content</label>
+        <label for="{{attachment.proxied_url}}" class="label-btn show-hide-sensitive-btn">show/hide sensitive content</label>
         <div>
         <div class="sensitive-attachment">
         <input class="sensitive-attachment-state" type="checkbox" id="{{attachment.proxied_url}}" aria-hidden="true">
-        <div class="sensitive-attachment-box">
+        <div class="sensitive-attachment-box attachment-orientation-{{orientation}}">
             <div></div>
     {% else %}
-            <div style="margin-top:20px;">
+            <div class="attachment-item attachment-orientation-{{orientation}}">
     {% endif %}
 
     {% if attachment.type == "Image" or (attachment | has_media_type("image")) %}
         {% if attachment.url not in object.inlined_images %}
-            <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment" style="margin:0;">
+        <a class="media-link" href="{{ attachment.proxied_url }}" target="_blank">
+            <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment">
+        </a>
         {% endif %}
     {% elif attachment.type == "Video" or (attachment | has_media_type("video")) %}
     <video controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %}></video>
     {% elif attachment.type == "Audio" or (attachment | has_media_type("audio")) %}
-    <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} style="width:480px;" class="attachment"></audio>
+    <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} class="attachment"></audio>
     {% elif attachment.type == "Link" %}
-        <a href="{{ attachment.url }}" class="attachment" style="display:inline-block;margin-bottom: 15px;">{{ attachment.url }}</a>
+    <a href="{{ attachment.url }}" class="attachment">{{ attachment.url | truncate(64, True) }}</a> ({{ attachment.mimetype}})
     {% else %}
-    <a href="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %} class="attachment">{{ attachment.url }}</a>
+    <a href="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.url }}"{% endif %} class="attachment">
+        {% if attachment.name %}{{ attachment.name }}{% else %}{{ attachment.url | truncate(64, True) }}{% endif %}
+    </a> ({{ attachment.mimetype }})
     {% endif %}
-    {% if object.sensitive %}
+    {% if object.sensitive and (attachment.type == "Image" or (attachment | has_media_type("image")) or attachment.type == "Video" or (attachment | has_media_type("video"))) %}
             </div>
         </div>
         </div>
@@ -339,13 +447,60 @@
     {% else %}
         </div>
     {% endif %}
+    {% endif %}
     {% endfor %}
+{% endblock %}
 {% endmacro %}
 
-{% macro display_object(object, likes=[], shares=[], webmentions=[], expanded=False, actors_metadata={}, is_object_page=False) %}
+{% macro display_webmention_reply(wm_reply) %}
+{% block display_webmention_reply scoped %}
+
+<div class="ap-object u-comment h-cite">
+<div class="actor-box h-card p-author">
+    <div class="icon-box">
+        <img src="{{ wm_reply.face.picture_url }}" alt="{{ wm_reply.face.name }}'s avatar" class="actor-icon u-photo">
+    </div>
+    <a href="{{ wm_reply.face.url }}" class="u-url">
+        <div><strong class="p-name">{{ wm_reply.face.name | clean_html_wm | safe }}</strong></div>
+        <div class="actor-handle">{{ wm_reply.face.url | truncate(64, True) }}</div>
+    </a>
+</div>
+
+    <p class="in-reply-to">in reply to <a href="{{ wm_reply.in_reply_to }}" title="{{ wm_reply.in_reply_to }}" rel="nofollow">
+        this note
+    </a></p>
+
+    <div class="obj-content margin-top-20">
+    <div class="e-content">
+        {{ wm_reply.content | clean_html_wm | safe }}
+    </div>
+    </div>
+
+    <nav class="flexbox activity-bar margin-top-20">
+    <ul>
+        <li>
+            <div><a href="{{ wm_reply.url }}" rel="nofollow" class="object-permalink u-url u-uid">permalink</a></div>
+        </li>
+        <li>
+            <time class="dt-published" datetime="{{ wm_reply.published_at.replace(microsecond=0).isoformat() }}" title="{{ wm_reply.published_at.replace(microsecond=0).isoformat() }}">{{ wm_reply.published_at | timeago }}</time>
+        </li>
+        {% if is_admin %}
+            <li>
+                {{ admin_force_delete_webmention_button(wm_reply.webmention_id) }}
+            </li>
+        {% endif %}
+    </ul>
+    </nav>
+</div>
+
+{% endblock %}
+{% endmacro %}
+
+{% macro display_object(object, likes=[], shares=[], webmentions=[], expanded=False, actors_metadata={}, is_object_page=False, is_h_entry=True) %}
+{% block display_object scoped %}
 {% set is_article_mode = object.is_from_outbox and object.ap_type == "Article" and is_object_page %}
-{% if object.ap_type in ["Note", "Article", "Video", "Page", "Question"] %}
+{% if object.ap_type in ["Note", "Article", "Video", "Page", "Question", "Event"] %}
-<div class="ap-object {% if expanded %}ap-object-expanded {% endif %}h-entry" id="{{ object.permalink_id }}">
+<div class="ap-object {% if expanded %}ap-object-expanded {% endif %}{% if is_h_entry %}h-entry{% endif %}" id="{{ object.permalink_id }}">
 
     {% if is_article_mode %}
     <data class="h-card">
@@ -358,13 +513,35 @@
     {% endif %}
 
     {% if object.in_reply_to %}
-        <a href="{% if is_admin %}{{ url_for("get_lookup") }}?query={% endif %}{{ object.in_reply_to }}" title="{{ object.in_reply_to }}" class="in-reply-to" rel="nofollow">
+    <p class="in-reply-to">in reply to <a href="{% if is_admin and object.is_in_reply_to_from_inbox %}{{ url_for("get_lookup") }}?query={% endif %}{{ object.in_reply_to }}" title="{{ object.in_reply_to }}" rel="nofollow">
-            in reply to {{ object.in_reply_to|truncate(64, True) }}
+            this {{ object.ap_type|lower }}
-        </a>
+        </a></p>
     {% endif %}
 
-    {% if object.ap_type == "Article" %}
+    {% if object.ap_type in ["Article", "Event"] %}
-        <h2 class="p-name" style="margin-top:0;">{{ object.name }}</h2>
+        <h2 class="p-name no-margin-top">{{ object.name }}</h2>
+    {% endif %}
+
+    {% if object.ap_type == "Event" %}
+        {% if object.ap_object.get("endTime") and object.ap_object.get("startTime") %}
+            <p>On {{ object.ap_object.startTime | parse_datetime | format_date }}
+            (ends {{ object.ap_object.endTime | parse_datetime | format_date }})</p>
+        {% endif %}
+    {% endif %}
+
+    {% if object.ap_object.get("location") %}
+        {% set loc = object.ap_object.get("location") %}
+        {% if loc.type == "Place" and loc.latitude and loc.longitude %}
+        <div class="ap-place">
+            <h3>Location</h3>
+            {% if loc.name %}{{ loc.name }}{% endif %}
+            <span class="h-geo">
+                <data class="p-latitude" value="{{ loc.latitude}}"></data>
+                <data class="p-longitude" value="{{ loc.longitude }}"></data>
+                <a href="https://www.openstreetmap.org/?mlat={{ loc.latitude }}&mlon={{ loc.longitude }}#map=16/{{loc.latitude}}/{{loc.longitude}}">{{loc.latitude}},{{loc.longitude}}</a>
+            </span>
+        </div>
+        {% endif %}
     {% endif %}
 
     {% if is_article_mode %}
@@ -394,11 +571,11 @@
         {% endif %}
 
         {% if object.poll_items %}
-            <ul style="list-style-type: none;padding:0;">
+            <ul class="poll-items">
             {% for item in object.poll_items %}
-            <li style="display:block;">
+            <li>
                 {% set pct = item | poll_item_pct(object.poll_voters_count) %}
-                <p style="margin:20px 0 10px 0;">
+                <p>
                     {% if can_vote %}
                     <input type="{% if object.is_one_of_poll %}radio{% else %}checkbox{% endif %}" name="name" value="{{ item.name }}" id="{{object.permalink_id}}-{{item.name}}">
                     <label for="{{object.permalink_id}}-{{item.name}}">
@@ -407,17 +584,17 @@
                     {{ item.name | clean_html(object) | safe }}
 
                     {% if object.voted_for_answers and item.name in object.voted_for_answers %}
-                    <span class="muted" style="padding-left:20px;">you voted for this answer</span>
+                    <span class="muted poll-vote">you voted for this answer</span>
                     {% endif %}
 
                     {% if can_vote %}
                     </label>
                     {% endif %}
 
-                    <span style="float:right;">{{ pct }}% <span class="muted">({{ item.replies.totalItems }} votes)</span></span>
+                    <span class="float-right">{{ pct }}% <span class="muted">({{ item.replies.totalItems }} votes)</span></span>
                 </p>
                 <svg class="poll-bar">
-                    <line x1="0" y1="10px" x2="{{ pct or 1 }}%" y2="10px" style="stroke-width: 20px;"></line>
+                    <line x1="0" y1="10px" x2="{{ pct or 1 }}%" y2="10px"></line>
                 </svg>
             </li>
             {% endfor %}
@@ -431,16 +608,17 @@
             </form>
         {% endif %}
 
-        {{ display_og_meta(object) }}
 
     {% endif %}
 
+        {{ display_og_meta(object) }}
+
         </div>
     {% if object.summary %}
     </div>
     {% endif %}
 
-    <div class="activity-attachment" style="margin-bottom:20px;">
+    <div class="activity-attachment">
     {{ display_attachments(object) }}
     </div>
 
@@ -449,6 +627,16 @@
         <li>
             <div><a href="{{ object.url }}"{% if object.is_from_inbox %} rel="nofollow"{% endif %} class="object-permalink u-url u-uid">permalink</a></div>
         </li>
+
+        {% if object.is_from_outbox and is_object_page and not is_admin and not request.url.path.startswith("/remote_interaction") %}
+            <li>
+                <a class="label-btn" href="{{ request.url_for("remote_interaction") }}?ap_id={{ object.ap_id }}">
+                    interact from your instance
+                </a>
+            </li>
+        {% endif %}
+
+
         {% if not is_article_mode %}
             <li>
                 <time class="dt-published" datetime="{{ object.ap_published_at.replace(microsecond=0).isoformat() }}" title="{{ object.ap_published_at.replace(microsecond=0).isoformat() }}">{{ object.ap_published_at | timeago }}</time>
@@ -496,7 +684,7 @@
 
         {% if (object.is_from_outbox or is_admin) and object.replies_count %}
         <li>
-            <a href="{% if is_admin and not object.is_from_outbox %}{{ url_for("admin_object") }}?ap_id={{ object.ap_id }}{% else %}{{ object.url }}{% endif %}"><strong>{{ object.replies_count }}</strong> repl{{ object.replies_count | pluralize("y", "ies") }}</a>
+            <a href="{% if is_admin and not object.is_from_outbox %}{{ url_for("admin_object") }}?ap_id={{ object.ap_id }}{% if object.in_reply_to %}#{{ object.permalink_id }}{% endif %}{% else %}{{ object.url }}{% endif %}"><strong>{{ object.replies_count }}</strong> repl{{ object.replies_count | pluralize("y", "ies") }}</a>
         </li>
         {% endif %}
 
@@ -508,7 +696,7 @@
     <ul>
         {% if object.is_from_outbox %}
             <li>
-                {{ admin_delete_button(object.ap_id) }}
+                {{ admin_delete_button(object) }}
             </li>
 
             <li>
@@ -544,7 +732,7 @@
             {% if object.visibility in [visibility_enum.PUBLIC, visibility_enum.UNLISTED] %}
             <li>
                 {% if object.announced_via_outbox_object_ap_id %}
-                    {{ admin_undo_button(object.liked_via_outbox_object_ap_id, "unshare") }}
+                    {{ admin_undo_button(object.announced_via_outbox_object_ap_id, "unshare") }}
                 {% else %}
                     {{ admin_announce_button(object.ap_id, permalink_id=object.permalink_id) }}
                 {% endif %}
@@ -559,7 +747,12 @@
         {% endif %}
         {% if object.is_from_inbox or object.is_from_outbox %}
             <li>
-                {{ admin_expand_button(object.ap_id) }}
+                {{ admin_expand_button(object) }}
+            </li>
+        {% endif %}
+        {% if object.is_from_inbox and not object.announced_via_outbox_object_ap_id %}
+            <li>
+                {{ admin_force_delete_button(object.ap_id) }}
             </li>
         {% endif %}
     </ul>
@@ -568,17 +761,17 @@
 
 
     {% if likes or shares or webmentions %}
-    <div style="display: flex;column-gap: 20px;flex-wrap: wrap;margin-top:20px;">
+    <div class="public-interactions">
         {% if likes %}
-            <div style="flex: 0 1 30%;max-width: 50%;">Likes
+            <div class="interactions-block">Likes
-                <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
+                <div class="facepile-wrapper">
                 {% for like in likes %}
-                    <a href="{% if is_admin %}{{ url_for("admin_profile") }}?actor_id={{ like.actor.ap_id }}{% else %}{{ like.actor.url }}{% endif %}" title="{{ like.actor.handle }}" style="height:50px;" rel="noreferrer">
+                    <a href="{% if is_admin and like.ap_actor_id %}{{ url_for("admin_profile") }}?actor_id={{ like.ap_actor_id }}{% else %}{{ like.url }}{% endif %}" title="{{ like.name }}" rel="noreferrer">
-                        <img src="{{ like.actor.resized_icon_url }}" alt="{{ like.actor.handle}}" style="max-width:50px;">
+                        <img src="{{ like.picture_url }}" alt="{{ like.name }}">
                     </a>
                 {% endfor %}
                 {% if object.likes_count > likes | length %}
-                    <div style="display:inline-block;align-self:center;">
+                    <div class="and-x-more">
                         and {{ object.likes_count - likes | length }} more.
                     </div>
                 {% endif %}
@@ -587,15 +780,15 @@
         {% endif %}
 
         {% if shares %}
-            <div style="flex: 0 1 30%;max-width: 50%;">Shares
+            <div class="interactions-block">Shares
-                <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
+                <div class="facepile-wrapper">
                 {% for share in shares %}
-                    <a href="{% if is_admin %}{{ url_for("admin_profile") }}?actor_id={{ share.actor.ap_id }}{% else %}{{ share.actor.url }}{% endif %}" title="{{ share.actor.handle }}" style="height:50px;" rel="noreferrer">
+                    <a href="{% if is_admin and share.ap_actor_id %}{{ url_for("admin_profile") }}?actor_id={{ share.ap_actor_id }}{% else %}{{ share.url }}{% endif %}" title="{{ share.name }}" rel="noreferrer">
-                        <img src="{{ share.actor.resized_icon_url }}" alt="{{ share.actor.handle}}" style="max-width:50px;">
+                        <img src="{{ share.picture_url }}" alt="{{ share.name }}">
                     </a>
                 {% endfor %}
                 {% if object.announces_count > shares | length %}
-                    <div style="display:inline-block;align-self:center;">
+                    <div class="and-x-more">
                         and {{ object.announces_count - shares | length }} more.
                     </div>
                 {% endif %}
@@ -604,13 +797,13 @@
         {% endif %}
 
         {% if webmentions %}
-            <div style="flex: 0 1 30%;max-width: 50%;">Webmentions
+            <div class="interactions-block">Webmentions
-                <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
+                <div class="facepile-wrapper">
                 {% for webmention in webmentions %}
                     {% set wm = webmention.as_facepile_item %}
                     {% if wm %}
-                        <a href="{{ wm.url }}" title="{{ wm.actor_name }}" style="height:50px;" rel="noreferrer">
+                        <a href="{{ wm.url }}" title="{{ wm.actor_name }}" rel="noreferrer">
-                            <img src="{{ wm.actor_icon_url | media_proxy_url }}" alt="{{ wm.actor_name }}" style="max-width:50px;">
+                            <img src="{{ wm.actor_icon_url | media_proxy_url }}" alt="{{ wm.actor_name }}">
                         </a>
                     {% endif %}
                 {% endfor %}
@@ -624,4 +817,5 @@
 
 </div>
 {% endif %}
+{% endblock %}
 {% endmacro %}

app/uploads.py

@@ -5,6 +5,7 @@ import blurhash  # type: ignore
 from fastapi import UploadFile
 from loguru import logger
 from PIL import Image
+from PIL import ImageOps
 from sqlalchemy import select
 
 from app import activitypub as ap
@@ -45,11 +46,13 @@ async def save_upload(db_session: AsyncSession, f: UploadFile) -> models.Upload:
     width = None
     height = None
 
-    if f.content_type.startswith("image"):
+    if f.content_type.startswith("image") and not f.content_type == "image/gif":
-        image_blurhash = blurhash.encode(f.file, x_components=4, y_components=3)
+        with Image.open(f.file) as _original_image:
-        f.file.seek(0)
+            # Fix image orientation (as we will remove the info from the EXIF
+            # metadata)
+            original_image = ImageOps.exif_transpose(_original_image)
 
-        with Image.open(f.file) as original_image:
+            # Re-creating the image drop the EXIF metadata
             destination_image = Image.new(
                 original_image.mode,
                 original_image.size,
@@ -57,15 +60,18 @@ async def save_upload(db_session: AsyncSession, f: UploadFile) -> models.Upload:
             destination_image.putdata(original_image.getdata())
             destination_image.save(
                 dest_filename,
-                format=original_image.format,
+                format=_original_image.format,
             )
 
+            with open(dest_filename, "rb") as dest_f:
+                image_blurhash = blurhash.encode(dest_f, x_components=4, y_components=3)
+
             try:
-                width, height = original_image.size
+                width, height = destination_image.size
-                original_image.thumbnail((740, 740))
+                destination_image.thumbnail((740, 740))
-                original_image.save(
+                destination_image.save(
                     UPLOAD_DIR / f"{content_hash}_resized",
-                    format=original_image.format,
+                    format="webp",
                 )
             except Exception:
                 logger.exception(

app/utils/custom_index_handler.py

@@ -0,0 +1,32 @@
+from typing import Any
+from typing import Awaitable
+from typing import Callable
+
+from fastapi import Depends
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+from app.actor import LOCAL_ACTOR
+from app.config import is_activitypub_requested
+from app.database import AsyncSession
+from app.database import get_db_session
+
+_Handler = Callable[[Request, AsyncSession], Awaitable[Any]]
+
+
+def build_custom_index_handler(handler: _Handler) -> _Handler:
+    async def custom_index(
+        request: Request,
+        db_session: AsyncSession = Depends(get_db_session),
+    ) -> Any:
+        # Serve the AP actor if requested
+        if is_activitypub_requested(request):
+            return JSONResponse(
+                LOCAL_ACTOR.ap_actor,
+                media_type="application/activity+json",
+            )
+
+        # Defer to the custom handler
+        return await handler(request, db_session)
+
+    return custom_index

app/utils/emoji.py

@@ -23,6 +23,8 @@ def _load_emojis(root_dir: Path, base_url: str) -> None:
             mt = mimetypes.guess_type(emoji.name)[0]
             if mt and mt.startswith("image/"):
                 name = emoji.name.split(".")[0]
+                if not re.match(EMOJI_REGEX, f":{name}:"):
+                    continue
                 ap_emoji: "RawObject" = {
                     "type": "Emoji",
                     "name": f":{name}:",

app/utils/facepile.py

@@ -0,0 +1,159 @@
+import datetime
+from dataclasses import dataclass
+from typing import Any
+from typing import Optional
+
+from loguru import logger
+
+from app import media
+from app.models import InboxObject
+from app.models import Webmention
+from app.utils.datetime import parse_isoformat
+from app.utils.url import make_abs
+
+
+@dataclass
+class Face:
+    ap_actor_id: str | None
+    url: str
+    name: str
+    picture_url: str
+    created_at: datetime.datetime
+
+    @classmethod
+    def from_inbox_object(cls, like: InboxObject) -> "Face":
+        return cls(
+            ap_actor_id=like.actor.ap_id,
+            url=like.actor.url,  # type: ignore
+            name=like.actor.handle,  # type: ignore
+            picture_url=like.actor.resized_icon_url,
+            created_at=like.created_at,  # type: ignore
+        )
+
+    @classmethod
+    def from_webmention(cls, webmention: Webmention) -> Optional["Face"]:
+        items = webmention.source_microformats.get("items", [])  # type: ignore
+        for item in items:
+            if item["type"][0] == "h-card":
+                try:
+                    return cls(
+                        ap_actor_id=None,
+                        url=(
+                            item["properties"]["url"][0]
+                            if item["properties"].get("url")
+                            else webmention.source
+                        ),
+                        name=item["properties"]["name"][0],
+                        picture_url=media.resized_media_url(
+                            make_abs(
+                                item["properties"]["photo"][0], webmention.source
+                            ),  # type: ignore
+                            50,
+                        ),
+                        created_at=webmention.created_at,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+            elif item["type"][0] == "h-entry":
+                author = item["properties"]["author"][0]
+                try:
+                    return cls(
+                        ap_actor_id=None,
+                        url=webmention.source,
+                        name=author["properties"]["name"][0],
+                        picture_url=media.resized_media_url(
+                            make_abs(
+                                author["properties"]["photo"][0], webmention.source
+                            ),  # type: ignore
+                            50,
+                        ),
+                        created_at=webmention.created_at,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+
+        return None
+
+
+def merge_faces(faces: list[Face]) -> list[Face]:
+    return sorted(
+        faces,
+        key=lambda f: f.created_at,
+        reverse=True,
+    )[:10]
+
+
+def _parse_face(webmention: Webmention, items: list[dict[str, Any]]) -> Face | None:
+    for item in items:
+        if item["type"][0] == "h-card":
+            try:
+                return Face(
+                    ap_actor_id=None,
+                    url=(
+                        item["properties"]["url"][0]
+                        if item["properties"].get("url")
+                        else webmention.source
+                    ),
+                    name=item["properties"]["name"][0],
+                    picture_url=media.resized_media_url(
+                        make_abs(
+                            item["properties"]["photo"][0], webmention.source
+                        ),  # type: ignore
+                        50,
+                    ),
+                    created_at=webmention.created_at,  # type: ignore
+                )
+            except Exception:
+                logger.exception(
+                    f"Failed to build Face for webmention id={webmention.id}"
+                )
+                break
+
+    return None
+
+
+@dataclass
+class WebmentionReply:
+    face: Face
+    content: str
+    url: str
+    published_at: datetime.datetime
+    in_reply_to: str
+    webmention_id: int
+
+    @classmethod
+    def from_webmention(cls, webmention: Webmention) -> Optional["WebmentionReply"]:
+        items = webmention.source_microformats.get("items", [])  # type: ignore
+        for item in items:
+            if item["type"][0] == "h-entry":
+                try:
+                    face = _parse_face(webmention, item["properties"].get("author", []))
+                    if not face:
+                        logger.info(
+                            "Failed to build WebmentionReply/Face for "
+                            f"webmention id={webmention.id}"
+                        )
+                        break
+                    return cls(
+                        face=face,
+                        content=item["properties"]["content"][0]["html"],
+                        url=item["properties"]["url"][0],
+                        published_at=parse_isoformat(
+                            item["properties"]["published"][0]
+                        ).replace(tzinfo=None),
+                        in_reply_to=webmention.target,  # type: ignore
+                        webmention_id=webmention.id,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+
+        return None

app/utils/highlight.py

@@ -1,3 +1,5 @@
+import base64
+import hashlib
 from functools import lru_cache
 
 from bs4 import BeautifulSoup  # type: ignore
@@ -11,6 +13,9 @@ from app.config import CODE_HIGHLIGHTING_THEME
 _FORMATTER = HtmlFormatter(style=CODE_HIGHLIGHTING_THEME)
 
 HIGHLIGHT_CSS = _FORMATTER.get_style_defs()
+HIGHLIGHT_CSS_HASH = base64.b64encode(
+    hashlib.sha256(HIGHLIGHT_CSS.encode()).digest()
+).decode()
 
 
 @lru_cache(256)

app/utils/opengraph.py

@@ -1,13 +1,18 @@
+import asyncio
 import mimetypes
 import re
+import signal
+from concurrent.futures import TimeoutError
 from typing import Any
 from urllib.parse import urlparse
 
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from loguru import logger
+from pebble import concurrent  # type: ignore
 from pydantic import BaseModel
 
+from app import activitypub as ap
 from app import ap_object
 from app import config
 from app.actor import LOCAL_ACTOR
@@ -27,7 +32,11 @@ class OpenGraphMeta(BaseModel):
     site_name: str
 
 
+@concurrent.process(timeout=5)
 def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
+    # Prevent SIGTERM to bubble up to the worker
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
     soup = BeautifulSoup(html, "html5lib")
     ogs = {
         og.attrs["property"]: og.attrs.get("content")
@@ -36,7 +45,7 @@ def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
     # FIXME some page have no <title>
     raw = {
         "url": url,
-        "title": soup.find("title").text,
+        "title": soup.find("title").text.strip(),
         "image": None,
         "description": None,
         "site_name": urlparse(url).hostname,
@@ -56,6 +65,10 @@ def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
     return OpenGraphMeta.parse_obj(raw)
 
 
+def scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
+    return _scrap_og_meta(url, html).result()
+
+
 async def external_urls(
     db_session: AsyncSession,
     ro: ap_object.RemoteObject | OutboxObject | InboxObject,
@@ -68,7 +81,12 @@ async def external_urls(
             tags_hrefs.add(tag_href)
         if tag.get("type") == "Mention":
             if tag["href"] != LOCAL_ACTOR.ap_id:
-                mentioned_actor = await fetch_actor(db_session, tag["href"])
+                try:
+                    mentioned_actor = await fetch_actor(db_session, tag["href"])
+                except (ap.FetchError, ap.NotAnObjectError):
+                    tags_hrefs.add(tag["href"])
+                    continue
+
                 tags_hrefs.add(mentioned_actor.url)
                 tags_hrefs.add(mentioned_actor.ap_id)
             else:
@@ -80,18 +98,25 @@ async def external_urls(
         soup = BeautifulSoup(ro.content, "html5lib")
         for link in soup.find_all("a"):
             h = link.get("href")
-            ph = urlparse(h)
+            if not h:
-            mimetype, _ = mimetypes.guess_type(h)
+                continue
-            if (
+
-                ph.scheme in {"http", "https"}
+            try:
-                and ph.hostname != note_host
+                ph = urlparse(h)
-                and is_url_valid(h)
+                mimetype, _ = mimetypes.guess_type(h)
-                and (
+                if (
-                    not mimetype
+                    ph.scheme in {"http", "https"}
-                    or mimetype.split("/")[0] not in ["image", "video", "audio"]
+                    and ph.hostname != note_host
-                )
+                    and is_url_valid(h)
-            ):
+                    and (
-                urls.add(h)
+                        not mimetype
+                        or mimetype.split("/")[0] not in ["image", "video", "audio"]
+                    )
+                ):
+                    urls.add(h)
+            except Exception:
+                logger.exception(f"Failed to check {h}")
+                continue
 
     return urls - tags_hrefs
 
@@ -112,7 +137,10 @@ async def _og_meta_from_url(url: str) -> OpenGraphMeta | None:
         return None
 
     try:
-        return _scrap_og_meta(url, resp.text)
+        return scrap_og_meta(url, resp.text)
+    except TimeoutError:
+        logger.info(f"Timed out when scraping OG meta for {url}")
+        return None
     except Exception:
         logger.info(f"Failed to scrap OG meta for {url}")
         return None
@@ -124,9 +152,21 @@ async def og_meta_from_note(
 ) -> list[dict[str, Any]]:
     og_meta = []
     urls = await external_urls(db_session, ro)
+    logger.debug(f"Lookig OG metadata in {urls=}")
     for url in urls:
+        logger.debug(f"Processing {url}")
         try:
-            maybe_og_meta = await _og_meta_from_url(url)
+            maybe_og_meta = None
+            try:
+                maybe_og_meta = await asyncio.wait_for(
+                    _og_meta_from_url(url),
+                    timeout=5,
+                )
+            except asyncio.TimeoutError:
+                logger.info(f"Timing out fetching {url}")
+            except Exception:
+                logger.exception(f"Failed scrap OG meta for {url}")
+
             if maybe_og_meta:
                 og_meta.append(maybe_og_meta.dict())
         except httpx.HTTPError:

app/utils/text.py

@@ -0,0 +1,8 @@
+import re
+import unicodedata
+
+
+def slugify(text: str) -> str:
+    value = unicodedata.normalize("NFKC", text)
+    value = re.sub(r"[^\w\s-]", "", value.lower())
+    return re.sub(r"[-\s]+", "-", value).strip("-_")

app/utils/url.py

@@ -58,6 +58,10 @@ def is_url_valid(url: str) -> bool:
         logger.warning(f"{parsed.hostname} is blocked")
         return False
 
+    if parsed.hostname.endswith(".onion"):
+        logger.warning(f"{url} is an onion service")
+        return False
+
     ip_address = _getaddrinfo(
         parsed.hostname, parsed.port or (80 if parsed.scheme == "http" else 443)
     )

app/utils/version.py

@@ -0,0 +1,12 @@
+import subprocess
+
+
+def get_version_commit() -> str:
+    try:
+        return (
+            subprocess.check_output(["git", "rev-parse", "--short=8", "v2"])
+            .split()[0]
+            .decode()
+        )
+    except Exception:
+        return "dev"

app/utils/workers.py

@@ -54,15 +54,20 @@ class Worker(Generic[T]):
                 {task, stop_task}, return_when=asyncio.FIRST_COMPLETED
             )
             logger.info(f"Waiting for tasks to finish {done=}/{pending=}")
-            await asyncio.sleep(5)
             tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
             logger.info(f"Cancelling {len(tasks)} tasks")
             [task.cancel() for task in tasks]
 
-        await asyncio.gather(*tasks, return_exceptions=True)
+        try:
+            await asyncio.wait_for(
+                asyncio.gather(*tasks, return_exceptions=True),
+                timeout=15,
+            )
+        except asyncio.TimeoutError:
+            logger.info("Tasks failed to cancel")
 
         logger.info("stopping loop")
 
     async def _shutdown(self, sig: signal.Signals) -> None:
-        logger.info(f"Caught {signal=}")
+        logger.info(f"Caught {sig=}")
         self._stop_event.set()

app/utils/yunohost.py

@@ -6,7 +6,6 @@ from typing import Any
 
 import bcrypt
 import tomli_w
-from markdown import markdown  # type: ignore
 
 from app.key import generate_key
 
@@ -44,7 +43,7 @@ def setup_config_file(
     dat["username"] = username
     dat["admin_password"] = bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
     dat["name"] = name
-    dat["summary"] = markdown(summary)
+    dat["summary"] = summary
     dat["https"] = True
     proto = "https"
     dat["icon_url"] = f'{proto}://{dat["domain"]}/static/nopic.png'

app/webfinger.py

@@ -12,6 +12,7 @@ async def webfinger(
     resource: str,
 ) -> dict[str, Any] | None:  # noqa: C901
     """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL."""
+    resource = resource.strip()
     logger.info(f"performing webfinger resolution for {resource}")
     protos = ["https", "http"]
     if resource.startswith("http://"):
@@ -29,6 +30,7 @@ async def webfinger(
 
     is_404 = False
 
+    resp: httpx.Response | None = None
     async with httpx.AsyncClient() as client:
         for i, proto in enumerate(protos):
             try:
@@ -59,7 +61,10 @@ async def webfinger(
     if is_404:
         return None
 
-    return resp.json()
+    if resp:
+        return resp.json()
+    else:
+        return None
 
 
 async def get_remote_follow_template(resource: str) -> str | None:

app/webmentions.py

@@ -1,3 +1,5 @@
+from urllib.parse import urlparse
+
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from fastapi import APIRouter
@@ -6,13 +8,20 @@ from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
 from loguru import logger
+from sqlalchemy import func
 from sqlalchemy import select
 
 from app import models
+from app.boxes import _get_outbox_announces_count
+from app.boxes import _get_outbox_likes_count
+from app.boxes import _get_outbox_replies_count
 from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import get_outbox_object_by_slug_and_short_id
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.utils import microformats
+from app.utils.facepile import Face
+from app.utils.facepile import WebmentionReply
 from app.utils.url import check_url
 from app.utils.url import is_url_valid
 
@@ -47,6 +56,7 @@ async def webmention_endpoint(
 
         check_url(source)
         check_url(target)
+        parsed_target_url = urlparse(target)
     except Exception:
         logger.exception("Invalid webmention request")
         raise HTTPException(status_code=400, detail="Invalid payload")
@@ -65,6 +75,16 @@ async def webmention_endpoint(
         logger.info("Found existing Webmention, will try to update or delete")
 
     mentioned_object = await get_outbox_object_by_ap_id(db_session, target)
+
+    if not mentioned_object and parsed_target_url.path.startswith("/articles/"):
+        try:
+            _, _, short_id, slug = parsed_target_url.path.split("/")
+            mentioned_object = await get_outbox_object_by_slug_and_short_id(
+                db_session, slug, short_id
+            )
+        except Exception:
+            logger.exception(f"Failed to match {target}")
+
     if not mentioned_object:
         logger.info(f"Invalid target {target=}")
 
@@ -90,8 +110,13 @@ async def webmention_endpoint(
         logger.warning(f"target {target=} not found in source")
         if existing_webmention_in_db:
             logger.info("Deleting existing Webmention")
-            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
             existing_webmention_in_db.is_deleted = True
+            await db_session.flush()
+
+            # Revert side effects
+            await _handle_webmention_side_effects(
+                db_session, existing_webmention_in_db, mentioned_object
+            )
 
             notif = models.Notification(
                 notification_type=models.NotificationType.DELETED_WEBMENTION,
@@ -110,10 +135,14 @@ async def webmention_endpoint(
         else:
             return JSONResponse(content={}, status_code=200)
 
+    webmention_type = models.WebmentionType.UNKNOWN
+    webmention: models.Webmention
     if existing_webmention_in_db:
         # Undelete if needed
         existing_webmention_in_db.is_deleted = False
         existing_webmention_in_db.source_microformats = data
+        await db_session.flush()
+        webmention = existing_webmention_in_db
 
         notif = models.Notification(
             notification_type=models.NotificationType.UPDATED_WEBMENTION,
@@ -127,9 +156,11 @@ async def webmention_endpoint(
             target=target,
             source_microformats=data,
             outbox_object_id=mentioned_object.id,
+            webmention_type=webmention_type,
         )
         db_session.add(new_webmention)
         await db_session.flush()
+        webmention = new_webmention
 
         notif = models.Notification(
             notification_type=models.NotificationType.NEW_WEBMENTION,
@@ -138,8 +169,60 @@ async def webmention_endpoint(
         )
         db_session.add(notif)
 
-        mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
+    # Determine the webmention type
+    for item in data.get("items", []):
+        if target in item.get("properties", {}).get(
+            "in-reply-to", []
+        ) and WebmentionReply.from_webmention(webmention):
+            webmention_type = models.WebmentionType.REPLY
+            break
+        elif target in item.get("properties", {}).get(
+            "like-of", []
+        ) and Face.from_webmention(webmention):
+            webmention_type = models.WebmentionType.LIKE
+            break
+        elif target in item.get("properties", {}).get(
+            "repost-of", []
+        ) and Face.from_webmention(webmention):
+            webmention_type = models.WebmentionType.REPOST
+            break
 
+    if webmention_type != models.WebmentionType.UNKNOWN:
+        webmention.webmention_type = webmention_type
+        await db_session.flush()
+
+    # Handle side effect
+    await _handle_webmention_side_effects(db_session, webmention, mentioned_object)
     await db_session.commit()
 
     return JSONResponse(content={}, status_code=200)
+
+
+async def _handle_webmention_side_effects(
+    db_session: AsyncSession,
+    webmention: models.Webmention,
+    mentioned_object: models.OutboxObject,
+) -> None:
+    if webmention.webmention_type == models.WebmentionType.UNKNOWN:
+        # TODO: recount everything
+        mentioned_object.webmentions_count = await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == mentioned_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.UNKNOWN,
+            )
+        )
+    elif webmention.webmention_type == models.WebmentionType.LIKE:
+        mentioned_object.likes_count = await _get_outbox_likes_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPOST:
+        mentioned_object.announces_count = await _get_outbox_announces_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPLY:
+        mentioned_object.replies_count = await _get_outbox_replies_count(
+            db_session, mentioned_object
+        )
+    else:
+        raise ValueError(f"Unhandled {webmention.webmention_type} webmention")

data/_theme.scss

@@ -1 +0,0 @@
-// override vars for theming here

data/templates/app

@@ -0,0 +1 @@
+../../app/templates/

data/tests.toml

@@ -5,6 +5,7 @@ admin_password = "$2b$12$OwCyZM33uXQUVrChgER.h.qgFJ4fBp6tdFwArR3Lm1LV8NgMvIxVa"
 name = "test"
 summary = "<p>Hello</p>"
 https = false
+id = "http://localhost:8000"
 icon_url = "https://localhost:8000/static/nopic.png"
 secret = "1dd4079e0474d1a519052b8fe3cb5fa6"
 debug = true

docs/developer_guide.md

@@ -1,6 +1,6 @@
 # Developer's guide
 
-This guide assume you have some knoweldge of [ActivityPub](https://activitypub.rocks/).
+This guide assumes you have some knowledge of [ActivityPub](https://activitypub.rocks/).
 
 [TOC]
 
@@ -10,12 +10,13 @@ Microblog.pub is a "modern" Python application with "old-school" server-rendered
 
  - [Poetry](https://python-poetry.org/) is used for dependency management.
  - Most of the code is asynchronous, using [asyncio](https://docs.python.org/3/library/asyncio.html).
- - SQLite3 is the default database.
+ - SQLite3 for data storage
 
-The server has 2 components:
+The server has 3 components:
 
  - The web server (powered by [FastAPI](https://fastapi.tiangolo.com/) and [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/) templates)
- - An additional process that takes care of sending "outgoing actities" 
+ - One process that takes care of sending "outgoing activities" 
+ - One process that takes care of processing "incoming activities" 
 
 ### Tasks
 
@@ -29,7 +30,7 @@ inv -l
 
 ### Media storage
 
-The uploads are stored in the `data/` directory, using a simple content-addressed storage (file contents hash is the name of the store BLOB).
+The uploads are stored in the `data/` directory, using a simple content-addressed storage system (file contents hash is BLOB filename).
 Files metadata are stored in the database.
 
 ## Installation
@@ -57,3 +58,24 @@ And check out the result by starting a static server using Python standard libra
 cd docs/dist
 python -m http.server 8001
 ```
+
+## Contributing
+
+Contributions/patches are welcome, but please start a discussion in a [ticket](https://todo.sr.ht/~tsileo/microblog.pub) or a [thread in the mailing list](https://lists.sr.ht/~tsileo/microblog.pub-devel) before working on anything consequent.
+
+### Patches
+
+Please ensure your code passes the code quality checks:
+
+```bash
+inv autoformat
+inv lint
+```
+
+And that the tests suite is passing:
+
+```bash
+inv tests
+```
+
+Please also consider adding new test cases if needed.

docs/install.md

@@ -11,7 +11,7 @@ For now, there's no image published on Docker Hub, this means you will have to b
 Clone the repository, replace `you-domain.tld` by your own domain.
 
 Note that if you want to serve static assets via your reverse proxy (like nginx), clone it in a place
-where accessible by your reverse proxy user.
+where it is accessible by your reverse proxy user.
 
 ```bash
 git clone https://git.sr.ht/~tsileo/microblog.pub your-domain.tld
@@ -55,6 +55,12 @@ docker compose stop
 docker compose up -d
 ```
 
+As you probably already know, Docker can (and will) eat a lot of disk space, when updating you should [prune old images](https://docs.docker.com/config/pruning/#prune-images) from time to time:
+
+```bash
+docker image prune -a --filter "until=24h"
+```
+
 ## Python developer edition
 
 Assuming you have a working **Python 3.10+** environment. 
@@ -83,6 +89,12 @@ Setup config.
 poetry run inv configuration-wizard
 ```
 
+Setup the database.
+
+```bash
+poetry run inv migrate-db
+```
+
 Grab your virtualenv path.
 
 ```bash
@@ -99,7 +111,7 @@ Setup a reverse proxy (see the next section).
 
 ### Updating 
 
-To update microblogpub locally, pull the remote changes and run the `update` task to regeneratee the CSS and run any DB migrations.
+To update microblogpub locally, pull the remote changes and run the `update` task to regenerate the CSS and run any DB migrations.
 
 ```bash
 git pull
@@ -130,6 +142,11 @@ server {
     # [...]
 }
 
+# This should be outside the `server` block
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  '' close;
+}
 ```
 
 Optionally, you can serve static files using NGINX directly, with an additional `location` block.
@@ -147,8 +164,37 @@ server {
        # path for static files
        rewrite ^/static/(.*) /$1 break;
        root /path/to/your-domain.tld/app/static/;
+       expires 1y;
     }
 
     # [...]
 }
 ```
+
+### NGINX config tips
+
+Enable HTTP2 (which is disabled by default):
+
+```nginx
+server {
+    # [...]
+    listen [::]:443 ssl http2;
+}
+```
+
+Tweak `/etc/nginx/nginx.conf` and add gzip compression for ActivityPub responses:
+
+```nginx
+http {
+    # [...]
+    gzip_types text/plain text/css application/json application/javascript application/activity+json application/octet-stream;
+}
+```
+
+## YunoHost edition
+
+[YunoHost](https://yunohost.org/) support is available (although it is not an official package for now): <https://git.sr.ht/~tsileo/microblog.pub_ynh>.
+
+## Available tutorial/guides
+
+ - [Opalstack](https://community.opalstack.com/d/1055-howto-install-and-run-microblogpub-on-opalstack), thanks to [@defulmere@mastodon.social](https://mastodon.online/@defulmere).

docs/templates/layout.html

@@ -63,7 +63,7 @@ nav a:hover, main a:hover, header p a:hover {
     max-width: 960px;
     margin: 50px auto;
 }
-pre code {
+pre {
     padding: 10px;
     overflow: auto;
     display: block;

docs/user_guide.md

@@ -29,18 +29,69 @@ You can tweak your profile by tweaking these items:
  - `summary` (using Markdown)
  - `icon_url`
 
-Whenever one of these config items is updated, an `Update` activity will be sent to all know server to update your remote profile.
+Whenever one of these config items is updated, an `Update` activity will be sent to all known servers to update your remote profile.
+
+The server will need to be restarted for taking changes into account.
+
+Before restarting the server, you can ensure you haven't made any mistakes by running the [configuration checking task](/user_guide.html#configuration-checking).
+
+
+### Profile metadata
+
+You can add metadata to your profile with the `metadata` config item.
+
+Markdown is supported in the `value` field.
+
+Be aware that most other software like Mastodon will limit the number of key/value to 4.
+
+```toml
+metadata = [
+  {key = "Documentation", value = "[https://docs.microblog.pub](https://docs.microblog.pub)"},
+  {key = "Source code", value = "[https://sr.ht/~tsileo/microblog.pub/](https://sr.ht/~tsileo/microblog.pub/)"},
+]
+```
+
+### Manually approving followers
+
+If you wish to manually approve followers, add this config item to `profile.toml`:
+
+```toml
+manually_approves_followers = true
+```
+
+The default value is `false`.
+
+### Hiding followers
+
+If you wish to hide your followers, add this config item to `profile.toml`:
+
+```toml
+hides_followers = true
+```
+
+The default value is `false`.
+
+### Hiding who you are following
+
+If you wish to hide who you are following, add this config item to `profile.toml`:
+
+```toml
+hides_following = true
+```
+
+The default value is `false`.
 
 ### Privacy replace
 
-You can define domain to be rewrited to more "privacy friendly" alternatives, like [Invidious](https://invidious.io/)
+You can define domains to be rewritten to more "privacy friendly" alternatives, like [Invidious](https://invidious.io/)
 or [Nitter](https://nitter.net/about).
 
-To do so, just add as these extra config items, this is a sample config that rewrite URLs for Twitter, Youtube, Reddit and Medium:
+To do so, add these extra config items. This is a sample config that rewrite URLs for Twitter, Youtube, Reddit and Medium:
 
 ```toml
 privacy_replace = [
     {domain = "youtube.com", replace_by  = "yewtu.be"},
+    {domain = "youtu.be", replace_by  = "yewtu.be"},
     {domain = "twitter.com", replace_by = "nitter.fdn.fr"},
     {domain = "medium.com", replace_by = "scribe.rip"},
     {domain = "reddit.com", replace_by = "teddit.net"},
@@ -49,9 +100,20 @@ privacy_replace = [
 
 ### Customization
 
+#### Default emoji
+
+If you don't like cats, or need more emoji, you can add your favorite emoji in `profile.toml` and it will replace the default ones:
+
+```
+emoji = "🙂🐹📌"
+```
+
+You can copy/paste them from [getemoji.com](https://getemoji.com/).
+
 #### Custom emoji
 
 You can add custom emoji in the `data/custom_emoji` directory and they will be picked automatically.
+Do not use exotic characters in filename - only letters, numbers, and underscore symbol `_` are allowed.
 
 #### Custom CSS
 
@@ -64,19 +126,57 @@ $primary-color: #e14eea;
 $secondary-color: #32cd32;
 ```
 
-See `app/scss/main.scss` to see what variables can be overidden.
+See `app/scss/main.scss` to see what variables can be overridden.
+
+#### Custom templates
+
+If you'd like to customize your instance's theme beyond CSS, you can modify the app's HTML by placing templates in `data/templates` which overwrite the defaults in `app/templates`.
+
+#### Custom Content Security Policy (CSP)
+
+You can override the default Content Security Policy by adding a line in `data/profile.toml`:
+
+```toml
+custom_content_security_policy = "default-src 'self'; style-src 'self' 'sha256-{HIGHLIGHT_CSS_HASH}'; frame-ancestors 'none'; base-uri 'self'; form-action 'self';"
+```
+
+This example will output the default CSP, note that `{HIGHLIGHT_CSS_HASH}` will be dynamically replaced by the correct value (the hash of the CSS needed for syntax highlighting).
+
+#### Code highlighting theme
+
+You can switch to one of the [styles supported by Pygments](https://pygments.org/styles/) by adding a line in `data/profile.toml`:
+
+```toml
+code_highlighting_theme = "solarized-dark"
+```
+
+### Blocking servers
+
+In addition to blocking "single actors" via the admin interface, you can also prevent any communication with entire servers.
+
+Add a `blocked_servers` config item into `profile.toml`.
+
+The `reason` field is just there to help you document/remember why a server was blocked.
+
+You should unfollow any account from a server before blocking it.
+
+```toml
+blocked_servers = [
+    {hostname = "bad.tld", reason = "Bot spam"},
+]
+```
 
 ## Public website
 
 Public notes will be visible on the homepage.
 
-Only the last 20 followers/follows you be showing on the public website.
+Only the last 20 followers/follows you have will be shown on the public website.
 
 And only the last 20 interactions (likes/shares/webmentions) will be displayed, to keep things simple/clean.
 
 ## Admin section
 
-You can login to the admin section by clicking on the `Admin` link in the footer or by visiting `https://yourdomain.tld/admin`.
+You can login to the admin section by clicking on the `Admin` link in the footer or by visiting `https://yourdomain.tld/admin/login`.
 The password is the one set during the initial configuration.
 
 ### Lookup
@@ -129,35 +229,36 @@ microblog.pub supports the most common interactions supported by the Fediverse.
 
 ### Shares
 
-Sharing an object will relay it to your followers and notify the author.
+Sharing (or announcing) an object will relay it to your followers and notify the author.
 It will also be displayed on the homepage.
 
 Most receiving servers will increment the number of shares.
 
-TODO receiving
+Receiving a share will trigger a notification, increment the shares counter on the object and the actor avatar will be displayed on the object permalink.
 
 ### Likes
 
 Liking an object will notify the author.
-Unkike sharing, liked object are not displayed on the homepage.
+
+Unlike sharing, liked objects are not displayed on the homepage.
 
 Most receiving servers will increment the number of likes.
 
-TODO receiving
+Receiving a like will trigger a notification, increment the likes counter on the object and the actor avatar will be displayed on the object permalink.
 
 ### Bookmarks
 
 Bookmarks allow you to like objects without notifying the author.
 
-It is basically a "private like", and allow you to easily access them later.
+It is basically a "private like", and allows you to easily access them later.
 
-TODO receiving
+It will also prevent objects to be pruned.
 
 ### Webmentions
 
-Sending webmention to ping mentioned website is done automatically once a note is authored, see TODO.
+Sending webmentions to ping mentioned websites is done automatically once a public note is authored.
 
-TODO side-effect of receiving a webmention.
+Receiving a webmention will trigger a notification, increment the webmentions counter on the object and the source page will be displayed on the object permalink.
 
 ## Backup and restore
 
@@ -171,3 +272,201 @@ All the data generated by the server is located in the `data/` directory:
  - Uploaded media
 
 Restoring is as easy as adding your backed up `data/` directory into a fresh deployment.
+
+## Moving from another instance
+
+If you want to move followers from your existing account, ensure it is supported in your software documentation.
+
+For [Mastodon you can look at Moving or leaving accounts](https://docs.joinmastodon.org/user/moving/).
+
+If you wish to move **to** another instance, see [Moving to another instance](/user_guide.html#moving-to-another-instance).
+
+First you need to grab the "ActivityPub actor URL" for your existing account:
+
+### Python edition
+
+```bash
+# For a Python install
+poetry run inv webfinger username@domain.tld
+```
+
+Edit the config.
+
+### Docker edition
+
+```bash
+# For a Docker install
+make account=username@domain.tld webfinger
+```
+
+Edit the config.
+
+### Edit the config
+
+And add a reference to your old/existing account in `profile.toml`:
+
+```toml
+also_known_as = "my@old-account.com"
+```
+
+Restart the server, and you should be able to complete the move from your existing account.
+
+## Tasks
+
+### Configuration checking
+
+You can confirm that your configuration file (`data/profile.toml`) is valid using the `check-config`
+
+#### Python edition
+
+```bash
+poetry run inv check-config
+```
+
+#### Docker edition
+
+```bash
+make check-config
+```
+
+### Recompiling CSS files
+
+You can ensure your custom theme is valid by recompiling the CSS manually using the `compile-scss` task.
+
+#### Python edition
+
+```bash
+poetry run inv compile-scss
+```
+
+#### Docker edition
+
+```bash
+make compile-scss
+```
+
+
+### Password reset
+
+If have lost your password, you can generate a new one using the `reset-password` task.
+
+#### Python edition
+
+```bash
+# shutdown supervisord
+poetry run inv reset-password
+# edit data/profile.toml
+# restart supervisord
+```
+
+#### Docker edition
+
+```bash
+docker compose stop
+make reset-password
+# edit data/profile.toml
+docker compose up -d
+```
+
+### Pruning old data
+
+You should prune old data from time to time to free disk space.
+
+The default retention for the inbox data is 15 days.
+
+It's configurable via the `inbox_retention_days` config item in `profile.toml`:
+
+```toml
+inbox_retention_days = 30
+```
+
+Data owned by the server will never be deleted (at least for now), along with:
+
+ - bookmarked objects
+ - liked objects
+ - shared objects
+ - inbox objects mentioning the local actor
+ - objects related to local conversations (i.e. direct messages, replies) 
+
+For now, it's recommended to make a backup before running the task in case it deletes unwanted data.
+
+You should shutdown the server before running the task.
+
+#### Python edition
+
+```bash
+# shutdown supervisord
+cp -r data/microblogpub.db data/microblogpub.db.bak
+poetry run inv prune-old-data
+# relaunch supervisord and ensure it works as expected
+rm data/microblogpub.db.bak
+```
+
+#### Docker edition
+
+```bash
+docker compose stop
+cp -r data/microblogpub.db data/microblogpub.db.bak
+make prune-old-data
+docker compose up -d
+rm data/microblogpub.db.bak
+```
+
+### Moving to another instance
+
+If you want to migrate to another instance, you have the ability to move your existing followers to your new account.
+
+Your new account should reference the existing one, refer to your software configuration (for example [Moving or leaving accounts from the Mastodon doc](https://docs.joinmastodon.org/user/moving/)).
+
+If you wish to move **from** another instance, see [Moving from another instance](/user_guide.html#moving-from-another-instance).
+
+Execute the Move task:
+
+#### Python edition
+
+```bash
+# For a Python install
+poetry run inv move-to username@domain.tld
+```
+
+#### Docker edition
+
+```bash
+# For a Docker install
+make account=username@domain.tld move-to
+```
+
+### Deleting the instance
+
+If you want to delete your instance, you can request other instances to delete your remote profile.
+
+Note that this is a best-effort delete as some instances may not delete your data.
+
+The command won't remove any local data, it just broadcasts account deletion messages to all known servers.
+
+After executing the command, you should let the server run until all the outgoing delete tasks are sent.
+
+Once deleted, you won't be able to use your instance anymore, but you will be able to perform a fresh re-install of any ActivityPub software.
+
+#### Python edition
+
+```bash
+# For a Python install
+poetry run inv self-destruct
+```
+
+#### Docker edition
+
+```bash
+# For a Docker install
+make self-destruct
+```
+
+## Troubleshooting
+
+If the server is not (re)starting, you can:
+
+ - [Ensure that the configuration is valid](/user_guide.html#configuration-checking).
+ - [Verify if you haven't any syntax error in the custom theme by recompiling the CSS](/user_guide.html#recompiling-css-files).
+ - Look at the log files (in `data/uvicorn.log`, `data/incoming.log` and `data/outgoing.log`).
+ - If the CSS is not working, ensure your reverse proxy is serving the static file correctly.

misc/docker-supervisord.conf

@@ -4,11 +4,10 @@ logfile=/dev/null
 logfile_maxbytes=0
 pidfile=data/supervisord.pid
 
-[fcgi-program:uvicorn]
+[program:uvicorn]
-socket=tcp://0.0.0.0:8000
+command=uvicorn app.main:app --no-server-header --host 0.0.0.0
-command=uvicorn app.main:app --no-server-header --fd 0
+numprocs=1
-numprocs=2
+autorestart=true
-process_name=uvicorn-%(process_num)d
 redirect_stderr=true
 stdout_logfile=data/uvicorn.log
 stdout_logfile_maxbytes=50MB
@@ -16,6 +15,7 @@ stdout_logfile_maxbytes=50MB
 [program:incoming_worker]
 command=inv process-incoming-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=data/incoming.log
 stdout_logfile_maxbytes=50MB
@@ -23,6 +23,7 @@ stdout_logfile_maxbytes=50MB
 [program:outgoing_worker]
 command=inv process-outgoing-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=data/outgoing.log
 stdout_logfile_maxbytes=50MB

misc/supervisord.conf

@@ -1,24 +1,25 @@
 [supervisord]
 
-[fcgi-program:uvicorn]
+[program:uvicorn]
-socket=tcp://localhost:8000
+command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header
-command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header --fd 0
+numprocs=1
-numprocs=2
+autorestart=true
-process_name=uvicorn-%(process_num)d
 redirect_stderr=true
 stdout_logfile=uvicorn.log
-stdout_logfile_maxbytes=0
+stdout_logfile_maxbytes=50MB
 
 [program:incoming_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-incoming-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=incoming_worker.log
-stdout_logfile_maxbytes=0
+stdout_logfile_maxbytes=50MB
 
 [program:outgoing_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-outgoing-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=outgoing_worker.log
-stdout_logfile_maxbytes=0
+stdout_logfile_maxbytes=50MB

misc/ynh-supervisord.conf

@@ -1,24 +1,26 @@
 [supervisord]
 
-[fcgi-program:uvicorn]
+[program:uvicorn]
-socket=tcp://localhost:%(ENV_UVICORN_PORT)s
+command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header
-command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header --fd 0
+numprocs=1
-numprocs=2
+autorestart=true
 process_name=uvicorn-%(process_num)d
 redirect_stderr=true
-stdout_logfile=uvicorn.log
+stdout_logfile=%(ENV_LOG_PATH)s/uvicorn.log
 stdout_logfile_maxbytes=0
 
 [program:incoming_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-incoming-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
-stdout_logfile=incoming_worker.log
+stdout_logfile=%(ENV_LOG_PATH)s/incoming.log
 stdout_logfile_maxbytes=0
 
 [program:outgoing_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-outgoing-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
-stdout_logfile=outgoing_worker.log
+stdout_logfile=%(ENV_LOG_PATH)s/outgoing.log
 stdout_logfile_maxbytes=0

pyproject.toml

@@ -9,7 +9,6 @@ license = "AGPL-3.0"
 python = "^3.10"
 Jinja2 = "^3.1.2"
 fastapi = "^0.78.0"
-uvicorn = "^0.17.6"
 pycryptodome = "^3.14.1"
 bcrypt = "^3.2.2"
 itsdangerous = "^2.1.2"
@@ -19,7 +18,6 @@ httpx = {extras = ["http2"], version = "^0.23.0"}
 SQLAlchemy = {extras = ["asyncio"], version = "^1.4.39"}
 alembic = "^1.8.0"
 bleach = "^5.0.0"
-Markdown = "^3.3.7"
 prompt-toolkit = "^3.0.29"
 tomli-w = "^1.0.0"
 python-dateutil = "^2.8.2"
@@ -28,7 +26,6 @@ html5lib = "^1.1"
 mf2py = "^1.1.2"
 Pygments = "^2.12.0"
 loguru = "^0.6.0"
-mdx-linkify = "^2.1"
 Pillow = "^9.1.1"
 blurhash-python = "^1.1.3"
 html2text = "^2020.1.16"
@@ -43,6 +40,11 @@ asgiref = "^3.5.2"
 supervisor = "^4.2.4"
 invoke = "^1.7.1"
 boussole = "^2.0.0"
+uvicorn = {extras = ["standard"], version = "^0.18.3"}
+Brotli = "^1.0.9"
+greenlet = "^1.1.3"
+mistletoe = "^0.9.0"
+Pebble = "^5.0.2"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"

scripts/build_docs.py

@@ -1,19 +1,115 @@
+import re
 import shutil
+import typing
 from pathlib import Path
+from typing import Any
 
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 from jinja2 import select_autoescape
-from markdown import markdown
+from mistletoe import Document  # type: ignore
+from mistletoe import HTMLRenderer  # type: ignore
+from mistletoe import block_token  # type: ignore
+from pygments import highlight  # type: ignore
+from pygments.formatters import HtmlFormatter  # type: ignore
+from pygments.lexers import get_lexer_by_name as get_lexer  # type: ignore
+from pygments.lexers import guess_lexer  # type: ignore
 
 from app.config import VERSION
+from app.source import CustomRenderer
 from app.utils.datetime import now
 
+_FORMATTER = HtmlFormatter()
+_FORMATTER.noclasses = True
 
-def markdownify(content: str) -> str:
+
-    return markdown(
+class DocRenderer(CustomRenderer):
-        content, extensions=["mdx_linkify", "fenced_code", "codehilite", "toc"]
+    def __init__(
-    )
+        self,
+        depth=5,
+        omit_title=True,
+        filter_conds=[],
+    ) -> None:
+        super().__init__(
+            enable_mentionify=False,
+            enable_hashtagify=False,
+        )
+        self._headings: list[tuple[int, str, str]] = []
+        self._ids: set[str] = set()
+        self.depth = depth
+        self.omit_title = omit_title
+        self.filter_conds = filter_conds
+
+    @property
+    def toc(self):
+        """
+        Returns table of contents as a block_token.List instance.
+        """
+
+        def get_indent(level):
+            if self.omit_title:
+                level -= 1
+            return " " * 4 * (level - 1)
+
+        def build_list_item(heading):
+            level, content, title_id = heading
+            template = '{indent}- <a href="#{id}" rel="nofollow">{content}</a>\n'
+            return template.format(
+                indent=get_indent(level), content=content, id=title_id
+            )
+
+        lines = [build_list_item(heading) for heading in self._headings]
+        items = block_token.tokenize(lines)
+        return items[0]
+
+    def render_heading(self, token):
+        """
+        Overrides super().render_heading; stores rendered heading first,
+        then returns it.
+        """
+        template = '<h{level} id="{id}">{inner}</h{level}>'
+        inner = self.render_inner(token)
+        title_id = inner.lower().replace(" ", "-")
+        if title_id in self._ids:
+            i = 1
+            while 1:
+                title_id = f"{title_id}_{i}"
+                if title_id not in self._ids:
+                    break
+        self._ids.add(title_id)
+        rendered = template.format(level=token.level, inner=inner, id=title_id)
+        content = self.parse_rendered_heading(rendered)
+
+        if not (
+            self.omit_title
+            and token.level == 1
+            or token.level > self.depth
+            or any(cond(content) for cond in self.filter_conds)
+        ):
+            self._headings.append((token.level, content, title_id))
+        return rendered
+
+    @staticmethod
+    def parse_rendered_heading(rendered):
+        """
+        Helper method; converts rendered heading to plain text.
+        """
+        return re.sub(r"<.+?>", "", rendered)
+
+    def render_block_code(self, token: typing.Any) -> str:
+        code = token.children[0].content
+        lexer = get_lexer(token.language) if token.language else guess_lexer(code)
+        return highlight(code, lexer, _FORMATTER)
+
+
+def markdownify(content: str) -> tuple[str, Any]:
+    with DocRenderer() as renderer:
+        rendered_content = renderer.render(Document(content))
+
+    with HTMLRenderer() as html_renderer:
+        toc = html_renderer.render(renderer.toc)
+
+    return rendered_content, toc
 
 
 def main() -> None:
@@ -30,32 +126,36 @@ def main() -> None:
     last_updated = now().replace(second=0, microsecond=0).isoformat()
 
     readme = Path("README.md")
+    content, toc = markdownify(readme.read_text().removeprefix("# microblog.pub"))
     template.stream(
-        content=markdownify(readme.read_text().removeprefix("# microblog.pub")),
+        content=content,
         version=VERSION,
         path="/",
         last_updated=last_updated,
     ).dump("docs/dist/index.html")
 
     install = Path("docs/install.md")
+    content, toc = markdownify(install.read_text())
     template.stream(
-        content=markdownify(install.read_text()),
+        content=content.replace("[TOC]", toc),
         version=VERSION,
         path="/installing.html",
         last_updated=last_updated,
     ).dump("docs/dist/installing.html")
 
     user_guide = Path("docs/user_guide.md")
+    content, toc = markdownify(user_guide.read_text())
     template.stream(
-        content=markdownify(user_guide.read_text()),
+        content=content.replace("[TOC]", toc),
         version=VERSION,
         path="/user_guide.html",
         last_updated=last_updated,
     ).dump("docs/dist/user_guide.html")
 
     developer_guide = Path("docs/developer_guide.md")
+    content, toc = markdownify(developer_guide.read_text())
     template.stream(
-        content=markdownify(developer_guide.read_text()),
+        content=content.replace("[TOC]", toc),
         version=VERSION,
         path="/developer_guide.html",
         last_updated=last_updated,

scripts/config_wizard.py

@@ -6,7 +6,6 @@ from typing import Any
 
 import bcrypt
 import tomli_w
-from markdown import markdown  # type: ignore
 from prompt_toolkit import prompt
 from prompt_toolkit.key_binding import KeyBindings
 
@@ -26,6 +25,10 @@ def _(event):
 
 
 def main() -> None:
+    theme_file = Path("data/_theme.scss")
+    if not theme_file.exists():
+        theme_file.write_text("// override vars for theming here")
+
     print("Welcome to microblog.pub setup wizard\n")
     print("Generating key...")
     if _KEY_PATH.exists():
@@ -58,15 +61,13 @@ def main() -> None:
         prompt("admin password: ", is_password=True).encode(), bcrypt.gensalt()
     ).decode()
     dat["name"] = prompt("name (e.g. John Doe): ", default=dat["username"])
-    dat["summary"] = markdown(
+    dat["summary"] = prompt(
-        prompt(
+        (
-            (
+            "summary (short description, in markdown, "
-                "summary (short description, in markdown, "
+            "press [CTRL] + [SPACE] to submit):\n"
-                "press [CTRL] + [SPACE] to submit):\n"
+        ),
-            ),
+        key_bindings=_kb,
-            key_bindings=_kb,
+        multiline=True,
-            multiline=True,
-        )
     )
     dat["https"] = True
     proto = "https"
@@ -78,9 +79,10 @@ def main() -> None:
         proto = "http"
 
     print("Note that you can put your icon/avatar in the static/ directory")
-    dat["icon_url"] = prompt(
+    if icon_url := prompt(
         "icon URL: ", default=f'{proto}://{dat["domain"]}/static/nopic.png'
-    )
+    ):
+        dat["icon_url"] = icon_url
     dat["secret"] = os.urandom(16).hex()
 
     with config_file.open("w") as f:

tasks.py

@@ -1,6 +1,6 @@
 import asyncio
 import io
-import subprocess
+import shutil
 import tarfile
 from contextlib import contextmanager
 from pathlib import Path
@@ -46,11 +46,16 @@ def compile_scss(ctx, watch=False):
     # type: (Context, bool) -> None
     from app.utils.favicon import build_favicon
 
-    build_favicon()
     theme_file = Path("data/_theme.scss")
     if not theme_file.exists():
         theme_file.write_text("// override vars for theming here")
 
+    favicon_file = Path("data/favicon.ico")
+    if not favicon_file.exists():
+        build_favicon()
+    else:
+        shutil.copy2(favicon_file, "app/static/favicon.ico")
+
     if watch:
         run("boussole watch", echo=True)
     else:
@@ -164,14 +169,12 @@ def stats(ctx):
 
 @contextmanager
 def embed_version() -> Generator[None, None, None]:
+    from app.utils.version import get_version_commit
+
     version_file = Path("app/_version.py")
     version_file.unlink(missing_ok=True)
-    version = (
+    version_commit = get_version_commit()
-        subprocess.check_output(["git", "rev-parse", "--short=8", "v2"])
+    version_file.write_text(f'VERSION_COMMIT = "{version_commit}"')
-        .split()[0]
-        .decode()
-    )
-    version_file.write_text(f'VERSION_COMMIT = "{version}"')
     try:
         yield
     finally:
@@ -193,6 +196,109 @@ def prune_old_data(ctx):
     asyncio.run(run_prune_old_data())
 
 
+@task
+def webfinger(ctx, account):
+    # type: (Context, str) -> None
+    import traceback
+
+    from loguru import logger
+
+    from app.source import _MENTION_REGEX
+    from app.webfinger import get_actor_url
+
+    logger.disable("app")
+    if not account.startswith("@"):
+        account = f"@{account}"
+    if not _MENTION_REGEX.match(account):
+        print(f"Invalid acccount {account}")
+        return
+
+    print(f"Resolving {account}")
+    try:
+        maybe_actor_url = asyncio.run(get_actor_url(account))
+        if maybe_actor_url:
+            print(f"SUCCESS: {maybe_actor_url}")
+        else:
+            print(f"ERROR: Failed to resolve {account}")
+    except Exception as exc:
+        print(f"ERROR: Failed to resolve {account}")
+        print("".join(traceback.format_exception(exc)))
+
+
+@task
+def move_to(ctx, moved_to):
+    # type: (Context, str) -> None
+    import traceback
+
+    from loguru import logger
+
+    from app.actor import LOCAL_ACTOR
+    from app.actor import fetch_actor
+    from app.boxes import send_move
+    from app.database import async_session
+    from app.source import _MENTION_REGEX
+    from app.webfinger import get_actor_url
+
+    logger.disable("app")
+
+    if not moved_to.startswith("@"):
+        moved_to = f"@{moved_to}"
+    if not _MENTION_REGEX.match(moved_to):
+        print(f"Invalid acccount {moved_to}")
+        return
+
+    async def _send_move():
+        print(f"Initiating move to {moved_to}")
+        async with async_session() as db_session:
+            try:
+                moved_to_actor_id = await get_actor_url(moved_to)
+            except Exception as exc:
+                print(f"ERROR: Failed to resolve {moved_to}")
+                print("".join(traceback.format_exception(exc)))
+                return
+
+            if not moved_to_actor_id:
+                print("ERROR: Failed to resolve {moved_to}")
+                return
+
+            new_actor = await fetch_actor(db_session, moved_to_actor_id)
+
+            if LOCAL_ACTOR.ap_id not in new_actor.ap_actor.get("alsoKnownAs", []):
+                print(
+                    f"{new_actor.handle}/{moved_to_actor_id} is missing "
+                    f"{LOCAL_ACTOR.ap_id} in alsoKnownAs"
+                )
+                return
+
+            await send_move(db_session, new_actor.ap_id)
+
+        print("Done")
+
+    asyncio.run(_send_move())
+
+
+@task
+def self_destruct(ctx):
+    # type: (Context) -> None
+    from loguru import logger
+
+    from app.boxes import send_self_destruct
+    from app.database import async_session
+
+    logger.disable("app")
+
+    async def _send_self_destruct():
+        if input("Initiating self destruct, type yes to confirm: ") != "yes":
+            print("Aborting")
+
+        async with async_session() as db_session:
+            await send_self_destruct(db_session)
+
+        print("Done")
+
+    asyncio.run(_send_self_destruct())
+
+
 @task
 def yunohost_config(
     ctx,
@@ -212,3 +318,38 @@ def yunohost_config(
         summary=summary,
         password=password,
     )
+
+
+@task
+def reset_password(ctx):
+    # type: (Context) -> None
+    import bcrypt
+    from prompt_toolkit import prompt
+
+    new_password = bcrypt.hashpw(
+        prompt("New admin password: ", is_password=True).encode(), bcrypt.gensalt()
+    ).decode()
+
+    print()
+    print("Update data/profile.toml with:")
+    print(f'admin_password = "{new_password}"')
+
+
+@task
+def check_config(ctx):
+    # type: (Context) -> None
+    import sys
+    import traceback
+
+    from loguru import logger
+
+    logger.disable("app")
+
+    try:
+        from app import config  # noqa: F401
+    except Exception as exc:
+        print("Config error, please fix data/profile.toml:\n")
+        print("".join(traceback.format_exception(exc)))
+        sys.exit(1)
+    else:
+        print("Config is OK")

tests/factories.py

@@ -68,6 +68,20 @@ def build_accept_activity(
     }
 
 
+def build_block_activity(
+    from_remote_actor: actor.RemoteActor,
+    for_remote_actor: actor.RemoteActor,
+    outbox_public_id: str | None = None,
+) -> ap.RawObject:
+    return {
+        "@context": ap.AS_CTX,
+        "type": "Block",
+        "id": from_remote_actor.ap_id + "/block/" + (outbox_public_id or uuid4().hex),
+        "actor": from_remote_actor.ap_id,
+        "object": for_remote_actor.ap_id,
+    }
+
+
 def build_move_activity(
     from_remote_actor: actor.RemoteActor,
     for_remote_object: actor.RemoteActor,
@@ -84,12 +98,13 @@ def build_move_activity(
 
 
 def build_note_object(
-    from_remote_actor: actor.RemoteActor,
+    from_remote_actor: actor.RemoteActor | models.Actor,
     outbox_public_id: str | None = None,
     content: str = "Hello",
     to: list[str] = None,
     cc: list[str] = None,
     tags: list[ap.RawObject] = None,
+    in_reply_to: str | None = None,
 ) -> ap.RawObject:
     published = now().replace(microsecond=0).isoformat().replace("+00:00", "Z")
     context = from_remote_actor.ap_id + "/ctx/" + uuid4().hex
@@ -108,8 +123,8 @@ def build_note_object(
         "url": from_remote_actor.ap_id + "/note/" + note_id,
         "tag": tags or [],
         "summary": None,
-        "inReplyTo": None,
         "sensitive": False,
+        "inReplyTo": in_reply_to,
     }
 
 

tests/test_actor.py

@@ -52,4 +52,4 @@ def test_sqlalchemy_factory(db: Session) -> None:
         ap_actor=ra.ap_actor,
         ap_id=ra.ap_id,
     )
-    assert actor_in_db.id == db.query(models.Actor).one().id
+    assert actor_in_db.id == db.execute(select(models.Actor)).scalar_one().id

tests/test_inbox.py

@@ -75,7 +75,7 @@ def test_inbox_incoming_follow_request(
     assert inbox_object.ap_object == follow_activity.ap_object
 
     # And a follower was internally created
-    follower = db.query(models.Follower).one()
+    follower = db.execute(select(models.Follower)).scalar_one()
     assert follower.ap_actor_id == ra.ap_id
     assert follower.actor_id == saved_actor.id
     assert follower.inbox_object_id == inbox_object.id
@@ -414,3 +414,62 @@ def test_inbox__move_activity(
         )
         == 1
     )
+
+    # And a notification was created
+    notif = db.execute(
+        select(models.Notification).where(
+            models.Notification.notification_type == models.NotificationType.MOVE
+        )
+    ).scalar_one()
+    assert notif.actor.ap_id == new_ra.ap_id
+    assert notif.inbox_object_id == inbox_activity.id
+
+
+def test_inbox__block_activity(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # Given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    # Which is followed by the local actor
+    setup_remote_actor_as_following(ra)
+
+    # When receiving a Block activity
+    follow_activity = RemoteObject(
+        factories.build_block_activity(
+            from_remote_actor=ra,
+            for_remote_actor=LOCAL_ACTOR,
+        ),
+        ra,
+    )
+    with mock_httpsig_checker(ra):
+        response = client.post(
+            "/inbox",
+            headers={"Content-Type": ap.AS_CTX},
+            json=follow_activity.ap_object,
+        )
+
+    # Then the server returns a 202
+    assert response.status_code == 202
+
+    run_process_next_incoming_activity()
+
+    # And the actor was saved in DB
+    saved_actor = db.execute(select(models.Actor)).scalar_one()
+    assert saved_actor.ap_id == ra.ap_id
+
+    # And the Block activity was saved in the inbox
+    inbox_activity = db.execute(
+        select(models.InboxObject).where(models.InboxObject.ap_type == "Block")
+    ).scalar_one()
+
+    # And a notification was created
+    notif = db.execute(
+        select(models.Notification).where(
+            models.Notification.notification_type == models.NotificationType.BLOCKED
+        )
+    ).scalar_one()
+    assert notif.actor.ap_id == ra.ap_id
+    assert notif.inbox_object_id == inbox_activity.id

tests/test_outbox.py

@@ -2,13 +2,17 @@ from unittest import mock
 
 import respx
 from fastapi.testclient import TestClient
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from app import activitypub as ap
 from app import models
 from app import webfinger
+from app.actor import LOCAL_ACTOR
 from app.config import generate_csrf_token
 from tests.utils import generate_admin_session_cookies
+from tests.utils import setup_inbox_note
+from tests.utils import setup_outbox_note
 from tests.utils import setup_remote_actor
 from tests.utils import setup_remote_actor_as_follower
 
@@ -49,16 +53,184 @@ def test_send_follow_request(
     assert response.headers.get("Location") == "http://testserver/"
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Follow"
     assert outbox_object.activity_object_ap_id == ra.ap_id
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == ra.inbox_url
 
 
+def test_send_delete__reverts_side_effects(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    # who is a follower
+    follower = setup_remote_actor_as_follower(ra)
+    actor = follower.actor
+
+    # with a note that has existing replies
+    inbox_note = setup_inbox_note(actor)
+    # with a bogus counter
+    inbox_note.replies_count = 5
+    db.commit()
+
+    # and 2 local replies
+    setup_outbox_note(
+        to=[ap.AS_PUBLIC],
+        cc=[LOCAL_ACTOR.followers_collection_id],  # type: ignore
+        in_reply_to=inbox_note.ap_id,
+    )
+    outbox_note2 = setup_outbox_note(
+        to=[ap.AS_PUBLIC],
+        cc=[LOCAL_ACTOR.followers_collection_id],  # type: ignore
+        in_reply_to=inbox_note.ap_id,
+    )
+    db.commit()
+
+    # When deleting one of the replies
+    response = client.post(
+        "/admin/actions/delete",
+        data={
+            "redirect_url": "http://testserver/",
+            "ap_object_id": outbox_note2.ap_id,
+            "csrf_token": generate_csrf_token(),
+        },
+        cookies=generate_admin_session_cookies(),
+    )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+    assert response.headers.get("Location") == "http://testserver/"
+
+    # And the Delete activity was created in the outbox
+    outbox_object = db.execute(
+        select(models.OutboxObject).where(models.OutboxObject.ap_type == "Delete")
+    ).scalar_one()
+    assert outbox_object.ap_type == "Delete"
+    assert outbox_object.activity_object_ap_id == outbox_note2.ap_id
+
+    # And an outgoing activity was queued
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
+    assert outgoing_activity.outbox_object_id == outbox_object.id
+    assert outgoing_activity.recipient == ra.inbox_url
+
+    # And the replies count of the replied object was refreshed correctly
+    db.refresh(inbox_note)
+    assert inbox_note.replies_count == 1
+
+
+def test_send_create_activity__no_content(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    with mock.patch.object(webfinger, "get_actor_url", return_value=ra.ap_id):
+        response = client.post(
+            "/admin/actions/new",
+            data={
+                "redirect_url": "http://testserver/",
+                "visibility": ap.VisibilityEnum.PUBLIC.name,
+                "csrf_token": generate_csrf_token(),
+            },
+            cookies=generate_admin_session_cookies(),
+        )
+
+    # Then the server returns a 422
+    assert response.status_code == 422
+
+
+def test_send_create_activity__with_attachment(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    with mock.patch.object(webfinger, "get_actor_url", return_value=ra.ap_id):
+        response = client.post(
+            "/admin/actions/new",
+            data={
+                "content": "hello",
+                "redirect_url": "http://testserver/",
+                "visibility": ap.VisibilityEnum.PUBLIC.name,
+                "csrf_token": generate_csrf_token(),
+            },
+            files=[
+                ("files", ("attachment.txt", "hello")),
+            ],
+            cookies=generate_admin_session_cookies(),
+        )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+
+    # And the Follow activity was created in the outbox
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
+    assert outbox_object.ap_type == "Note"
+    assert outbox_object.summary is None
+    assert outbox_object.content == "<p>hello</p>\n"
+    assert len(outbox_object.attachments) == 1
+    attachment = outbox_object.attachments[0]
+    assert attachment.type == "Document"
+
+    attachment_response = client.get(attachment.url)
+    assert attachment_response.status_code == 200
+    assert attachment_response.content == b"hello"
+
+    upload = db.execute(select(models.Upload)).scalar_one()
+    assert upload.content_hash == (
+        "324dcf027dd4a30a932c441f365a25e86b173defa4b8e58948253471b81b72cf"
+    )
+
+    outbox_attachment = db.execute(select(models.OutboxObjectAttachment)).scalar_one()
+    assert outbox_attachment.upload_id == upload.id
+    assert outbox_attachment.outbox_object_id == outbox_object.id
+    assert outbox_attachment.filename == "attachment.txt"
+
+
+def test_send_create_activity__no_content_with_cw_and_attachments(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    with mock.patch.object(webfinger, "get_actor_url", return_value=ra.ap_id):
+        response = client.post(
+            "/admin/actions/new",
+            data={
+                "content_warning": "cw",
+                "redirect_url": "http://testserver/",
+                "visibility": ap.VisibilityEnum.PUBLIC.name,
+                "csrf_token": generate_csrf_token(),
+            },
+            files={"files": ("attachment.txt", "hello")},
+            cookies=generate_admin_session_cookies(),
+        )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+
+    # And the Follow activity was created in the outbox
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
+    assert outbox_object.ap_type == "Note"
+    assert outbox_object.summary is None
+    assert outbox_object.content == "<p>cw</p>\n"
+    assert len(outbox_object.attachments) == 1
+
+
 def test_send_create_activity__no_followers_and_with_mention(
     db: Session,
     client: TestClient,
@@ -83,11 +255,11 @@ def test_send_create_activity__no_followers_and_with_mention(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Note"
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == ra.inbox_url
 
@@ -119,11 +291,11 @@ def test_send_create_activity__with_followers(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Note"
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url
 
@@ -159,7 +331,7 @@ def test_send_create_activity__question__one_of(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Question"
     assert outbox_object.is_one_of_poll is True
     assert len(outbox_object.poll_items) == 2
@@ -167,7 +339,7 @@ def test_send_create_activity__question__one_of(
     assert outbox_object.is_poll_ended is False
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url
 
@@ -205,7 +377,7 @@ def test_send_create_activity__question__any_of(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Question"
     assert outbox_object.is_one_of_poll is False
     assert len(outbox_object.poll_items) == 4
@@ -213,7 +385,7 @@ def test_send_create_activity__question__any_of(
     assert outbox_object.is_poll_ended is False
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url
 
@@ -246,11 +418,11 @@ def test_send_create_activity__article(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Article"
     assert outbox_object.ap_object["name"] == "Article"
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url

tests/test_public.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 import pytest
 from fastapi.testclient import TestClient
 from sqlalchemy.orm import Session
@@ -31,7 +33,19 @@ def test_followers__ap(client, db) -> None:
     response = client.get("/followers", headers={"Accept": ap.AP_CONTENT_TYPE})
     assert response.status_code == 200
     assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
-    assert response.json()["id"].endswith("/followers")
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/followers")
+    assert "first" in json_resp
+
+
+def test_followers__ap_hides_followers(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWERS", True):
+        response = client.get("/followers", headers={"Accept": ap.AP_CONTENT_TYPE})
+    assert response.status_code == 200
+    assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/followers")
+    assert "first" not in json_resp
 
 
 def test_followers__html(client, db) -> None:
@@ -40,14 +54,40 @@ def test_followers__html(client, db) -> None:
     assert response.headers["content-type"].startswith("text/html")
 
 
+def test_followers__html_hides_followers(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWERS", True):
+        response = client.get("/followers", headers={"Accept": "text/html"})
+    assert response.status_code == 404
+    assert response.headers["content-type"].startswith("text/html")
+
+
 def test_following__ap(client, db) -> None:
     response = client.get("/following", headers={"Accept": ap.AP_CONTENT_TYPE})
     assert response.status_code == 200
     assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
-    assert response.json()["id"].endswith("/following")
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/following")
+    assert "first" in json_resp
+
+
+def test_following__ap_hides_following(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWING", True):
+        response = client.get("/following", headers={"Accept": ap.AP_CONTENT_TYPE})
+    assert response.status_code == 200
+    assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/following")
+    assert "first" not in json_resp
 
 
 def test_following__html(client, db) -> None:
     response = client.get("/following")
     assert response.status_code == 200
     assert response.headers["content-type"].startswith("text/html")
+
+
+def test_following__html_hides_following(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWING", True):
+        response = client.get("/following", headers={"Accept": "text/html"})
+    assert response.status_code == 404
+    assert response.headers["content-type"].startswith("text/html")

tests/test_tags.py

@@ -1,4 +1,5 @@
 from fastapi.testclient import TestClient
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from app import activitypub as ap
@@ -35,7 +36,7 @@ def test_tags__note_with_tag(db: Session, client: TestClient) -> None:
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Note"
     assert len(outbox_object.tags) == 1
     emoji_tag = outbox_object.tags[0]

tests/utils.py

@@ -169,6 +169,53 @@ def setup_remote_actor_as_following_and_follower(
     return following, follower
 
 
+def setup_outbox_note(
+    content: str = "Hello",
+    to: list[str] = None,
+    cc: list[str] = None,
+    tags: list[ap.RawObject] = None,
+    in_reply_to: str | None = None,
+) -> models.OutboxObject:
+    note_id = uuid4().hex
+    note_from_outbox = RemoteObject(
+        factories.build_note_object(
+            from_remote_actor=LOCAL_ACTOR,
+            outbox_public_id=note_id,
+            content=content,
+            to=to,
+            cc=cc,
+            tags=tags,
+            in_reply_to=in_reply_to,
+        ),
+        LOCAL_ACTOR,
+    )
+    return factories.OutboxObjectFactory.from_remote_object(note_id, note_from_outbox)
+
+
+def setup_inbox_note(
+    actor: models.Actor,
+    content: str = "Hello",
+    to: list[str] = None,
+    cc: list[str] = None,
+    tags: list[ap.RawObject] = None,
+    in_reply_to: str | None = None,
+) -> models.OutboxObject:
+    note_id = uuid4().hex
+    note_from_outbox = RemoteObject(
+        factories.build_note_object(
+            from_remote_actor=actor,
+            outbox_public_id=note_id,
+            content=content,
+            to=to,
+            cc=cc,
+            tags=tags,
+            in_reply_to=in_reply_to,
+        ),
+        actor,
+    )
+    return factories.InboxObjectFactory.from_remote_object(note_from_outbox, actor)
+
+
 def setup_inbox_delete(
     actor: models.Actor, deleted_object_ap_id: str
 ) -> models.InboxObject: