Quick hotfix for retries

Also expand documentation about custom templates

AUTHORS

@@ -3,7 +3,9 @@ Kevin Wallace <doof@doof.net>
 Miguel Jacq <mig@mig5.net>
 Alexey Shpakovsky <alexey@shpakovsky.ru>
 Josh Washburne <josh@jodh.us>
+João Costa <jdpc557@gmail.com>
 Sam <samr1.dev@pm.me>
 Ash McAllan <acegiak@gmail.com>
 Cassio Zen <cassio@hey.com>
 Cocoa <momijizukamori@gmail.com>
+Jane <jane@janeirl.dev>

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim as python-base
+FROM python:3.11-slim as python-base
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     POETRY_HOME="/opt/poetry" \

Makefile

@@ -28,7 +28,7 @@ move-to:
 
 .PHONY: self-destruct
 self-destruct:
-	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv self-destruct
+	-docker run --rm --it --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv self-destruct
 
 .PHONY: reset-password
 reset-password:
@@ -41,3 +41,7 @@ check-config:
 .PHONY: compile-scss
 compile-scss:
 	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv compile-scss
+
+.PHONY: import-mastodon-following-accounts 
+import-mastodon-following-accounts:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv import-mastodon-following-accounts $(path)

README.md

@@ -10,6 +10,7 @@ Instances in the wild:
  - [microblog.pub](https://microblog.pub/) (follow to get updated about the project)
  - [hexa.ninja](https://hexa.ninja) (theme customization example)
  - [testing.microblog.pub](https://testing.microblog.pub/)
+ - [Irish Left Archive](https://posts.leftarchive.ie/) (another theme customization example)
 
 There are still some rough edges, but the server is mostly functional.
 

alembic/versions/2022_12_12_1926-9b404c47970a_add_option_to_hide_announces_from_actor.py

@@ -0,0 +1,32 @@
+"""Add option to hide announces from actor
+
+Revision ID: 9b404c47970a
+Revises: fadfd359ce78
+Create Date: 2022-12-12 19:26:36.912763+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '9b404c47970a'
+down_revision = 'fadfd359ce78'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('actor', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('are_announces_hidden_from_stream', sa.Boolean(), server_default='0', nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('actor', schema=None) as batch_op:
+        batch_op.drop_column('are_announces_hidden_from_stream')
+
+    # ### end Alembic commands ###

alembic/versions/2022_12_16_1730-4ab54becec04_add_oauth_client.py

@@ -0,0 +1,48 @@
+"""Add OAuth client
+
+Revision ID: 4ab54becec04
+Revises: 9b404c47970a
+Create Date: 2022-12-16 17:30:54.520477+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '4ab54becec04'
+down_revision = '9b404c47970a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('oauth_client',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('client_name', sa.String(), nullable=False),
+    sa.Column('redirect_uris', sa.JSON(), nullable=True),
+    sa.Column('client_uri', sa.String(), nullable=True),
+    sa.Column('logo_uri', sa.String(), nullable=True),
+    sa.Column('scope', sa.String(), nullable=True),
+    sa.Column('client_id', sa.String(), nullable=False),
+    sa.Column('client_secret', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('client_secret')
+    )
+    with op.batch_alter_table('oauth_client', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_oauth_client_client_id'), ['client_id'], unique=True)
+        batch_op.create_index(batch_op.f('ix_oauth_client_id'), ['id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('oauth_client', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_oauth_client_id'))
+        batch_op.drop_index(batch_op.f('ix_oauth_client_client_id'))
+
+    op.drop_table('oauth_client')
+    # ### end Alembic commands ###

alembic/versions/2022_12_18_1126-a209f0333f5a_add_oauth_refresh_token_support.py

@@ -0,0 +1,36 @@
+"""Add OAuth refresh token support
+
+Revision ID: a209f0333f5a
+Revises: 4ab54becec04
+Create Date: 2022-12-18 11:26:31.976348+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'a209f0333f5a'
+down_revision = '4ab54becec04'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('indieauth_access_token', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('refresh_token', sa.String(), nullable=True))
+        batch_op.add_column(sa.Column('was_refreshed', sa.Boolean(), server_default='0', nullable=False))
+        batch_op.create_index(batch_op.f('ix_indieauth_access_token_refresh_token'), ['refresh_token'], unique=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('indieauth_access_token', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_indieauth_access_token_refresh_token'))
+        batch_op.drop_column('was_refreshed')
+        batch_op.drop_column('refresh_token')
+
+    # ### end Alembic commands ###

app/actor.py

@@ -6,6 +6,7 @@ from functools import cached_property
 from typing import Union
 from urllib.parse import urlparse
 
+import httpx
 from loguru import logger
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
@@ -13,6 +14,9 @@ from sqlalchemy.orm import joinedload
 from app import activitypub as ap
 from app import media
 from app.config import BASE_URL
+from app.config import USER_AGENT
+from app.config import USERNAME
+from app.config import WEBFINGER_DOMAIN
 from app.database import AsyncSession
 from app.utils.datetime import as_utc
 from app.utils.datetime import now
@@ -27,7 +31,38 @@ def _handle(raw_actor: ap.RawObject) -> str:
     if not domain.hostname:
         raise ValueError(f"Invalid actor ID {ap_id}")
 
-    return f'@{raw_actor["preferredUsername"]}@{domain.hostname}'  # type: ignore
+    handle = f'@{raw_actor["preferredUsername"]}@{domain.hostname}'  # type: ignore
+
+    # TODO: cleanup this
+    # Next, check for custom webfinger domains
+    resp: httpx.Response | None = None
+    for url in {
+        f"https://{domain.hostname}/.well-known/webfinger",
+        f"http://{domain.hostname}/.well-known/webfinger",
+    }:
+        try:
+            logger.info(f"Webfinger {handle} at {url}")
+            resp = httpx.get(
+                url,
+                params={"resource": f"acct:{handle[1:]}"},
+                headers={
+                    "User-Agent": USER_AGENT,
+                },
+                follow_redirects=True,
+            )
+            resp.raise_for_status()
+            break
+        except Exception:
+            logger.exception(f"Failed to webfinger {handle}")
+
+    if resp:
+        try:
+            json_resp = resp.json()
+            if json_resp.get("subject", "").startswith("acct:"):
+                return "@" + json_resp["subject"].removeprefix("acct:")
+        except Exception:
+            logger.exception(f"Failed to parse webfinger response for {handle}")
+    return handle
 
 
 class Actor:
@@ -61,7 +96,7 @@ class Actor:
             return self.name
         return self.preferred_username
 
-    @property
+    @cached_property
     def handle(self) -> str:
         return _handle(self.ap_actor)
 
@@ -143,13 +178,18 @@ class Actor:
 
 
 class RemoteActor(Actor):
-    def __init__(self, ap_actor: ap.RawObject) -> None:
+    def __init__(self, ap_actor: ap.RawObject, handle: str | None = None) -> None:
         if (ap_type := ap_actor.get("type")) not in ap.ACTOR_TYPES:
             raise ValueError(f"Unexpected actor type: {ap_type}")
 
         self._ap_actor = ap_actor
         self._ap_type = ap_type
 
+        if handle is None:
+            handle = _handle(ap_actor)
+
+        self._handle = handle
+
     @property
     def ap_actor(self) -> ap.RawObject:
         return self._ap_actor
@@ -162,8 +202,12 @@ class RemoteActor(Actor):
     def is_from_db(self) -> bool:
         return False
 
+    @property
+    def handle(self) -> str:
+        return self._handle
 
-LOCAL_ACTOR = RemoteActor(ap_actor=ap.ME)
+
+LOCAL_ACTOR = RemoteActor(ap_actor=ap.ME, handle=f"@{USERNAME}@{WEBFINGER_DOMAIN}")
 
 
 async def save_actor(db_session: AsyncSession, ap_actor: ap.RawObject) -> "ActorModel":

app/admin.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from urllib.parse import quote
 
 import httpx
 from fastapi import APIRouter
@@ -59,7 +60,9 @@ async def user_session_or_redirect(
 
     _RedirectToLoginPage = HTTPException(
         status_code=302,
-        headers={"Location": request.url_for("login") + f"?redirect={redirect_url}"},
+        headers={
+            "Location": request.url_for("login") + f"?redirect={quote(redirect_url)}"
+        },
     )
 
     if not session:
@@ -186,8 +189,11 @@ async def admin_new(
             content += f"{in_reply_to_object.actor.handle} "
         for tag in in_reply_to_object.tags:
             if tag.get("type") == "Mention" and tag["name"] != LOCAL_ACTOR.handle:
-                mentioned_actor = await fetch_actor(db_session, tag["href"])
-                content += f"{mentioned_actor.handle} "
+                try:
+                    mentioned_actor = await fetch_actor(db_session, tag["href"])
+                    content += f"{mentioned_actor.handle} "
+                except Exception:
+                    logger.exception(f"Failed to lookup {mentioned_actor}")
 
         # Copy the content warning if any
         if in_reply_to_object.summary:
@@ -959,6 +965,34 @@ async def admin_actions_unblock(
     return RedirectResponse(redirect_url, status_code=302)
 
 
+@router.post("/actions/hide_announces")
+async def admin_actions_hide_announces(
+    request: Request,
+    ap_actor_id: str = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    actor = await fetch_actor(db_session, ap_actor_id)
+    actor.are_announces_hidden_from_stream = True
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
+@router.post("/actions/show_announces")
+async def admin_actions_show_announces(
+    request: Request,
+    ap_actor_id: str = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    actor = await fetch_actor(db_session, ap_actor_id)
+    actor.are_announces_hidden_from_stream = False
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
 @router.post("/actions/delete")
 async def admin_actions_delete(
     request: Request,
@@ -1151,7 +1185,7 @@ async def admin_actions_new(
     elif name:
         ap_type = "Article"
 
-    public_id = await boxes.send_create(
+    public_id, _ = await boxes.send_create(
         db_session,
         ap_type=ap_type,
         source=content,

app/ap_object.py

@@ -12,6 +12,7 @@ from app import activitypub as ap
 from app.actor import LOCAL_ACTOR
 from app.actor import Actor
 from app.actor import RemoteActor
+from app.config import ID
 from app.media import proxied_media_url
 from app.utils.datetime import now
 from app.utils.datetime import parse_isoformat
@@ -212,6 +213,15 @@ class Object:
     def in_reply_to(self) -> str | None:
         return self.ap_object.get("inReplyTo")
 
+    @property
+    def is_local_reply(self) -> bool:
+        if not self.in_reply_to:
+            return False
+
+        return bool(
+            self.in_reply_to.startswith(ID) and self.content  # Hide votes from Question
+        )
+
     @property
     def is_in_reply_to_from_inbox(self) -> bool | None:
         if not self.in_reply_to:

app/boxes.py

@@ -28,7 +28,6 @@ from app.actor import save_actor
 from app.actor import update_actor_if_needed
 from app.ap_object import RemoteObject
 from app.config import BASE_URL
-from app.config import BLOCKED_SERVERS
 from app.config import ID
 from app.config import MANUALLY_APPROVES_FOLLOWERS
 from app.config import set_moved_to
@@ -46,10 +45,22 @@ from app.utils.datetime import now
 from app.utils.datetime import parse_isoformat
 from app.utils.facepile import WebmentionReply
 from app.utils.text import slugify
+from app.utils.url import is_hostname_blocked
 
 AnyboxObject = models.InboxObject | models.OutboxObject
 
 
+def is_notification_enabled(notification_type: models.NotificationType) -> bool:
+    """Checks if a given notification type is enabled."""
+    if notification_type.value == "pending_incoming_follower":
+        # This one cannot be disabled as it would prevent manually reviewing
+        # follow requests.
+        return True
+    if notification_type.value in config.CONFIG.disabled_notifications:
+        return False
+    return True
+
+
 def allocate_outbox_id() -> str:
     return uuid.uuid4().hex
 
@@ -168,12 +179,13 @@ async def send_block(db_session: AsyncSession, ap_actor_id: str) -> None:
     await new_outgoing_activity(db_session, actor.inbox_url, outbox_object.id)
 
     # 4. Create a notification
-    notif = models.Notification(
-        notification_type=models.NotificationType.BLOCK,
-        actor_id=actor.id,
-        outbox_object_id=outbox_object.id,
-    )
-    db_session.add(notif)
+    if is_notification_enabled(models.NotificationType.BLOCK):
+        notif = models.Notification(
+            notification_type=models.NotificationType.BLOCK,
+            actor_id=actor.id,
+            outbox_object_id=outbox_object.id,
+        )
+        db_session.add(notif)
 
     await db_session.commit()
 
@@ -427,7 +439,9 @@ async def _send_undo(db_session: AsyncSession, ap_object_id: str) -> None:
         announced_object.announced_via_outbox_object_ap_id = None
 
         # Send the Undo to the original recipients
-        recipients = await _compute_recipients(db_session, announced_object.ap_object)
+        recipients = await _compute_recipients(
+            db_session, outbox_object_to_undo.ap_object
+        )
         for rcp in recipients:
             await new_outgoing_activity(db_session, rcp, outbox_object.id)
     elif outbox_object_to_undo.ap_type == "Block":
@@ -447,12 +461,13 @@ async def _send_undo(db_session: AsyncSession, ap_object_id: str) -> None:
             outbox_object.id,
         )
 
-        notif = models.Notification(
-            notification_type=models.NotificationType.UNBLOCK,
-            actor_id=blocked_actor.id,
-            outbox_object_id=outbox_object.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.UNBLOCK):
+            notif = models.Notification(
+                notification_type=models.NotificationType.UNBLOCK,
+                actor_id=blocked_actor.id,
+                outbox_object_id=outbox_object.id,
+            )
+            db_session.add(notif)
 
     else:
         raise ValueError("Should never happen")
@@ -577,7 +592,7 @@ async def send_create(
     poll_answers: list[str] | None = None,
     poll_duration_in_minutes: int | None = None,
     name: str | None = None,
-) -> str:
+) -> tuple[str, models.OutboxObject]:
     note_id = allocate_outbox_id()
     published = now().replace(microsecond=0).isoformat().replace("+00:00", "Z")
     context = f"{ID}/contexts/" + uuid.uuid4().hex
@@ -752,7 +767,7 @@ async def send_create(
 
     await db_session.commit()
 
-    return note_id
+    return note_id, outbox_object
 
 
 async def send_vote(
@@ -935,7 +950,7 @@ async def compute_all_known_recipients(db_session: AsyncSession) -> set[str]:
     }
 
 
-async def _get_following(db_session: AsyncSession) -> list[models.Follower]:
+async def _get_following(db_session: AsyncSession) -> list[models.Following]:
     return (
         (
             await db_session.scalars(
@@ -1525,11 +1540,12 @@ async def _send_accept(
         raise ValueError("Should never happen")
     await new_outgoing_activity(db_session, from_actor.inbox_url, outbox_activity.id)
 
-    notif = models.Notification(
-        notification_type=models.NotificationType.NEW_FOLLOWER,
-        actor_id=from_actor.id,
-    )
-    db_session.add(notif)
+    if is_notification_enabled(models.NotificationType.NEW_FOLLOWER):
+        notif = models.Notification(
+            notification_type=models.NotificationType.NEW_FOLLOWER,
+            actor_id=from_actor.id,
+        )
+        db_session.add(notif)
 
 
 async def send_reject(
@@ -1568,11 +1584,12 @@ async def _send_reject(
         raise ValueError("Should never happen")
     await new_outgoing_activity(db_session, from_actor.inbox_url, outbox_activity.id)
 
-    notif = models.Notification(
-        notification_type=models.NotificationType.REJECTED_FOLLOWER,
-        actor_id=from_actor.id,
-    )
-    db_session.add(notif)
+    if is_notification_enabled(models.NotificationType.REJECTED_FOLLOWER):
+        notif = models.Notification(
+            notification_type=models.NotificationType.REJECTED_FOLLOWER,
+            actor_id=from_actor.id,
+        )
+        db_session.add(notif)
 
 
 async def _handle_undo_activity(
@@ -1598,11 +1615,12 @@ async def _handle_undo_activity(
                 models.Follower.inbox_object_id == ap_activity_to_undo.id
             )
         )
-        notif = models.Notification(
-            notification_type=models.NotificationType.UNFOLLOW,
-            actor_id=from_actor.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.UNFOLLOW):
+            notif = models.Notification(
+                notification_type=models.NotificationType.UNFOLLOW,
+                actor_id=from_actor.id,
+            )
+            db_session.add(notif)
 
     elif ap_activity_to_undo.ap_type == "Like":
         if not ap_activity_to_undo.activity_object_ap_id:
@@ -1618,14 +1636,21 @@ async def _handle_undo_activity(
             )
             return
 
-        liked_obj.likes_count = models.OutboxObject.likes_count - 1
-        notif = models.Notification(
-            notification_type=models.NotificationType.UNDO_LIKE,
-            actor_id=from_actor.id,
-            outbox_object_id=liked_obj.id,
-            inbox_object_id=ap_activity_to_undo.id,
+        liked_obj.likes_count = (
+            await _get_outbox_likes_count(
+                db_session,
+                liked_obj,
+            )
+            - 1
         )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.UNDO_LIKE):
+            notif = models.Notification(
+                notification_type=models.NotificationType.UNDO_LIKE,
+                actor_id=from_actor.id,
+                outbox_object_id=liked_obj.id,
+                inbox_object_id=ap_activity_to_undo.id,
+            )
+            db_session.add(notif)
 
     elif ap_activity_to_undo.ap_type == "Announce":
         if not ap_activity_to_undo.activity_object_ap_id:
@@ -1643,20 +1668,22 @@ async def _handle_undo_activity(
                 announced_obj_from_outbox.announces_count = (
                     models.OutboxObject.announces_count - 1
                 )
-                notif = models.Notification(
-                    notification_type=models.NotificationType.UNDO_ANNOUNCE,
-                    actor_id=from_actor.id,
-                    outbox_object_id=announced_obj_from_outbox.id,
-                    inbox_object_id=ap_activity_to_undo.id,
-                )
-                db_session.add(notif)
+                if is_notification_enabled(models.NotificationType.UNDO_ANNOUNCE):
+                    notif = models.Notification(
+                        notification_type=models.NotificationType.UNDO_ANNOUNCE,
+                        actor_id=from_actor.id,
+                        outbox_object_id=announced_obj_from_outbox.id,
+                        inbox_object_id=ap_activity_to_undo.id,
+                    )
+                    db_session.add(notif)
     elif ap_activity_to_undo.ap_type == "Block":
-        notif = models.Notification(
-            notification_type=models.NotificationType.UNBLOCKED,
-            actor_id=from_actor.id,
-            inbox_object_id=ap_activity_to_undo.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.UNBLOCKED):
+            notif = models.Notification(
+                notification_type=models.NotificationType.UNBLOCKED,
+                actor_id=from_actor.id,
+                inbox_object_id=ap_activity_to_undo.id,
+            )
+            db_session.add(notif)
     else:
         logger.warning(f"Don't know how to undo {ap_activity_to_undo.ap_type} activity")
 
@@ -1720,12 +1747,13 @@ async def _handle_move_activity(
     else:
         logger.info(f"Already following target {new_actor_id}")
 
-    notif = models.Notification(
-        notification_type=models.NotificationType.MOVE,
-        actor_id=new_actor.id,
-        inbox_object_id=move_activity.id,
-    )
-    db_session.add(notif)
+    if is_notification_enabled(models.NotificationType.MOVE):
+        notif = models.Notification(
+            notification_type=models.NotificationType.MOVE,
+            actor_id=new_actor.id,
+            inbox_object_id=move_activity.id,
+        )
+        db_session.add(notif)
 
 
 async def _handle_update_activity(
@@ -1883,11 +1911,7 @@ async def _process_note_object(
 
     is_from_following = ro.actor.ap_id in {f.ap_actor_id for f in following}
     is_reply = bool(ro.in_reply_to)
-    is_local_reply = bool(
-        ro.in_reply_to
-        and ro.in_reply_to.startswith(BASE_URL)
-        and ro.content  # Hide votes from Question
-    )
+    is_local_reply = ro.is_local_reply
     is_mention = False
     hashtags = []
     tags = ro.ap_object.get("tag", [])
@@ -1999,7 +2023,7 @@ async def _process_note_object(
                     inbox_object_id=parent_activity.id,
                 )
 
-    if is_mention:
+    if is_mention and is_notification_enabled(models.NotificationType.MENTION):
         notif = models.Notification(
             notification_type=models.NotificationType.MENTION,
             actor_id=from_actor.id,
@@ -2098,13 +2122,14 @@ async def _handle_announce_activity(
             models.OutboxObject.announces_count + 1
         )
 
-        notif = models.Notification(
-            notification_type=models.NotificationType.ANNOUNCE,
-            actor_id=actor.id,
-            outbox_object_id=relates_to_outbox_object.id,
-            inbox_object_id=announce_activity.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.ANNOUNCE):
+            notif = models.Notification(
+                notification_type=models.NotificationType.ANNOUNCE,
+                actor_id=actor.id,
+                outbox_object_id=relates_to_outbox_object.id,
+                inbox_object_id=announce_activity.id,
+            )
+            db_session.add(notif)
     else:
         # Only show the announce in the stream if it comes from an actor
         # in the following collection
@@ -2180,7 +2205,10 @@ async def _handle_announce_activity(
                 db_session.add(announced_inbox_object)
                 await db_session.flush()
                 announce_activity.relates_to_inbox_object_id = announced_inbox_object.id
-                announce_activity.is_hidden_from_stream = not is_from_following
+                announce_activity.is_hidden_from_stream = (
+                    not is_from_following
+                    or announce_activity.actor.are_announces_hidden_from_stream
+                )
 
 
 async def _handle_like_activity(
@@ -2202,13 +2230,14 @@ async def _handle_like_activity(
             relates_to_outbox_object,
         )
 
-        notif = models.Notification(
-            notification_type=models.NotificationType.LIKE,
-            actor_id=actor.id,
-            outbox_object_id=relates_to_outbox_object.id,
-            inbox_object_id=like_activity.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.LIKE):
+            notif = models.Notification(
+                notification_type=models.NotificationType.LIKE,
+                actor_id=actor.id,
+                outbox_object_id=relates_to_outbox_object.id,
+                inbox_object_id=like_activity.id,
+            )
+            db_session.add(notif)
 
 
 async def _handle_block_activity(
@@ -2225,12 +2254,13 @@ async def _handle_block_activity(
         return
 
     # Create a notification
-    notif = models.Notification(
-        notification_type=models.NotificationType.BLOCKED,
-        actor_id=actor.id,
-        inbox_object_id=block_activity.id,
-    )
-    db_session.add(notif)
+    if is_notification_enabled(models.NotificationType.BLOCKED):
+        notif = models.Notification(
+            notification_type=models.NotificationType.BLOCKED,
+            actor_id=actor.id,
+            inbox_object_id=block_activity.id,
+        )
+        db_session.add(notif)
 
 
 async def _process_transient_object(
@@ -2285,7 +2315,7 @@ async def save_to_inbox(
         logger.exception("Failed to fetch actor")
         return
 
-    if actor.server in BLOCKED_SERVERS:
+    if is_hostname_blocked(actor.server):
         logger.warning(f"Server {actor.server} is blocked")
         return
 
@@ -2433,12 +2463,13 @@ async def save_to_inbox(
                     if activity_ro.ap_type == "Accept"
                     else models.NotificationType.FOLLOW_REQUEST_REJECTED
                 )
-                notif = models.Notification(
-                    notification_type=notif_type,
-                    actor_id=actor.id,
-                    inbox_object_id=inbox_object.id,
-                )
-                db_session.add(notif)
+                if is_notification_enabled(notif_type):
+                    notif = models.Notification(
+                        notification_type=notif_type,
+                        actor_id=actor.id,
+                        inbox_object_id=inbox_object.id,
+                    )
+                    db_session.add(notif)
 
                 if activity_ro.ap_type == "Accept":
                     following = models.Following(

app/config.py

@@ -44,11 +44,14 @@ except FileNotFoundError:
 JS_HASH = "none"
 try:
     # To keep things simple, we keep a single hash for the 2 files
-    js_data_common = (ROOT_DIR / "app" / "static" / "common-admin.js").read_bytes()
-    js_data_new = (ROOT_DIR / "app" / "static" / "new.js").read_bytes()
-    JS_HASH = hashlib.md5(
-        js_data_common + js_data_new, usedforsecurity=False
-    ).hexdigest()
+    dat = b""
+    for j in [
+        ROOT_DIR / "app" / "static" / "common.js",
+        ROOT_DIR / "app" / "static" / "common-admin.js",
+        ROOT_DIR / "app" / "static" / "new.js",
+    ]:
+        dat += j.read_bytes()
+    JS_HASH = hashlib.md5(dat, usedforsecurity=False).hexdigest()
 except FileNotFoundError:
     pass
 
@@ -114,11 +117,16 @@ class Config(pydantic.BaseModel):
 
     custom_content_security_policy: str | None = None
 
+    webfinger_domain: str | None = None
+
     # Config items to make tests easier
     sqlalchemy_database: str | None = None
     key_path: str | None = None
 
     session_timeout: int = 3600 * 24 * 3  # in seconds, 3 days by default
+    csrf_token_exp: int = 3600
+
+    disabled_notifications: list[str] = []
 
     # Only set when the app is served on a non-root path
     id: str | None = None
@@ -163,6 +171,10 @@ ID = f"{_SCHEME}://{DOMAIN}"
 if CONFIG.id:
     ID = CONFIG.id
 USERNAME = CONFIG.username
+
+# Allow to use @handle@webfinger-domain.tld while hosting the server at domain.tld
+WEBFINGER_DOMAIN = CONFIG.webfinger_domain or DOMAIN
+
 MANUALLY_APPROVES_FOLLOWERS = CONFIG.manually_approves_followers
 HIDES_FOLLOWERS = CONFIG.hides_followers
 HIDES_FOLLOWING = CONFIG.hides_following
@@ -252,7 +264,7 @@ def verify_csrf_token(
     if redirect_url:
         please_try_again = f'<a href="{redirect_url}">please try again</a>'
     try:
-        csrf_serializer.loads(csrf_token, max_age=1800)
+        csrf_serializer.loads(csrf_token, max_age=CONFIG.csrf_token_exp)
     except (itsdangerous.BadData, itsdangerous.SignatureExpired):
         logger.exception("Failed to verify CSRF token")
         raise HTTPException(

app/customization.py

@@ -146,9 +146,10 @@ _StreamVisibilityCallback = Callable[[ObjectInfo], bool]
 
 
 def default_stream_visibility_callback(object_info: ObjectInfo) -> bool:
-    logger.info(f"{object_info=}")
-    return (
+    result = (
         (not object_info.is_reply and object_info.is_from_following)
         or object_info.is_mention
         or object_info.is_local_reply
     )
+    logger.info(f"{object_info=}/{result=}")
+    return result

app/httpsig.py

@@ -23,12 +23,12 @@ from sqlalchemy import select
 
 from app import activitypub as ap
 from app import config
-from app.config import BLOCKED_SERVERS
 from app.config import KEY_PATH
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.key import Key
 from app.utils.datetime import now
+from app.utils.url import is_hostname_blocked
 
 _KEY_CACHE: MutableMapping[str, Key] = LFUCache(256)
 
@@ -184,7 +184,7 @@ async def httpsig_checker(
         )
 
     server = urlparse(key_id).hostname
-    if server in BLOCKED_SERVERS:
+    if is_hostname_blocked(server):
         return HTTPSigInfo(
             has_valid_signature=False,
             server=server,

app/incoming_activities.py

@@ -60,7 +60,7 @@ def _set_next_try(
     if not outgoing_activity.tries:
         raise ValueError("Should never happen")
 
-    if outgoing_activity.tries == _MAX_RETRIES:
+    if outgoing_activity.tries >= _MAX_RETRIES:
         outgoing_activity.is_errored = True
         outgoing_activity.next_try = None
     else:

app/indieauth.py

@@ -10,9 +10,12 @@ from fastapi import Form
 from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
-from fastapi.responses import RedirectResponse
+from fastapi.security import HTTPBasic
+from fastapi.security import HTTPBasicCredentials
 from loguru import logger
+from pydantic import BaseModel
 from sqlalchemy import select
+from sqlalchemy.orm import joinedload
 
 from app import config
 from app import models
@@ -21,9 +24,12 @@ from app.admin import user_session_or_redirect
 from app.config import verify_csrf_token
 from app.database import AsyncSession
 from app.database import get_db_session
+from app.redirect import redirect
 from app.utils import indieauth
 from app.utils.datetime import now
 
+basic_auth = HTTPBasic()
+
 router = APIRouter()
 
 
@@ -38,9 +44,55 @@ async def well_known_authorization_server(
         "code_challenge_methods_supported": ["S256"],
         "revocation_endpoint": request.url_for("indieauth_revocation_endpoint"),
         "revocation_endpoint_auth_methods_supported": ["none"],
+        "registration_endpoint": request.url_for("oauth_registration_endpoint"),
+        "introspection_endpoint": request.url_for("oauth_introspection_endpoint"),
     }
 
 
+class OAuthRegisterClientRequest(BaseModel):
+    client_name: str
+    redirect_uris: list[str] | str
+
+    client_uri: str | None = None
+    logo_uri: str | None = None
+    scope: str | None = None
+
+
+@router.post("/oauth/register")
+async def oauth_registration_endpoint(
+    register_client_request: OAuthRegisterClientRequest,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> JSONResponse:
+    """Implements OAuth 2.0 Dynamic Registration."""
+
+    client = models.OAuthClient(
+        client_name=register_client_request.client_name,
+        redirect_uris=[register_client_request.redirect_uris]
+        if isinstance(register_client_request.redirect_uris, str)
+        else register_client_request.redirect_uris,
+        client_uri=register_client_request.client_uri,
+        logo_uri=register_client_request.logo_uri,
+        scope=register_client_request.scope,
+        client_id=secrets.token_hex(16),
+        client_secret=secrets.token_hex(32),
+    )
+
+    db_session.add(client)
+    await db_session.commit()
+
+    return JSONResponse(
+        content={
+            **register_client_request.dict(),
+            "client_id_issued_at": int(client.created_at.timestamp()),  # type: ignore
+            "grant_types": ["authorization_code", "refresh_token"],
+            "client_secret_expires_at": 0,
+            "client_id": client.client_id,
+            "client_secret": client.client_secret,
+        },
+        status_code=201,
+    )
+
+
 @router.get("/auth")
 async def indieauth_authorization_endpoint(
     request: Request,
@@ -56,12 +108,29 @@ async def indieauth_authorization_endpoint(
     code_challenge = request.query_params.get("code_challenge", "")
     code_challenge_method = request.query_params.get("code_challenge_method", "")
 
+    # Check if the authorization request is coming from an OAuth client
+    registered_client = (
+        await db_session.scalars(
+            select(models.OAuthClient).where(
+                models.OAuthClient.client_id == client_id,
+            )
+        )
+    ).one_or_none()
+    if registered_client:
+        client = {
+            "name": registered_client.client_name,
+            "logo": registered_client.logo_uri,
+            "url": registered_client.client_uri,
+        }
+    else:
+        client = await indieauth.get_client_id_data(client_id)  # type: ignore
+
     return await templates.render_template(
         db_session,
         request,
         "indieauth_flow.html",
         dict(
-            client=await indieauth.get_client_id_data(client_id),
+            client=client,
             scopes=scope,
             redirect_uri=redirect_uri,
             state=state,
@@ -80,7 +149,7 @@ async def indieauth_flow(
     db_session: AsyncSession = Depends(get_db_session),
     csrf_check: None = Depends(verify_csrf_token),
     _: None = Depends(user_session_or_redirect),
-) -> RedirectResponse:
+) -> templates.TemplateResponse:
     form_data = await request.form()
     logger.info(f"{form_data=}")
 
@@ -114,9 +183,8 @@ async def indieauth_flow(
     db_session.add(auth_request)
     await db_session.commit()
 
-    return RedirectResponse(
-        redirect_uri + f"?code={code}&state={state}&iss={iss}",
-        status_code=302,
+    return await redirect(
+        request, db_session, redirect_uri + f"?code={code}&state={state}&iss={iss}"
     )
 
 
@@ -207,29 +275,54 @@ async def indieauth_token_endpoint(
     form_data = await request.form()
     logger.info(f"{form_data=}")
     grant_type = form_data.get("grant_type", "authorization_code")
-    if grant_type != "authorization_code":
+    if grant_type not in ["authorization_code", "refresh_token"]:
         raise ValueError(f"Invalid grant_type {grant_type}")
 
-    code = form_data["code"]
-
     # These must match the params from the first request
     client_id = form_data["client_id"]
-    redirect_uri = form_data["redirect_uri"]
-    # code_verifier is optional for backward compat
     code_verifier = form_data.get("code_verifier")
 
-    is_code_valid, auth_code_request = await _check_auth_code(
-        db_session,
-        code=code,
-        client_id=client_id,
-        redirect_uri=redirect_uri,
-        code_verifier=code_verifier,
-    )
-    if not is_code_valid or (auth_code_request and not auth_code_request.scope):
-        return JSONResponse(
-            content={"error": "invalid_grant"},
-            status_code=400,
+    if grant_type == "authorization_code":
+        code = form_data["code"]
+        redirect_uri = form_data["redirect_uri"]
+        # code_verifier is optional for backward compat
+        is_code_valid, auth_code_request = await _check_auth_code(
+            db_session,
+            code=code,
+            client_id=client_id,
+            redirect_uri=redirect_uri,
+            code_verifier=code_verifier,
         )
+        if not is_code_valid or (auth_code_request and not auth_code_request.scope):
+            return JSONResponse(
+                content={"error": "invalid_grant"},
+                status_code=400,
+            )
+
+    elif grant_type == "refresh_token":
+        refresh_token = form_data["refresh_token"]
+        access_token = (
+            await db_session.scalars(
+                select(models.IndieAuthAccessToken)
+                .where(
+                    models.IndieAuthAccessToken.refresh_token == refresh_token,
+                    models.IndieAuthAccessToken.was_refreshed.is_(False),
+                )
+                .options(
+                    joinedload(
+                        models.IndieAuthAccessToken.indieauth_authorization_request
+                    )
+                )
+            )
+        ).one_or_none()
+        if not access_token:
+            raise ValueError("invalid refresh token")
+
+        if access_token.indieauth_authorization_request.client_id != client_id:
+            raise ValueError("invalid client ID")
+
+        auth_code_request = access_token.indieauth_authorization_request
+        access_token.was_refreshed = True
 
     if not auth_code_request:
         raise ValueError("Should never happen")
@@ -237,6 +330,7 @@ async def indieauth_token_endpoint(
     access_token = models.IndieAuthAccessToken(
         indieauth_authorization_request_id=auth_code_request.id,
         access_token=secrets.token_urlsafe(32),
+        refresh_token=secrets.token_urlsafe(32),
         expires_in=3600,
         scope=auth_code_request.scope,
     )
@@ -246,6 +340,7 @@ async def indieauth_token_endpoint(
     return JSONResponse(
         content={
             "access_token": access_token.access_token,
+            "refresh_token": access_token.refresh_token,
             "token_type": "Bearer",
             "scope": auth_code_request.scope,
             "me": config.ID + "/",
@@ -261,8 +356,10 @@ async def _check_access_token(
 ) -> tuple[bool, models.IndieAuthAccessToken | None]:
     access_token_info = (
         await db_session.scalars(
-            select(models.IndieAuthAccessToken).where(
-                models.IndieAuthAccessToken.access_token == token
+            select(models.IndieAuthAccessToken)
+            .where(models.IndieAuthAccessToken.access_token == token)
+            .options(
+                joinedload(models.IndieAuthAccessToken.indieauth_authorization_request)
             )
         )
     ).one_or_none()
@@ -285,6 +382,9 @@ async def _check_access_token(
 @dataclass(frozen=True)
 class AccessTokenInfo:
     scopes: list[str]
+    client_id: str | None
+    access_token: str
+    exp: int
 
 
 async def verify_access_token(
@@ -311,9 +411,71 @@ async def verify_access_token(
 
     return AccessTokenInfo(
         scopes=access_token.scope.split(),
+        client_id=(
+            access_token.indieauth_authorization_request.client_id
+            if access_token.indieauth_authorization_request
+            else None
+        ),
+        access_token=access_token.access_token,
+        exp=int(
+            (
+                access_token.created_at.replace(tzinfo=timezone.utc)
+                + timedelta(seconds=access_token.expires_in)
+            ).timestamp()
+        ),
     )
 
 
+async def check_access_token(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> AccessTokenInfo | None:
+    token = request.headers.get("Authorization", "").removeprefix("Bearer ")
+    if not token:
+        return None
+
+    is_token_valid, access_token = await _check_access_token(db_session, token)
+    if not is_token_valid:
+        return None
+
+    if not access_token or not access_token.scope:
+        raise ValueError("Should never happen")
+
+    access_token_info = AccessTokenInfo(
+        scopes=access_token.scope.split(),
+        client_id=(
+            access_token.indieauth_authorization_request.client_id
+            if access_token.indieauth_authorization_request
+            else None
+        ),
+        access_token=access_token.access_token,
+        exp=int(
+            (
+                access_token.created_at.replace(tzinfo=timezone.utc)
+                + timedelta(seconds=access_token.expires_in)
+            ).timestamp()
+        ),
+    )
+
+    logger.info(
+        "Authenticated with access token from client_id="
+        f"{access_token_info.client_id} scopes={access_token.scope}"
+    )
+
+    return access_token_info
+
+
+async def enforce_access_token(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> AccessTokenInfo:
+    maybe_access_token_info = await check_access_token(request, db_session)
+    if not maybe_access_token_info:
+        raise HTTPException(status_code=401, detail="access token required")
+
+    return maybe_access_token_info
+
+
 @router.post("/revoke_token")
 async def indieauth_revocation_endpoint(
     request: Request,
@@ -333,3 +495,58 @@ async def indieauth_revocation_endpoint(
         content={},
         status_code=200,
     )
+
+
+@router.post("/token_introspection")
+async def oauth_introspection_endpoint(
+    request: Request,
+    credentials: HTTPBasicCredentials = Depends(basic_auth),
+    db_session: AsyncSession = Depends(get_db_session),
+    token: str = Form(),
+) -> JSONResponse:
+    registered_client = (
+        await db_session.scalars(
+            select(models.OAuthClient).where(
+                models.OAuthClient.client_id == credentials.username,
+                models.OAuthClient.client_secret == credentials.password,
+            )
+        )
+    ).one_or_none()
+    if not registered_client:
+        raise HTTPException(status_code=401, detail="unauthenticated")
+
+    access_token = (
+        await db_session.scalars(
+            select(models.IndieAuthAccessToken)
+            .where(models.IndieAuthAccessToken.access_token == token)
+            .join(
+                models.IndieAuthAuthorizationRequest,
+                models.IndieAuthAccessToken.indieauth_authorization_request_id
+                == models.IndieAuthAuthorizationRequest.id,
+            )
+            .where(
+                models.IndieAuthAuthorizationRequest.client_id == credentials.username
+            )
+        )
+    ).one_or_none()
+    if not access_token:
+        return JSONResponse(content={"active": False})
+
+    is_token_valid, _ = await _check_access_token(db_session, token)
+    if not is_token_valid:
+        return JSONResponse(content={"active": False})
+
+    return JSONResponse(
+        content={
+            "active": True,
+            "client_id": credentials.username,
+            "scope": access_token.scope,
+            "exp": int(
+                (
+                    access_token.created_at.replace(tzinfo=timezone.utc)
+                    + timedelta(seconds=access_token.expires_in)
+                ).timestamp()
+            ),
+        },
+        status_code=200,
+    )

app/ldsig.py

@@ -23,6 +23,13 @@ requests_loader = pyld.documentloader.requests.requests_document_loader()
 def _loader(url, options={}):
     # See https://github.com/digitalbazaar/pyld/issues/133
     options["headers"]["Accept"] = "application/ld+json"
+
+    # XXX: temp fix/hack is it seems to be down for now
+    if url == "https://w3id.org/identity/v1":
+        url = (
+            "https://raw.githubusercontent.com/web-payments/web-payments.org"
+            "/master/contexts/identity-v1.jsonld"
+        )
     return requests_loader(url, options)
 
 
@@ -34,7 +41,7 @@ def _options_hash(doc: ap.RawObject) -> str:
     for k in ["type", "id", "signatureValue"]:
         if k in doc:
             del doc[k]
-    doc["@context"] = "https://w3id.org/identity/v1"
+    doc["@context"] = "https://w3id.org/security/v1"
     normalized = jsonld.normalize(
         doc, {"algorithm": "URDNA2015", "format": "application/nquads"}
     )

app/main.py

@@ -62,6 +62,7 @@ from app.config import DOMAIN
 from app.config import ID
 from app.config import USER_AGENT
 from app.config import USERNAME
+from app.config import WEBFINGER_DOMAIN
 from app.config import is_activitypub_requested
 from app.config import verify_csrf_token
 from app.customization import get_custom_router
@@ -80,8 +81,8 @@ from app.utils.highlight import HIGHLIGHT_CSS_HASH
 from app.utils.url import check_url
 from app.webfinger import get_remote_follow_template
 
-# Only images <1MB will be cached, so 64MB of data will be cached
-_RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCache(64)
+# Only images <1MB will be cached, so 32MB of data will be cached
+_RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCache(32)
 
 
 # TODO(ts):
@@ -296,7 +297,7 @@ async def index(
         models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC,
         models.OutboxObject.is_deleted.is_(False),
         models.OutboxObject.is_hidden_from_homepage.is_(False),
-        models.OutboxObject.ap_type != "Article",
+        models.OutboxObject.ap_type.in_(["Announce", "Note", "Video", "Question"]),
     )
     q = select(models.OutboxObject).where(*where)
     total_count = await db_session.scalar(
@@ -464,7 +465,12 @@ async def followers(
     _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse:
     if is_activitypub_requested(request):
-        if config.HIDES_FOLLOWERS:
+        maybe_access_token_info = await indieauth.check_access_token(
+            request,
+            db_session,
+        )
+
+        if config.HIDES_FOLLOWERS and not maybe_access_token_info:
             return ActivityPubResponse(
                 await _empty_followx_collection(
                     db_session=db_session,
@@ -523,7 +529,12 @@ async def following(
     _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse:
     if is_activitypub_requested(request):
-        if config.HIDES_FOLLOWING:
+        maybe_access_token_info = await indieauth.check_access_token(
+            request,
+            db_session,
+        )
+
+        if config.HIDES_FOLLOWING and not maybe_access_token_info:
             return ActivityPubResponse(
                 await _empty_followx_collection(
                     db_session=db_session,
@@ -579,22 +590,34 @@ async def following(
 
 @app.get("/outbox")
 async def outbox(
+    request: Request,
     db_session: AsyncSession = Depends(get_db_session),
     _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse:
+    maybe_access_token_info = await indieauth.check_access_token(
+        request,
+        db_session,
+    )
+
+    # Default restrictions unless the request is authenticated with an access token
+    restricted_where = [
+        models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC,
+        models.OutboxObject.ap_type.in_(["Create", "Note", "Article", "Announce"]),
+    ]
+
     # By design, we only show the last 20 public activities in the oubox
     outbox_objects = (
         await db_session.scalars(
             select(models.OutboxObject)
             .where(
-                models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC,
                 models.OutboxObject.is_deleted.is_(False),
-                models.OutboxObject.ap_type.in_(["Create", "Announce"]),
+                *([] if maybe_access_token_info else restricted_where),
             )
             .order_by(models.OutboxObject.ap_published_at.desc())
             .limit(20)
         )
     ).all()
+
     return ActivityPubResponse(
         {
             "@context": ap.AS_EXTENDED_CTX,
@@ -609,6 +632,49 @@ async def outbox(
     )
 
 
+@app.post("/outbox")
+async def post_outbox(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    access_token_info: indieauth.AccessTokenInfo = Depends(
+        indieauth.enforce_access_token
+    ),
+) -> ActivityPubResponse:
+    payload = await request.json()
+    logger.info(f"{payload=}")
+
+    if payload.get("type") == "Create":
+        assert payload["actor"] == ID
+        obj = payload["object"]
+
+        to_and_cc = obj.get("to", []) + obj.get("cc", [])
+        if ap.AS_PUBLIC in obj.get("to", []) and ID + "/followers" in to_and_cc:
+            visibility = ap.VisibilityEnum.PUBLIC
+        elif ap.AS_PUBLIC in to_and_cc and ID + "/followers" in to_and_cc:
+            visibility = ap.VisibilityEnum.UNLISTED
+        else:
+            visibility = ap.VisibilityEnum.DIRECT
+
+        object_id, outbox_object = await boxes.send_create(
+            db_session,
+            ap_type=obj["type"],
+            source=obj["content"],
+            uploads=[],
+            in_reply_to=obj.get("inReplyTo"),
+            visibility=visibility,
+            content_warning=obj.get("summary"),
+            is_sensitive=obj.get("sensitive", False),
+        )
+    else:
+        raise ValueError("TODO")
+
+    return ActivityPubResponse(
+        outbox_object.ap_object,
+        status_code=201,
+        headers={"Location": boxes.outbox_object_id(object_id)},
+    )
+
+
 @app.get("/featured")
 async def featured(
     db_session: AsyncSession = Depends(get_db_session),
@@ -646,6 +712,14 @@ async def _check_outbox_object_acl(
     if templates.is_current_user_admin(request):
         return None
 
+    maybe_access_token_info = await indieauth.check_access_token(
+        request,
+        db_session,
+    )
+    if maybe_access_token_info:
+        # TODO: check scopes
+        return None
+
     if ap_object.visibility in [
         ap.VisibilityEnum.PUBLIC,
         ap.VisibilityEnum.UNLISTED,
@@ -1015,6 +1089,78 @@ def emoji_by_name(name: str) -> ActivityPubResponse:
     return ActivityPubResponse({"@context": ap.AS_EXTENDED_CTX, **emoji})
 
 
+@app.get("/inbox")
+async def get_inbox(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    access_token_info: indieauth.AccessTokenInfo = Depends(
+        indieauth.enforce_access_token
+    ),
+    page: bool | None = None,
+    next_cursor: str | None = None,
+) -> ActivityPubResponse:
+    where = [
+        models.InboxObject.ap_type.in_(
+            ["Create", "Follow", "Like", "Announce", "Undo", "Update"]
+        )
+    ]
+    total_items = await db_session.scalar(
+        select(func.count(models.InboxObject.id)).where(*where)
+    )
+
+    if not page and not next_cursor:
+        return ActivityPubResponse(
+            {
+                "@context": ap.AS_CTX,
+                "id": ID + "/inbox",
+                "first": ID + "/inbox?page=true",
+                "type": "OrderedCollection",
+                "totalItems": total_items,
+            }
+        )
+
+    q = (
+        select(models.InboxObject)
+        .where(*where)
+        .order_by(models.InboxObject.created_at.desc())
+    )  # type: ignore
+    if next_cursor:
+        q = q.where(
+            models.InboxObject.created_at
+            < pagination.decode_cursor(next_cursor)  # type: ignore
+        )
+    q = q.limit(20)
+
+    items = [item for item in (await db_session.scalars(q)).all()]
+    next_cursor = None
+    if (
+        items
+        and await db_session.scalar(
+            select(func.count(models.InboxObject.id)).where(
+                *where, models.InboxObject.created_at < items[-1].created_at
+            )
+        )
+        > 0
+    ):
+        next_cursor = pagination.encode_cursor(items[-1].created_at)
+
+    collection_page = {
+        "@context": ap.AS_CTX,
+        "id": (
+            ID + "/inbox?page=true"
+            if not next_cursor
+            else ID + f"/inbox?next_cursor={next_cursor}"
+        ),
+        "partOf": ID + "/inbox",
+        "type": "OrderedCollectionPage",
+        "orderedItems": [item.ap_object for item in items],
+    }
+    if next_cursor:
+        collection_page["next"] = ID + f"/inbox?next_cursor={next_cursor}"
+
+    return ActivityPubResponse(collection_page)
+
+
 @app.post("/inbox")
 async def inbox(
     request: Request,
@@ -1110,12 +1256,16 @@ async def post_remote_interaction(
 @app.get("/.well-known/webfinger")
 async def wellknown_webfinger(resource: str) -> JSONResponse:
     """Exposes/servers WebFinger data."""
-    if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]:
+    if resource not in [
+        f"acct:{USERNAME}@{WEBFINGER_DOMAIN}",
+        ID,
+        f"acct:{USERNAME}@{DOMAIN}",
+    ]:
         logger.info(f"Got invalid req for {resource}")
         raise HTTPException(status_code=404)
 
     out = {
-        "subject": f"acct:{USERNAME}@{DOMAIN}",
+        "subject": f"acct:{USERNAME}@{WEBFINGER_DOMAIN}",
         "aliases": [ID],
         "links": [
             {
@@ -1179,15 +1329,11 @@ async def nodeinfo(
     )
 
 
-proxy_client = httpx.AsyncClient(
-    http2=True,
-    follow_redirects=True,
-    timeout=httpx.Timeout(timeout=10.0),
-)
-
-
 async def _proxy_get(
-    request: starlette.requests.Request, url: str, stream: bool
+    proxy_client: httpx.AsyncClient,
+    request: starlette.requests.Request,
+    url: str,
+    stream: bool,
 ) -> httpx.Response:
     # Request the URL (and filter request headers)
     proxy_req = proxy_client.build_request(
@@ -1234,18 +1380,29 @@ async def serve_proxy_media(
     exp: int,
     sig: str,
     encoded_url: str,
+    background_tasks: fastapi.BackgroundTasks,
 ) -> StreamingResponse | PlainTextResponse:
     # Decode the base64-encoded URL
     url = base64.urlsafe_b64decode(encoded_url).decode()
     check_url(url)
     media.verify_proxied_media_sig(exp, url, sig)
 
-    proxy_resp = await _proxy_get(request, url, stream=True)
+    proxy_client = httpx.AsyncClient(
+        follow_redirects=True,
+        timeout=httpx.Timeout(timeout=10.0),
+        transport=httpx.AsyncHTTPTransport(retries=1),
+    )
+
+    async def _close_proxy_client():
+        await proxy_client.aclose()
+
+    background_tasks.add_task(_close_proxy_client)
+    proxy_resp = await _proxy_get(proxy_client, request, url, stream=True)
 
     if proxy_resp.status_code >= 300:
         logger.info(f"failed to proxy {url}, got {proxy_resp.status_code}")
+        await proxy_resp.aclose()
         return PlainTextResponse(
-            "proxy error",
             status_code=proxy_resp.status_code,
         )
 
@@ -1256,6 +1413,7 @@ async def serve_proxy_media(
             _filter_proxy_resp_headers(
                 proxy_resp,
                 [
+                    "content-encoding",
                     "content-length",
                     "content-type",
                     "content-range",
@@ -1278,16 +1436,19 @@ async def serve_proxy_media_resized(
     sig: str,
     encoded_url: str,
     size: int,
+    background_tasks: fastapi.BackgroundTasks,
 ) -> PlainTextResponse:
     if size not in {50, 740}:
         raise ValueError("Unsupported size")
 
+    is_webp_supported = "image/webp" in request.headers.get("accept")
+
     # Decode the base64-encoded URL
     url = base64.urlsafe_b64decode(encoded_url).decode()
     check_url(url)
     media.verify_proxied_media_sig(exp, url, sig)
 
-    if cached_resp := _RESIZED_CACHE.get((url, size)):
+    if (cached_resp := _RESIZED_CACHE.get((url, size))) and is_webp_supported:
         resized_content, resized_mimetype, resp_headers = cached_resp
         return PlainTextResponse(
             resized_content,
@@ -1295,11 +1456,21 @@ async def serve_proxy_media_resized(
             headers=resp_headers,
         )
 
-    proxy_resp = await _proxy_get(request, url, stream=False)
+    proxy_client = httpx.AsyncClient(
+        follow_redirects=True,
+        timeout=httpx.Timeout(timeout=10.0),
+        transport=httpx.AsyncHTTPTransport(retries=1),
+    )
+
+    async def _close_proxy_client():
+        await proxy_client.aclose()
+
+    background_tasks.add_task(_close_proxy_client)
+    proxy_resp = await _proxy_get(proxy_client, request, url, stream=False)
     if proxy_resp.status_code >= 300:
         logger.info(f"failed to proxy {url}, got {proxy_resp.status_code}")
+        await proxy_resp.aclose()
         return PlainTextResponse(
-            "proxy error",
             status_code=proxy_resp.status_code,
         )
 
@@ -1325,10 +1496,10 @@ async def serve_proxy_media_resized(
         is_webp = False
         try:
             resized_buf = BytesIO()
-            i.save(resized_buf, format="webp")
-            is_webp = True
+            i.save(resized_buf, format="webp" if is_webp_supported else i.format)
+            is_webp = is_webp_supported
         except Exception:
-            logger.exception("Failed to convert to webp")
+            logger.exception("Failed to create thumbnail")
             resized_buf = BytesIO()
             i.save(resized_buf, format=i.format)
         resized_buf.seek(0)
@@ -1386,6 +1557,7 @@ async def serve_attachment(
 
 @app.get("/attachments/thumbnails/{content_hash}/{filename}")
 async def serve_attachment_thumbnail(
+    request: Request,
     content_hash: str,
     filename: str,
     db_session: AsyncSession = Depends(get_db_session),
@@ -1400,11 +1572,20 @@ async def serve_attachment_thumbnail(
     if not upload or not upload.has_thumbnail:
         raise HTTPException(status_code=404)
 
-    return FileResponse(
-        UPLOAD_DIR / (content_hash + "_resized"),
-        media_type="image/webp",
-        headers={"Cache-Control": "max-age=31536000"},
-    )
+    is_webp_supported = "image/webp" in request.headers.get("accept")
+
+    if is_webp_supported:
+        return FileResponse(
+            UPLOAD_DIR / (content_hash + "_resized"),
+            media_type="image/webp",
+            headers={"Cache-Control": "max-age=31536000"},
+        )
+    else:
+        return FileResponse(
+            UPLOAD_DIR / content_hash,
+            media_type=upload.content_type,
+            headers={"Cache-Control": "max-age=31536000"},
+        )
 
 
 @app.get("/robots.txt", response_class=PlainTextResponse)
@@ -1464,23 +1645,26 @@ async def json_feed(
             }
         )
     result = {
-        "version": "https://jsonfeed.org/version/1",
+        "version": "https://jsonfeed.org/version/1.1",
         "title": f"{LOCAL_ACTOR.display_name}'s microblog'",
         "home_page_url": LOCAL_ACTOR.url,
         "feed_url": BASE_URL + "/feed.json",
-        "author": {
-            "name": LOCAL_ACTOR.display_name,
-            "url": LOCAL_ACTOR.url,
-        },
+        "authors": [
+            {
+                "name": LOCAL_ACTOR.display_name,
+                "url": LOCAL_ACTOR.url,
+            }
+        ],
         "items": data,
     }
     if LOCAL_ACTOR.icon_url:
-        result["author"]["avatar"] = LOCAL_ACTOR.icon_url  # type: ignore
+        result["authors"][0]["avatar"] = LOCAL_ACTOR.icon_url  # type: ignore
     return result
 
 
 async def _gen_rss_feed(
     db_session: AsyncSession,
+    is_rss: bool,
 ):
     fg = FeedGenerator()
     fg.id(BASE_URL + "/feed.rss")
@@ -1511,8 +1695,12 @@ async def _gen_rss_feed(
 
         fe = fg.add_entry()
         fe.id(outbox_object.url)
+        if outbox_object.name is not None:
+            fe.title(outbox_object.name)
+        elif not is_rss:  # Atom feeds require a title
+            fe.title(outbox_object.url)
+
         fe.link(href=outbox_object.url)
-        fe.title(outbox_object.url)
         fe.description(content)
         fe.content(content)
         fe.published(outbox_object.ap_published_at.replace(tzinfo=timezone.utc))
@@ -1525,7 +1713,7 @@ async def rss_feed(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> PlainTextResponse:
     return PlainTextResponse(
-        (await _gen_rss_feed(db_session)).rss_str(),
+        (await _gen_rss_feed(db_session, is_rss=True)).rss_str(),
         headers={"Content-Type": "application/rss+xml"},
     )
 
@@ -1535,6 +1723,6 @@ async def atom_feed(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> PlainTextResponse:
     return PlainTextResponse(
-        (await _gen_rss_feed(db_session)).atom_str(),
+        (await _gen_rss_feed(db_session, is_rss=False)).atom_str(),
         headers={"Content-Type": "application/atom+xml"},
     )

app/micropub.py

@@ -132,7 +132,7 @@ async def post_micropub_endpoint(
             h = form_data["h"]
         entry_type = f"h-{h}"
 
-    logger.info(f"Creating {entry_type}")
+    logger.info(f"Creating {entry_type=} with {access_token_info=}")
 
     if entry_type != "h-entry":
         return JSONResponse(
@@ -150,7 +150,7 @@ async def post_micropub_endpoint(
     else:
         content = form_data["content"]
 
-    public_id = await send_create(
+    public_id, _ = await send_create(
         db_session,
         "Note",
         content,

app/models.py

@@ -1,4 +1,5 @@
 import enum
+from datetime import datetime
 from typing import Any
 from typing import Optional
 from typing import Union
@@ -54,6 +55,10 @@ class Actor(Base, BaseActor):
     is_blocked = Column(Boolean, nullable=False, default=False, server_default="0")
     is_deleted = Column(Boolean, nullable=False, default=False, server_default="0")
 
+    are_announces_hidden_from_stream = Column(
+        Boolean, nullable=False, default=False, server_default="0"
+    )
+
     @property
     def is_from_db(self) -> bool:
         return True
@@ -432,7 +437,7 @@ class OutboxObjectAttachment(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
 
     upload_id = Column(Integer, ForeignKey("upload.id"), nullable=False)
-    upload = relationship(Upload, uselist=False)
+    upload: Mapped["Upload"] = relationship(Upload, uselist=False)
 
 
 class IndieAuthAuthorizationRequest(Base):
@@ -455,17 +460,45 @@ class IndieAuthAccessToken(Base):
     __tablename__ = "indieauth_access_token"
 
     id = Column(Integer, primary_key=True, index=True)
-    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+    created_at: Mapped[datetime] = Column(
+        DateTime(timezone=True), nullable=False, default=now
+    )
 
     # Will be null for personal access tokens
     indieauth_authorization_request_id = Column(
         Integer, ForeignKey("indieauth_authorization_request.id"), nullable=True
     )
+    indieauth_authorization_request = relationship(
+        IndieAuthAuthorizationRequest,
+        uselist=False,
+    )
 
-    access_token = Column(String, nullable=False, unique=True, index=True)
-    expires_in = Column(Integer, nullable=False)
+    access_token: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
+    refresh_token = Column(String, nullable=True, unique=True, index=True)
+    expires_in: Mapped[int] = Column(Integer, nullable=False)
     scope = Column(String, nullable=False)
     is_revoked = Column(Boolean, nullable=False, default=False)
+    was_refreshed = Column(Boolean, nullable=False, default=False, server_default="0")
+
+
+class OAuthClient(Base):
+    __tablename__ = "oauth_client"
+
+    id = Column(Integer, primary_key=True, index=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+
+    # Request
+    client_name = Column(String, nullable=False)
+    redirect_uris: Mapped[list[str]] = Column(JSON, nullable=True)
+
+    # Optional from request
+    client_uri = Column(String, nullable=True)
+    logo_uri = Column(String, nullable=True)
+    scope = Column(String, nullable=True)
+
+    # Response
+    client_id = Column(String, nullable=False, unique=True, index=True)
+    client_secret = Column(String, nullable=False, unique=True)
 
 
 @enum.unique

app/outgoing_activities.py

@@ -151,7 +151,7 @@ def _set_next_try(
     if not outgoing_activity.tries:
         raise ValueError("Should never happen")
 
-    if outgoing_activity.tries == _MAX_RETRIES:
+    if outgoing_activity.tries >= _MAX_RETRIES:
         outgoing_activity.is_errored = True
         outgoing_activity.next_try = None
     else:

app/prune.py

@@ -102,6 +102,8 @@ async def _prune_old_inbox_objects(
                     models.InboxObject.ap_type.in_(["Note"]),
                 )
             ),
+            # Keep Move object as they are linked to notifications
+            models.InboxObject.ap_type.not_in(["Move"]),
             # Filter by retention days
             models.InboxObject.ap_published_at
             < now() - timedelta(days=INBOX_RETENTION_DAYS),

app/redirect.py

@@ -0,0 +1,28 @@
+from fastapi import Request
+
+from app import templates
+from app.database import AsyncSession
+
+
+async def redirect(
+    request: Request,
+    db_session: AsyncSession,
+    url: str,
+) -> templates.TemplateResponse:
+    """
+    Similar to RedirectResponse, but uses a 200 response with HTML.
+
+    Needed for remote redirects on form submission endpoints,
+    since our CSP policy disallows remote form submission.
+    https://github.com/w3c/webappsec-csp/issues/8#issuecomment-810108984
+    """
+    return await templates.render_template(
+        db_session,
+        request,
+        "redirect.html",
+        {
+            "request": request,
+            "url": url,
+        },
+        headers={"Refresh": "0;url=" + url},
+    )

app/scss/main.scss

@@ -51,17 +51,20 @@ $code-highlight-background: #f0f0f0;
   .p-summary {
     display: inline-block;
   }
-  label {
+  .show-more-btn {
     margin-left: 5px;
   }
-  .show-more-state {
-    display: none;
+  summary {
+    display: inline-block;
   }
-  .show-more-state ~ .obj-content {
-    margin-top: 0;
+  summary::-webkit-details-marker {
+    display: none
   }
-  .show-more-state:checked ~ .obj-content {
-    display: none;
+  &:not([open]) .show-more-btn::after {
+    content: 'show more';
+  }
+  &[open] .show-more-btn::after {
+    content: 'show less';
   }
 }
 .sensitive-attachment {
@@ -429,8 +432,7 @@ a.label-btn {
     .activity-attachment {
         margin: 30px 0 20px 0;
         img, audio, video {
-            width: 100%;
-            max-width: 740px;
+            max-width: calc(min(740px, 100%));
         }
     }
     img.inline-img {
@@ -456,7 +458,7 @@ a.label-btn {
     border: 2px dashed $secondary-color;
 }
 
-.error-box {
+.error-box, .scolor {
     color: $secondary-color;
 }
 
@@ -548,3 +550,22 @@ a.label-btn {
 .margin-top-20 {
   margin-top: 20px;
 }
+
+.video-wrapper {
+    position: relative;
+}
+
+.video-gif-overlay {
+    display: none;
+}
+
+.video-gif-mode + .video-gif-overlay {
+    display: block;
+    position: absolute;
+    top: 5px;
+    left: 5px;
+    padding: 0 3px;
+    font-size: 0.8em;
+    background: rgba(0,0,0,.5);
+    color: #fff;
+}

app/source.py

@@ -3,12 +3,12 @@ import typing
 
 from loguru import logger
 from mistletoe import Document  # type: ignore
+from mistletoe.block_token import CodeFence  # type: ignore
 from mistletoe.html_renderer import HTMLRenderer  # type: ignore
 from mistletoe.span_token import SpanToken  # type: ignore
-from pygments import highlight  # type: ignore
 from pygments.formatters import HtmlFormatter  # type: ignore
 from pygments.lexers import get_lexer_by_name as get_lexer  # type: ignore
-from pygments.lexers import guess_lexer  # type: ignore
+from pygments.util import ClassNotFound  # type: ignore
 from sqlalchemy import select
 
 from app import webfinger
@@ -104,10 +104,16 @@ class CustomRenderer(HTMLRenderer):
         )
         return link
 
-    def render_block_code(self, token: typing.Any) -> str:
+    def render_block_code(self, token: CodeFence) -> str:
+        lexer_attr = ""
+        try:
+            lexer = get_lexer(token.language)
+            lexer_attr = f' data-microblogpub-lexer="{lexer.aliases[0]}"'
+        except ClassNotFound:
+            pass
+
         code = token.children[0].content
-        lexer = get_lexer(token.language) if token.language else guess_lexer(code)
-        return highlight(code, lexer, _FORMATTER)
+        return f"<pre><code{lexer_attr}>\n{code}\n</code></pre>"
 
 
 async def _prefetch_mentioned_actors(

app/static/common.js

@@ -0,0 +1,32 @@
+function hasAudio (video) {
+    return video.mozHasAudio ||
+    Boolean(video.webkitAudioDecodedByteCount) ||
+    Boolean(video.audioTracks && video.audioTracks.length);
+}
+
+function setVideoInGIFMode(video) {
+    if (!hasAudio(video)) {
+        if (typeof video.loop == 'boolean' && video.duration <= 10.0) {
+            video.classList.add("video-gif-mode");
+            video.loop = true;
+            video.controls = false;
+            video.addEventListener("mouseover", () => {
+                video.play();
+            })
+            video.addEventListener("mouseleave", () => {
+                video.pause();
+            })
+        }
+    };
+}
+
+var items = document.getElementsByTagName("video")
+for (var i = 0; i < items.length; i++) {
+    if (items[i].duration) {
+        setVideoInGIFMode(items[i]);
+    } else {
+        items[i].addEventListener("loadeddata", function() {
+            setVideoInGIFMode(this);
+        });
+    }
+}

app/templates/articles.html

@@ -11,8 +11,8 @@
 <ul class="h-feed" id="articles">
 <data class="p-name" value="{{ local_actor.display_name}}'s articles"></data>
 {% for outbox_object in objects %}
-    <li>
-        <span class="muted">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</span> <a href="{{ outbox_object.url }}">{{ outbox_object.name }}</a>
+    <li class="h-entry">
+        <time class="muted dt-published" datetime="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</time> <a href="{{ outbox_object.url }}" class="u-url u-uid p-name">{{ outbox_object.name }}</a>
     </li>
 {% endfor %}
 </ul>

app/templates/index.html

@@ -26,24 +26,30 @@
     <div class="h-feed">
     <data class="p-name" value="{{ local_actor.display_name}}'s notes"></data>
     {% for outbox_object in objects %}
-    {% if outbox_object.ap_type in ["Note", "Article", "Video", "Question"] %}
+    {% if outbox_object.ap_type in ["Note", "Video", "Question"] %}
     {{ utils.display_object(outbox_object) }}
     {% elif outbox_object.ap_type == "Announce" %}
-    <div class="shared-header"><strong>{{ utils.display_tiny_actor_icon(local_actor) }} {{ local_actor.display_name | clean_html(local_actor) | safe  }}</strong> shared <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
-    {{ utils.display_object(outbox_object.relates_to_anybox_object) }}
+    <div class="h-entry" id="{{ outbox_object.permalink_id }}">
+        <div class="shared-header"><strong><a class="p-author h-card" href="{{ local_actor.url }}">{{ utils.display_tiny_actor_icon(local_actor) }}  {{ local_actor.display_name | clean_html(local_actor) | safe }}</a></strong> shared <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
+        <div class="h-cite u-repost-of">
+            {{ utils.display_object(outbox_object.relates_to_anybox_object, is_h_entry=False) }}
+        </div>
+    </div>
     {% endif %}
     {% endfor %}
     </div>
 
-    <div class="box">
-        {% if has_previous_page %}
-        <a href="{{ url_for("index") }}?page={{ current_page - 1 }}">Previous</a>
-        {% endif %}
+    {% if has_previous_page or has_next_page %}
+        <div class="box">
+            {% if has_previous_page %}
+            <a href="{{ url_for("index") }}?page={{ current_page - 1 }}">Previous</a>
+            {% endif %}
 
-        {% if has_next_page %}
-        <a href="{{ url_for("index") }}?page={{ current_page + 1 }}">Next</a>
-        {% endif %}
-    </div>
+            {% if has_next_page %}
+            <a href="{{ url_for("index") }}?page={{ current_page + 1 }}">Next</a>
+            {% endif %}
+        </div>
+    {% endif %}
 
 {% else %}
     <div class="empty-state">

app/templates/indieauth_flow.html

@@ -10,8 +10,12 @@
         {% endif %}
         <div class="indieauth-details">
             <div>
-                <a class="lcolor" href="{{ client.url }}">{{ client.name }}</a>
-                <p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
+                {% if client.url %}
+                    <a class="scolor" href="{{ client.url }}">{{ client.name }}</a>
+                {% else %}
+                    <span class="scolor">{{ client.name }}</span>
+                {% endif %}
+                <p>wants you to login{% if me %} as <strong class="lcolor">{{ me }}</strong>{% endif %} with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
 
 
                 <form method="POST" action="{{ url_for('indieauth_flow') }}" class="form">

app/templates/layout.html

@@ -55,5 +55,6 @@
 {% if is_admin %}
 <script src="{{ BASE_URL }}/static/common-admin.js?v={{ JS_HASH }}"></script>
 {% endif %}
+<script src="{{ BASE_URL }}/static/common.js?v={{ JS_HASH }}"></script>
 </body>
 </html>

app/templates/notifications.html

@@ -51,7 +51,7 @@
             {% elif notif.notification_type.value == "unblock" %}
                 {{ notif_actor_action(notif, "was unblocked") }}
                 {{ utils.display_actor(notif.actor, actors_metadata) }}
-            {%- elif notif.notification_type.value == "move" %}
+            {%- elif notif.notification_type.value == "move" and notif.inbox_object %}
                 {# for move notif, the actor is the target and the inbox object the Move activity #}
                 <div class="actor-action">
                     <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.inbox_object.actor.ap_id }}">

app/templates/object.html

@@ -15,7 +15,7 @@
 <meta content="article" property="og:type" />
 <meta content="{{ outbox_object.url }}" property="og:url" />
 <meta content="{{ local_actor.display_name }}'s microblog" property="og:site_name" />
-<meta content="{% if outbox_object.name %}{{ name }}{% else %}Note{% endif %}" property="og:title" />
+<meta content="{% if outbox_object.name %}{{ outbox_object.name }}{% else %}Note{% endif %}" property="og:title" />
 <meta content="{{ excerpt }}" property="og:description" />
 <meta content="{{ local_actor.icon_url }}" property="og:image" />
 <meta content="summary" property="twitter:card" />

app/templates/redirect.html

@@ -0,0 +1,15 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>{{ local_actor.display_name }}'s microblog - Redirect</title>
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    <p>You are being redirected to: <a href="{{ url }}">{{ url }}</a></p>
+</div>
+
+{% endblock %}

app/templates/utils.html

@@ -32,6 +32,29 @@
 {% endblock %}
 {% endmacro %}
 
+{% macro admin_hide_shares_button(actor) %}
+{% block admin_hide_shares_button scoped %}
+<form action="{{ request.url_for("admin_actions_hide_announces") }}" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url() }}
+    <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
+    <input type="submit" value="hide shares">
+</form>
+{% endblock %}
+{% endmacro %}
+
+{% macro admin_show_shares_button(actor) %}
+{% block admin_show_shares_button scoped %}
+<form action="{{ request.url_for("admin_actions_show_announces") }}" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url() }}
+    <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
+    <input type="submit" value="show shares">
+</form>
+{% endblock %}
+{% endmacro %}
+
+
 {% macro admin_follow_button(actor) %}
 {% block admin_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_follow") }}" method="POST">
@@ -247,7 +270,7 @@
 
 {% macro display_tiny_actor_icon(actor) %}
 {% block display_tiny_actor_icon scoped %}
-    <img class="tiny-actor-icon" src="{{ actor.resized_icon_url }}" alt="{{ actor.display_name }}'s avatar">
+    <img class="tiny-actor-icon" src="{{ actor.resized_icon_url }}" alt="">
 {% endblock %}
 {% endmacro %}
 
@@ -342,6 +365,11 @@
                     <li>rejected</li>
                 {% endif %}
             {% endif %}
+            {% if actor.are_announces_hidden_from_stream %}
+                <li>{{ admin_show_shares_button(actor) }}</li>
+            {% else %}
+                <li>{{ admin_hide_shares_button(actor) }}</li>
+            {% endif %}
             {% if with_details %}
                 <li><a href="{{ actor.url }}" class="label-btn">remote profile</a></li>
             {% endif %}
@@ -425,13 +453,16 @@
     {% if attachment.type == "Image" or (attachment | has_media_type("image")) %}
         {% if attachment.url not in object.inlined_images %}
         <a class="media-link" href="{{ attachment.proxied_url }}" target="_blank">
-            <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment">
+            <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment u-photo">
         </a>
         {% endif %}
     {% elif attachment.type == "Video" or (attachment | has_media_type("video")) %}
-    <video controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %}></video>
+    <div class="video-wrapper">
+    <video controls preload="metadata" src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %} class="u-video"></video>
+    <div class="video-gif-overlay">GIF</div>
+    </div>
     {% elif attachment.type == "Audio" or (attachment | has_media_type("audio")) %}
-    <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} class="attachment"></audio>
+    <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} class="attachment u-audio"></audio>
     {% elif attachment.type == "Link" %}
     <a href="{{ attachment.url }}" class="attachment">{{ attachment.url | truncate(64, True) }}</a> ({{ attachment.mimetype}})
     {% else %}
@@ -467,7 +498,7 @@
 </div>
 
     <p class="in-reply-to">in reply to <a href="{{ wm_reply.in_reply_to }}" title="{{ wm_reply.in_reply_to }}" rel="nofollow">
-        this note
+        this object
     </a></p>
 
     <div class="obj-content margin-top-20">
@@ -514,7 +545,7 @@
 
     {% if object.in_reply_to %}
     <p class="in-reply-to">in reply to <a href="{% if is_admin and object.is_in_reply_to_from_inbox %}{{ url_for("get_lookup") }}?query={% endif %}{{ object.in_reply_to }}" title="{{ object.in_reply_to }}" rel="nofollow">
-            this {{ object.ap_type|lower }}
+            this object
         </a></p>
     {% endif %}
 
@@ -549,12 +580,13 @@
     {% endif %}
 
     {% if object.summary %}
-    <div class="show-more-wrapper">
-        <div class="p-summary">
-            <p>{{ object.summary | clean_html(object) | safe }}</p>
-        </div>
-        <label for="show-more-{{ object.permalink_id }}" class="show-more-btn">show/hide more</label>
-        <input class="show-more-state" type="checkbox" aria-hidden="true" id="show-more-{{ object.permalink_id }}" checked>
+    <details class="show-more-wrapper">
+        <summary>
+            <div class="p-summary">
+                <p>{{ object.summary | clean_html(object) | safe }}</p>
+            </div>
+            <span class="show-more-btn" aria-hidden="true"></span>
+        </summary>
     {% endif %}
         <div class="obj-content">
         <div class="e-content">
@@ -615,7 +647,7 @@
 
         </div>
     {% if object.summary %}
-    </div>
+    </details>
     {% endif %}
 
     <div class="activity-attachment">
@@ -750,7 +782,7 @@
                 {{ admin_expand_button(object) }}
             </li>
         {% endif %}
-        {% if object.is_from_inbox and not object.announced_via_outbox_object_ap_id %}
+        {% if object.is_from_inbox and not object.announced_via_outbox_object_ap_id and object.is_local_reply %}
             <li>
                 {{ admin_force_delete_button(object.ap_id) }}
             </li>

app/uploads.py

@@ -60,7 +60,7 @@ async def save_upload(db_session: AsyncSession, f: UploadFile) -> models.Upload:
             destination_image.putdata(original_image.getdata())
             destination_image.save(
                 dest_filename,
-                format=_original_image.format,
+                format=_original_image.format,  # type: ignore
             )
 
             with open(dest_filename, "rb") as dest_f:

app/utils/facepile.py

@@ -1,5 +1,6 @@
 import datetime
 from dataclasses import dataclass
+from datetime import timezone
 from typing import Any
 from typing import Optional
 
@@ -9,7 +10,7 @@ from app import media
 from app.models import InboxObject
 from app.models import Webmention
 from app.utils.datetime import parse_isoformat
-from app.utils.url import make_abs
+from app.utils.url import must_make_abs
 
 
 @dataclass
@@ -39,13 +40,15 @@ class Face:
                     return cls(
                         ap_actor_id=None,
                         url=(
-                            item["properties"]["url"][0]
+                            must_make_abs(
+                                item["properties"]["url"][0], webmention.source
+                            )
                             if item["properties"].get("url")
                             else webmention.source
                         ),
                         name=item["properties"]["name"][0],
                         picture_url=media.resized_media_url(
-                            make_abs(
+                            must_make_abs(
                                 item["properties"]["photo"][0], webmention.source
                             ),  # type: ignore
                             50,
@@ -65,7 +68,7 @@ class Face:
                         url=webmention.source,
                         name=author["properties"]["name"][0],
                         picture_url=media.resized_media_url(
-                            make_abs(
+                            must_make_abs(
                                 author["properties"]["photo"][0], webmention.source
                             ),  # type: ignore
                             50,
@@ -96,13 +99,13 @@ def _parse_face(webmention: Webmention, items: list[dict[str, Any]]) -> Face | N
                 return Face(
                     ap_actor_id=None,
                     url=(
-                        item["properties"]["url"][0]
+                        must_make_abs(item["properties"]["url"][0], webmention.source)
                         if item["properties"].get("url")
                         else webmention.source
                     ),
                     name=item["properties"]["name"][0],
                     picture_url=media.resized_media_url(
-                        make_abs(
+                        must_make_abs(
                             item["properties"]["photo"][0], webmention.source
                         ),  # type: ignore
                         50,
@@ -140,13 +143,23 @@ class WebmentionReply:
                             f"webmention id={webmention.id}"
                         )
                         break
+
+                    if "published" in item["properties"]:
+                        published_at = (
+                            parse_isoformat(item["properties"]["published"][0])
+                            .astimezone(timezone.utc)
+                            .replace(tzinfo=None)
+                        )
+                    else:
+                        published_at = webmention.created_at  # type: ignore
+
                     return cls(
                         face=face,
                         content=item["properties"]["content"][0]["html"],
-                        url=item["properties"]["url"][0],
-                        published_at=parse_isoformat(
-                            item["properties"]["published"][0]
-                        ).replace(tzinfo=None),
+                        url=must_make_abs(
+                            item["properties"]["url"][0], webmention.source
+                        ),
+                        published_at=published_at,
                         in_reply_to=webmention.target,  # type: ignore
                         webmention_id=webmention.id,  # type: ignore
                     )

app/utils/highlight.py

@@ -32,23 +32,22 @@ def highlight(html: str) -> str:
 
         # If this comes from a microblog.pub instance we may have the language
         # in the class name
-        if "class" in code.attrs and code.attrs["class"][0].startswith("language-"):
+        if "data-microblogpub-lexer" in code.attrs:
             try:
-                lexer = get_lexer_by_name(
-                    code.attrs["class"][0].removeprefix("language-")
-                )
+                lexer = get_lexer_by_name(code.attrs["data-microblogpub-lexer"])
             except Exception:
                 lexer = guess_lexer(code_content)
-        else:
-            lexer = guess_lexer(code_content)
 
-        # Replace the code with Pygment output
-        # XXX: the HTML escaping causes issue with Python type annotations
-        code_content = code_content.replace(") -> ", ") -> ")
-        code.parent.replaceWith(
-            BeautifulSoup(
-                phighlight(code_content, lexer, _FORMATTER), "html5lib"
-            ).body.next
-        )
+            # Replace the code with Pygment output
+            # XXX: the HTML escaping causes issue with Python type annotations
+            code_content = code_content.replace(") -> ", ") -> ")
+            code.parent.replaceWith(
+                BeautifulSoup(
+                    phighlight(code_content, lexer, _FORMATTER), "html5lib"
+                ).body.next
+            )
+        else:
+            code.name = "div"
+            code["class"] = code.get("class", []) + ["highlight"]
 
     return soup.body.encode_contents().decode()

app/utils/indieauth.py

@@ -10,7 +10,7 @@ from app.utils.url import make_abs
 class IndieAuthClient:
     logo: str | None
     name: str
-    url: str
+    url: str | None
 
 
 def _get_prop(props: dict[str, Any], name: str, default=None) -> Any:

app/utils/mastodon.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+from loguru import logger
+
+from app.webfinger import get_actor_url
+
+
+def _load_mastodon_following_accounts_csv_file(path: str) -> list[str]:
+    handles = []
+    for line in Path(path).read_text().splitlines()[1:]:
+        handle = line.split(",")[0]
+        handles.append(handle)
+
+    return handles
+
+
+async def get_actor_urls_from_following_accounts_csv_file(
+    path: str,
+) -> list[tuple[str, str]]:
+    actor_urls = []
+    for handle in _load_mastodon_following_accounts_csv_file(path):
+        try:
+            actor_url = await get_actor_url(handle)
+        except Exception:
+            logger.error("Failed to fetch actor URL for {handle=}")
+        else:
+            if actor_url:
+                actor_urls.append((handle, actor_url))
+            else:
+                logger.info(f"No actor URL found for {handle=}")
+
+    return actor_urls

app/utils/opengraph.py

@@ -62,6 +62,13 @@ def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
         if u := raw.get(maybe_rel):
             raw[maybe_rel] = make_abs(u, url)
 
+            if not is_url_valid(raw[maybe_rel]):
+                logger.info(f"Invalid url {raw[maybe_rel]}")
+                if maybe_rel == "url":
+                    raw["url"] = url
+                elif maybe_rel == "image":
+                    raw["image"] = None
+
     return OpenGraphMeta.parse_obj(raw)
 
 

app/utils/url.py

@@ -21,6 +21,13 @@ def make_abs(url: str | None, parent: str) -> str | None:
     )
 
 
+def must_make_abs(url: str | None, parent: str) -> str:
+    abs_url = make_abs(url, parent)
+    if not abs_url:
+        raise ValueError("missing URL")
+    return abs_url
+
+
 class InvalidURLError(Exception):
     pass
 
@@ -54,7 +61,7 @@ def is_url_valid(url: str) -> bool:
     if not parsed.hostname or parsed.hostname.lower() in ["localhost"]:
         return False
 
-    if parsed.hostname in BLOCKED_SERVERS:
+    if is_hostname_blocked(parsed.hostname):
         logger.warning(f"{parsed.hostname} is blocked")
         return False
 
@@ -81,3 +88,11 @@ def check_url(url: str) -> None:
         raise InvalidURLError(f'"{url}" is invalid')
 
     return None
+
+
+@functools.lru_cache(maxsize=256)
+def is_hostname_blocked(hostname: str) -> bool:
+    for blocked_hostname in BLOCKED_SERVERS:
+        if hostname == blocked_hostname or hostname.endswith(f".{blocked_hostname}"):
+            return True
+    return False

app/utils/webmentions.py

@@ -24,7 +24,7 @@ async def _discover_webmention_endoint(url: str) -> str | None:
                 follow_redirects=True,
             )
             resp.raise_for_status()
-        except (httpx.HTTPError, httpx.HTTPStatusError):
+        except Exception:
             logger.exception(f"Failed to discover webmention endpoint for {url}")
             return None
 

app/webfinger.py

@@ -1,3 +1,4 @@
+import xml.etree.ElementTree as ET
 from typing import Any
 from urllib.parse import urlparse
 
@@ -8,33 +9,85 @@ from app import config
 from app.utils.url import check_url
 
 
+async def get_webfinger_via_host_meta(host: str) -> str | None:
+    resp: httpx.Response | None = None
+    is_404 = False
+    async with httpx.AsyncClient() as client:
+        for i, proto in enumerate({"http", "https"}):
+            try:
+                url = f"{proto}://{host}/.well-known/host-meta"
+                check_url(url)
+                resp = await client.get(
+                    url,
+                    headers={
+                        "User-Agent": config.USER_AGENT,
+                    },
+                    follow_redirects=True,
+                )
+                resp.raise_for_status()
+                break
+            except httpx.HTTPStatusError as http_error:
+                logger.exception("HTTP error")
+                if http_error.response.status_code in [403, 404, 410]:
+                    is_404 = True
+                    continue
+                raise
+            except httpx.HTTPError:
+                logger.exception("req failed")
+                # If we tried https first and the domain is "http only"
+                if i == 0:
+                    continue
+                break
+
+        if is_404:
+            return None
+
+    if resp:
+        tree = ET.fromstring(resp.text)
+        maybe_link = tree.find(
+            "./{http://docs.oasis-open.org/ns/xri/xrd-1.0}Link[@rel='lrdd']"
+        )
+        if maybe_link is not None:
+            return maybe_link.attrib.get("template")
+
+    return None
+
+
 async def webfinger(
     resource: str,
+    webfinger_url: str | None = None,
 ) -> dict[str, Any] | None:  # noqa: C901
     """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL."""
     resource = resource.strip()
     logger.info(f"performing webfinger resolution for {resource}")
-    protos = ["https", "http"]
-    if resource.startswith("http://"):
-        protos.reverse()
-        host = urlparse(resource).netloc
-    elif resource.startswith("https://"):
-        host = urlparse(resource).netloc
+    urls = []
+    host = None
+    if webfinger_url:
+        urls = [webfinger_url]
     else:
-        if resource.startswith("acct:"):
-            resource = resource[5:]
-        if resource.startswith("@"):
-            resource = resource[1:]
-        _, host = resource.split("@", 1)
-        resource = "acct:" + resource
+        if resource.startswith("http://"):
+            host = urlparse(resource).netloc
+            url = f"http://{host}/.well-known/webfinger"
+        elif resource.startswith("https://"):
+            host = urlparse(resource).netloc
+            url = f"https://{host}/.well-known/webfinger"
+        else:
+            protos = ["https", "http"]
+            _, host = resource.split("@", 1)
+            urls = [f"{proto}://{host}/.well-known/webfinger" for proto in protos]
+
+    if resource.startswith("acct:"):
+        resource = resource[5:]
+    if resource.startswith("@"):
+        resource = resource[1:]
+    resource = "acct:" + resource
 
     is_404 = False
 
     resp: httpx.Response | None = None
     async with httpx.AsyncClient() as client:
-        for i, proto in enumerate(protos):
+        for i, url in enumerate(urls):
             try:
-                url = f"{proto}://{host}/.well-known/webfinger"
                 check_url(url)
                 resp = await client.get(
                     url,
@@ -58,7 +111,14 @@ async def webfinger(
                 if i == 0:
                     continue
                 break
+
     if is_404:
+        if not webfinger_url and host:
+            if webfinger_url := (await get_webfinger_via_host_meta(host)):
+                return await webfinger(
+                    resource,
+                    webfinger_url=webfinger_url,
+                )
         return None
 
     if resp:

app/webmentions.py

@@ -17,6 +17,7 @@ from app.boxes import _get_outbox_likes_count
 from app.boxes import _get_outbox_replies_count
 from app.boxes import get_outbox_object_by_ap_id
 from app.boxes import get_outbox_object_by_slug_and_short_id
+from app.boxes import is_notification_enabled
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.utils import microformats
@@ -118,12 +119,13 @@ async def webmention_endpoint(
                 db_session, existing_webmention_in_db, mentioned_object
             )
 
-            notif = models.Notification(
-                notification_type=models.NotificationType.DELETED_WEBMENTION,
-                outbox_object_id=mentioned_object.id,
-                webmention_id=existing_webmention_in_db.id,
-            )
-            db_session.add(notif)
+            if is_notification_enabled(models.NotificationType.DELETED_WEBMENTION):
+                notif = models.Notification(
+                    notification_type=models.NotificationType.DELETED_WEBMENTION,
+                    outbox_object_id=mentioned_object.id,
+                    webmention_id=existing_webmention_in_db.id,
+                )
+                db_session.add(notif)
 
             await db_session.commit()
 
@@ -144,12 +146,13 @@ async def webmention_endpoint(
         await db_session.flush()
         webmention = existing_webmention_in_db
 
-        notif = models.Notification(
-            notification_type=models.NotificationType.UPDATED_WEBMENTION,
-            outbox_object_id=mentioned_object.id,
-            webmention_id=existing_webmention_in_db.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.UPDATED_WEBMENTION):
+            notif = models.Notification(
+                notification_type=models.NotificationType.UPDATED_WEBMENTION,
+                outbox_object_id=mentioned_object.id,
+                webmention_id=existing_webmention_in_db.id,
+            )
+            db_session.add(notif)
     else:
         new_webmention = models.Webmention(
             source=source,
@@ -162,12 +165,13 @@ async def webmention_endpoint(
         await db_session.flush()
         webmention = new_webmention
 
-        notif = models.Notification(
-            notification_type=models.NotificationType.NEW_WEBMENTION,
-            outbox_object_id=mentioned_object.id,
-            webmention_id=new_webmention.id,
-        )
-        db_session.add(notif)
+        if is_notification_enabled(models.NotificationType.NEW_WEBMENTION):
+            notif = models.Notification(
+                notification_type=models.NotificationType.NEW_WEBMENTION,
+                outbox_object_id=mentioned_object.id,
+                webmention_id=new_webmention.id,
+            )
+            db_session.add(notif)
 
     # Determine the webmention type
     for item in data.get("items", []):

docs/install.md

@@ -191,6 +191,68 @@ http {
 }
 ```
 
+
+## (Advanced) Running on a subdomain
+
+It is possible to run microblogpub on a subdomain (`sub.domain.tld`) while being reachable from the root root domain (`domain.tld`) using the `name@domain.tld` handle.
+
+This requires forwarding/proxying requests from the root domain to the subdomain, for example using NGINX:
+
+```nginx
+location /.well-known/webfinger {
+  add_header Access-Control-Allow-Origin '*';
+  return 301 https://sub.domain.tld$request_uri;
+}
+```
+
+And updating `data/profile.toml` to specify the root domain as the webfinger domain:
+
+```toml
+webfinger_domain = "domain.tld"
+```
+
+Once configured correctly, people will be able to follow you using `name@domain.tld`, while using `sub.domain.tld` for the web interface.
+
+
+## (Advanced) Running from subpath
+
+It is possible to configure microblogpub to run from subpath.
+To achieve this, do the following configuration _between_ config and start steps.
+i.e. _after_ you run `make config` or `poetry run inv configuration-wizard`,
+but _before_ you run `docker compose up` or `poetry run supervisord`.
+Changing this settings on an instance which has some posts or was seen by other instances will likely break links to these posts or federation (i.e. links to your instance, posts and profile from other instances).
+
+The following steps will explain how to configure instance to be available at `https://example.com/subdir`.
+Change them to your actual domain and subdir.
+
+* Edit `data/profile.toml` file, add this line:
+
+		id = "https://example.com/subdir"
+
+* Edit `misc/*-supervisord.conf` file which is relevant to you (it depends on how you start microblogpub - if in doubt, do the same change in all of them) - in `[program:uvicorn]` section, in the line which starts with `command`, add this argument at the very end: ` --root-path /subdir`
+
+Above two steps are enough to configure microblogpub.
+Next, you also need to configure reverse proxy.
+It might slightly differ if you plan to have other services running on the same domain, but for [NGINX config shown above](#reverse-proxy), the following changes are enough:
+
+* Add subdir to location, so location block starts like this:
+
+		location /subdir {
+
+* Add `/` at the end of `proxy_pass` directive, like this:
+
+		proxy_pass http://localhost:8000/;
+
+These two changes will instruct NGINX that requests sent to `https://example.com/subdir/...` should be forwarded to `http://localhost:8000/...`.
+
+* Inside `server` block, add redirects for well-known URLs (add these lines after `client_max_body_size`, remember to replace `subdir` with your actual subdir!):
+
+		location /.well-known/webfinger { return 301 /subdir$request_uri; }
+		location /.well-known/nodeinfo  { return 301 /subdir$request_uri; }
+		location /.well-known/oauth-authorization-server  { return 301 /subdir$request_uri; }
+
+* Optionally, [check robots.txt from a running microblogpub instance](https://microblog.pub/robots.txt) and integrate it into robots.txt file in the root of your server - remember to prepend `subdir` to URLs, so for example `Disallow: /admin` becomes `Disallow: /subdir/admin`.
+
 ## YunoHost edition
 
 [YunoHost](https://yunohost.org/) support is available (although it is not an official package for now): <https://git.sr.ht/~tsileo/microblog.pub_ynh>.

docs/user_guide.md

@@ -25,9 +25,10 @@ As these two config items define your ActivityPub handle `@handle@domain`.
 
 You can tweak your profile by tweaking these items:
 
- - `name`
- - `summary` (using Markdown)
- - `icon_url`
+ - `name`: The name shown with your profile.
+ - `summary`: The summary or 'bio' part of your profile, written in Markdown.
+ - `icon_url`: Your profile image or avatar.
+ - `image_url`: This provides a 'header' or 'banner' image. Note that it is not shown by the default Microblog.pub templates. It will be used by Mastodon (which uses a 3:1 ratio image) and Pleroma. Pixelfed and Peertube, for example, don't show these images by default.
 
 Whenever one of these config items is updated, an `Update` activity will be sent to all known servers to update your remote profile.
 
@@ -35,6 +36,15 @@ The server will need to be restarted for taking changes into account.
 
 Before restarting the server, you can ensure you haven't made any mistakes by running the [configuration checking task](/user_guide.html#configuration-checking).
 
+Note that currently `image_url` is not used anywhere in microblog.pub itself, but other clients/servers do occasionally use it when showing remote profiles as a background image.
+Also, this image _can_ be used in microblog.pub - just add this:
+
+```html
+<img src="{{ local_actor.image_url | media_proxy_url }}">
+```
+
+to an appropriate place of your template (most likely, `header.html`).
+For more information, see a section about [custom templates](/user_guide.html#custom-templates) further in this document.
 
 ### Profile metadata
 
@@ -98,6 +108,39 @@ privacy_replace = [
 ]
 ```
 
+### Disabling certain notification types
+
+All notifications are enabled by default.
+
+You can disabled specific notifications by adding them to the `disabled_notifications` list.
+
+This example disables likes and shares notifications:
+
+```
+disabled_notifications = ["like", "announce"]
+```
+
+#### Available notification types
+
+ - `new_follower`
+ - `rejected_follower`
+ - `unfollow`
+ - `follow_request_accepted`
+ - `follow_request_rejected`
+ - `move`
+ - `like`
+ - `undo_like`
+ - `announce`
+ - `undo_announce`
+ - `mention`
+ - `new_webmention`
+ - `updated_webmention`
+ - `deleted_webmention`
+ - `blocked`
+ - `unblocked`
+ - `block`
+ - `unblock`
+
 ### Customization
 
 #### Default emoji
@@ -128,10 +171,35 @@ $secondary-color: #32cd32;
 
 See `app/scss/main.scss` to see what variables can be overridden.
 
+You will need to [recompile CSS](#recompiling-css-files) after doing any CSS changes (for actual css files to be updates) and restart microblog.pub (for css link in HTML documents to be updated with a new checksum - otherwise, browsers that downloaded old CSS will keep using it).
+
+#### Custom favicon
+
+By default, microblog.pub favicon is a square of `$primary-color` CSS color (see above section on how to redefine CSS colors).
+You can change it to any icon you like - just save a desired file as `data/favicon.ico`.
+After that, run the "[recompile CSS](#recompiling-css-files)" task to copy it to `app/static/favicon.ico`.
+
 #### Custom templates
 
 If you'd like to customize your instance's theme beyond CSS, you can modify the app's HTML by placing templates in `data/templates` which overwrite the defaults in `app/templates`.
 
+Templates are written using [Jinja](https://jinja.palletsprojects.com/en/latest/templates/) templating language.
+Moreover, `utils.html` has scoped blocks around the body of every macro.
+This allows macros to be overridden individually in `data/templates/utils.html`, without copying the whole file.
+For example, to only override the display of a specific actor's name/icon, you can create `data/templates/utils.html` file with following content:
+
+```jinja
+{% extends "app/utils.html" %}
+
+{% block display_actor %}
+	{% if actor.ap_id == "https://me.example.com" %}
+		<!-- custom actor display -->
+	{% else %}
+		{{ super() }}
+	{% endif %}
+{% endblock %}
+```
+
 #### Custom Content Security Policy (CSP)
 
 You can override the default Content Security Policy by adding a line in `data/profile.toml`:
@@ -287,7 +355,7 @@ First you need to grab the "ActivityPub actor URL" for your existing account:
 
 ```bash
 # For a Python install
-poetry run inv webfinger username@domain.tld
+poetry run inv webfinger username@instance-you-want-to-move-from.tld
 ```
 
 Edit the config.
@@ -296,7 +364,7 @@ Edit the config.
 
 ```bash
 # For a Docker install
-make account=username@domain.tld webfinger
+make account=username@instance-you-want-to-move-from.tld webfinger
 ```
 
 Edit the config.
@@ -306,11 +374,35 @@ Edit the config.
 And add a reference to your old/existing account in `profile.toml`:
 
 ```toml
-also_known_as = "my@old-account.com"
+also_known_as = "https://instance-you-want-to-move-form.tld/users/username"
 ```
 
 Restart the server, and you should be able to complete the move from your existing account.
 
+Note that if you already have a redirect in place on Mastodon, you may have to remove it before initiating the migration.
+
+## Import follows from Mastodon
+
+You can import the list of follows/following accounts from Mastodon.
+
+It requires downloading the "Follows" CSV file from your Mastodon instance via "Settings" / "Import and export" / "Data export".
+
+Then you need to run the import task:
+
+### Python edition
+
+```bash
+# For a Python install
+poetry run inv import-mastodon-following-accounts following_accounts.csv
+```
+
+### Docker edition
+
+```bash
+# For a Docker install
+make path=following_accounts.csv import-mastodon-following-accounts
+```
+
 ## Tasks
 
 ### Configuration checking

pyproject.toml

@@ -14,7 +14,7 @@ bcrypt = "^3.2.2"
 itsdangerous = "^2.1.2"
 python-multipart = "^0.0.5"
 tomli = "^2.0.1"
-httpx = {extras = ["http2"], version = "^0.23.0"}
+httpx = {version = "0.23.0", extras = ["http2"]}
 SQLAlchemy = {extras = ["asyncio"], version = "^1.4.39"}
 alembic = "^1.8.0"
 bleach = "^5.0.0"

tasks.py

@@ -2,17 +2,49 @@ import asyncio
 import io
 import shutil
 import tarfile
+from collections import namedtuple
 from contextlib import contextmanager
+from inspect import getfullargspec
 from pathlib import Path
 from typing import Generator
 from typing import Optional
+from unittest.mock import patch
 
 import httpx
+import invoke  # type: ignore
 from invoke import Context  # type: ignore
 from invoke import run  # type: ignore
 from invoke import task  # type: ignore
 
 
+def fix_annotations():
+    """
+    Pyinvoke doesn't accept annotations by default, this fix that
+    Based on: @zelo's fix in https://github.com/pyinvoke/invoke/pull/606
+    Context in: https://github.com/pyinvoke/invoke/issues/357
+        Python 3.11 https://github.com/pyinvoke/invoke/issues/833
+    """
+
+    ArgSpec = namedtuple("ArgSpec", ["args", "defaults"])
+
+    def patched_inspect_getargspec(func):
+        spec = getfullargspec(func)
+        return ArgSpec(spec.args, spec.defaults)
+
+    org_task_argspec = invoke.tasks.Task.argspec
+
+    def patched_task_argspec(*args, **kwargs):
+        with patch(
+            target="inspect.getargspec", new=patched_inspect_getargspec, create=True
+        ):
+            return org_task_argspec(*args, **kwargs)
+
+    invoke.tasks.Task.argspec = patched_task_argspec
+
+
+fix_annotations()
+
+
 @task
 def generate_db_migration(ctx, message):
     # type: (Context, str) -> None
@@ -353,3 +385,40 @@ def check_config(ctx):
         sys.exit(1)
     else:
         print("Config is OK")
+
+
+@task
+def import_mastodon_following_accounts(ctx, path):
+    # type: (Context, str) -> None
+    from loguru import logger
+
+    from app.boxes import _get_following
+    from app.boxes import _send_follow
+    from app.database import async_session
+    from app.utils.mastodon import get_actor_urls_from_following_accounts_csv_file
+
+    async def _import_following() -> int:
+        count = 0
+        async with async_session() as db_session:
+            followings = {
+                following.ap_actor_id for following in await _get_following(db_session)
+            }
+            for (
+                handle,
+                actor_url,
+            ) in await get_actor_urls_from_following_accounts_csv_file(path):
+                if actor_url in followings:
+                    logger.info(f"Already following {handle}")
+                    continue
+
+                logger.info(f"Importing {actor_url=}")
+
+                await _send_follow(db_session, actor_url)
+                count += 1
+
+            await db_session.commit()
+
+        return count
+
+    count = asyncio.run(_import_following())
+    logger.info(f"Import done, {count} follow requests sent")

tests/test_actor.py

@@ -20,12 +20,16 @@ async def test_fetch_actor(async_db_session: AsyncSession, respx_mock) -> None:
         public_key="pk",
     )
     respx_mock.get(ra.ap_id).mock(return_value=httpx.Response(200, json=ra.ap_actor))
+    respx_mock.get(
+        "https://example.com/.well-known/webfinger",
+        params={"resource": "acct%3Atoto%40example.com"},
+    ).mock(return_value=httpx.Response(200, json={"subject": "acct:toto@example.com"}))
 
     # When fetching this actor for the first time
     saved_actor = await fetch_actor(async_db_session, ra.ap_id)
 
     # Then it has been fetched and saved in DB
-    assert respx.calls.call_count == 1
+    assert respx.calls.call_count == 2
     assert (
         await async_db_session.execute(select(models.Actor))
     ).scalar_one().ap_id == saved_actor.ap_id
@@ -38,7 +42,7 @@ async def test_fetch_actor(async_db_session: AsyncSession, respx_mock) -> None:
     assert (
         await async_db_session.execute(select(func.count(models.Actor.id)))
     ).scalar_one() == 1
-    assert respx.calls.call_count == 1
+    assert respx.calls.call_count == 2
 
 
 def test_sqlalchemy_factory(db: Session) -> None:

tests/test_utils.py

@@ -0,0 +1,19 @@
+from unittest import mock
+
+import pytest
+
+from app.utils.url import is_hostname_blocked
+
+
+@pytest.mark.parametrize(
+    "hostname,should_be_blocked",
+    [
+        ("example.com", True),
+        ("subdomain.example.com", True),
+        ("example.xyz", False),
+    ],
+)
+def test_is_hostname_blocked(hostname: str, should_be_blocked: bool) -> None:
+    with mock.patch("app.utils.url.BLOCKED_SERVERS", ["example.com"]):
+        is_hostname_blocked.cache_clear()
+        assert is_hostname_blocked(hostname) is should_be_blocked