Remove data/_theme.scss from git history

- Remove data/_theme.scss from git history, to prevent accidental overwrites/commits
- Add a step to the configuration wizard to create a blank file if one doesn't exist
- Move the file existence check in compile-scss to before favicon building (as it errors if the file is missing)

Fixes https://todo.sr.ht/~tsileo/microblog.pub/67

AUTHORS

@@ -0,0 +1,8 @@
+Thomas Sileo <t@a4.io>
+Kevin Wallace <doof@doof.net>
+Miguel Jacq <mig@mig5.net>
+Josh Washburne <josh@jodh.us>
+Alexey Shpakovsky <alexey@shpakovsky.ru>
+Ash McAllan <acegiak@gmail.com>
+Cassio Zen <cassio@hey.com>
+Cocoa <momijizukamori@gmail.com>

README.md

@@ -58,7 +58,7 @@ All the development takes place on [sourcehut](https://sr.ht/~tsileo/microblog.p
  - [Issue tracker](https://todo.sr.ht/~tsileo/microblog.pub)
  - [Mailing list](https://sr.ht/~tsileo/microblog.pub/lists)
 
-Contributions are welcomed, check out the [documentation](https://docs.microblog.pub) for more details.
+Contributions are welcomed, check out the [contributing section of the documentation](https://docs.microblog.pub/developer_guide.html#contributing) for more details.
 
 
 ## License

alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py

@@ -0,0 +1,32 @@
+"""Add Webmention.webmention_type
+
+Revision ID: fadfd359ce78
+Revises: b28c0551c236
+Create Date: 2022-11-16 19:42:56.925512+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'fadfd359ce78'
+down_revision = 'b28c0551c236'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('webmention_type', sa.Enum('UNKNOWN', 'LIKE', 'REPLY', 'REPOST', name='webmentiontype'), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.drop_column('webmention_type')
+
+    # ### end Alembic commands ###

app/activitypub.py

@@ -135,11 +135,6 @@ ME = {
     "url": config.ID + "/",  # XXX: the path is important for Mastodon compat
     "manuallyApprovesFollowers": config.CONFIG.manually_approves_followers,
     "attachment": _LOCAL_ACTOR_METADATA,
-    "icon": {
-        "mediaType": mimetypes.guess_type(config.CONFIG.icon_url)[0],
-        "type": "Image",
-        "url": config.CONFIG.icon_url,
-    },
     "publicKey": {
         "id": f"{config.ID}#main-key",
         "owner": config.ID,
@@ -148,6 +143,13 @@ ME = {
     "tag": dedup_tags(_LOCAL_ACTOR_TAGS),
 }
 
+if config.CONFIG.icon_url:
+    ME["icon"] = {
+        "mediaType": mimetypes.guess_type(config.CONFIG.icon_url)[0],
+        "type": "Image",
+        "url": config.CONFIG.icon_url,
+    }
+
 if ALSO_KNOWN_AS:
     ME["alsoKnownAs"] = [ALSO_KNOWN_AS]
 

app/actor.py

@@ -12,6 +12,7 @@ from sqlalchemy.orm import joinedload
 
 from app import activitypub as ap
 from app import media
+from app.config import BASE_URL
 from app.database import AsyncSession
 from app.utils.datetime import as_utc
 from app.utils.datetime import now
@@ -111,14 +112,14 @@ class Actor:
         if self.icon_url:
             return media.proxied_media_url(self.icon_url)
         else:
-            return "/static/nopic.png"
+            return BASE_URL + "/static/nopic.png"
 
     @property
     def resized_icon_url(self) -> str:
         if self.icon_url:
             return media.resized_media_url(self.icon_url, 50)
         else:
-            return "/static/nopic.png"
+            return BASE_URL + "/static/nopic.png"
 
     @property
     def tags(self) -> list[ap.RawObject]:

app/admin.py

@@ -11,6 +11,7 @@ from fastapi.exceptions import HTTPException
 from fastapi.responses import RedirectResponse
 from loguru import logger
 from sqlalchemy import and_
+from sqlalchemy import delete
 from sqlalchemy import func
 from sqlalchemy import or_
 from sqlalchemy import select
@@ -29,6 +30,7 @@ from app.boxes import send_block
 from app.boxes import send_follow
 from app.boxes import send_unblock
 from app.config import EMOJIS
+from app.config import SESSION_TIMEOUT
 from app.config import generate_csrf_token
 from app.config import session_serializer
 from app.config import verify_csrf_token
@@ -61,14 +63,17 @@ async def user_session_or_redirect(
     )
 
     if not session:
+        logger.info("No existing admin session")
         raise _RedirectToLoginPage
 
     try:
-        loaded_session = session_serializer.loads(session, max_age=3600 * 12)
+        loaded_session = session_serializer.loads(session, max_age=SESSION_TIMEOUT)
     except Exception:
+        logger.exception("Failed to validate admin session")
         raise _RedirectToLoginPage
 
     if not loaded_session.get("is_logged_in"):
+        logger.info(f"Admin session invalidated: {loaded_session}")
         raise _RedirectToLoginPage
 
     return None
@@ -439,6 +444,7 @@ async def admin_direct_messages(
                     models.InboxObject.ap_context.is_not(None),
                     # Skip transient object like poll relies
                     models.InboxObject.is_transient.is_(False),
+                    models.InboxObject.is_deleted.is_(False),
                 )
                 .group_by(models.InboxObject.ap_context, models.InboxObject.actor_id)
             )
@@ -461,6 +467,7 @@ async def admin_direct_messages(
                     models.OutboxObject.ap_context.is_not(None),
                     # Skip transient object like poll relies
                     models.OutboxObject.is_transient.is_(False),
+                    models.OutboxObject.is_deleted.is_(False),
                 )
                 .group_by(models.OutboxObject.ap_context)
             )
@@ -716,13 +723,9 @@ async def get_notifications(
     actors_metadata = await get_actors_metadata(
         db_session, [notif.actor for notif in notifications if notif.actor]
     )
-
-    for notif in notifications:
-        notif.is_new = False
-    await db_session.commit()
-
     more_unread_count = 0
     next_cursor = None
+
     if notifications and remaining_count > page_size:
         decoded_next_cursor = notifications[-1].created_at
         next_cursor = pagination.encode_cursor(decoded_next_cursor)
@@ -736,7 +739,8 @@ async def get_notifications(
             )
         )
 
-    return await templates.render_template(
+    # Render the template before we change the new flag on notifications
+    tpl_resp = await templates.render_template(
         db_session,
         request,
         "notifications.html",
@@ -748,6 +752,13 @@ async def get_notifications(
         },
     )
 
+    if len({notif.id for notif in notifications if notif.is_new}):
+        for notif in notifications:
+            notif.is_new = False
+        await db_session.commit()
+
+    return tpl_resp
+
 
 @router.get("/object")
 async def admin_object(
@@ -874,6 +885,42 @@ async def admin_actions_force_delete(
     return RedirectResponse(redirect_url, status_code=302)
 
 
+@router.post("/actions/force_delete_webmention")
+async def admin_actions_force_delete_webmention(
+    request: Request,
+    webmention_id: int = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    webmention = await boxes.get_webmention_by_id(db_session, webmention_id)
+    if not webmention:
+        raise ValueError(f"Cannot find {webmention_id}")
+    if not webmention.outbox_object:
+        raise ValueError(f"Missing related outbox object for {webmention_id}")
+
+    # TODO: move this
+    logger.info(f"Deleting {webmention_id}")
+    webmention.is_deleted = True
+    await db_session.flush()
+    from app.webmentions import _handle_webmention_side_effects
+
+    await _handle_webmention_side_effects(
+        db_session, webmention, webmention.outbox_object
+    )
+    # Delete related notifications
+    notif_deletion_result = await db_session.execute(
+        delete(models.Notification)
+        .where(models.Notification.webmention_id == webmention.id)
+        .execution_options(synchronize_session=False)
+    )
+    logger.info(
+        f"Deleted {notif_deletion_result.rowcount} notifications"  # type: ignore
+    )
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
 @router.post("/actions/follow")
 async def admin_actions_follow(
     request: Request,

app/boxes.py

@@ -1,4 +1,5 @@
 """Actions related to the AP inbox/outbox."""
+import datetime
 import uuid
 from collections import defaultdict
 from dataclasses import dataclass
@@ -31,6 +32,8 @@ from app.config import BLOCKED_SERVERS
 from app.config import ID
 from app.config import MANUALLY_APPROVES_FOLLOWERS
 from app.config import set_moved_to
+from app.config import stream_visibility_callback
+from app.customization import ObjectInfo
 from app.database import AsyncSession
 from app.outgoing_activities import new_outgoing_activity
 from app.source import dedup_tags
@@ -41,6 +44,7 @@ from app.utils import webmentions
 from app.utils.datetime import as_utc
 from app.utils.datetime import now
 from app.utils.datetime import parse_isoformat
+from app.utils.facepile import WebmentionReply
 from app.utils.text import slugify
 
 AnyboxObject = models.InboxObject | models.OutboxObject
@@ -201,7 +205,7 @@ async def send_delete(db_session: AsyncSession, ap_object_id: str) -> None:
         raise ValueError("Should never happen")
 
     outbox_object_to_delete.is_deleted = True
-    await db_session.commit()
+    await db_session.flush()
 
     # Compute the original recipients
     recipients = await _compute_recipients(
@@ -216,14 +220,17 @@ async def send_delete(db_session: AsyncSession, ap_object_id: str) -> None:
             db_session, outbox_object_to_delete.in_reply_to
         )
         if replied_object:
-            new_replies_count = await _get_replies_count(
+            if replied_object.is_from_outbox:
-                db_session, replied_object.ap_id
+                # Different helper here because we also count webmentions
-            )
+                new_replies_count = await _get_outbox_replies_count(
+                    db_session, replied_object  # type: ignore
+                )
+            else:
+                new_replies_count = await _get_replies_count(
+                    db_session, replied_object.ap_id
+                )
 
             replied_object.replies_count = new_replies_count
-            if replied_object.replies_count < 0:
-                logger.warning("negative replies count for {replied_object.ap_id}")
-                replied_object.replies_count = 0
         else:
             logger.info(f"{outbox_object_to_delete.in_reply_to} not found")
 
@@ -1048,6 +1055,32 @@ async def get_outbox_object_by_ap_id(
     )  # type: ignore
 
 
+async def get_outbox_object_by_slug_and_short_id(
+    db_session: AsyncSession,
+    slug: str,
+    short_id: str,
+) -> models.OutboxObject | None:
+    return (
+        (
+            await db_session.execute(
+                select(models.OutboxObject)
+                .options(
+                    joinedload(models.OutboxObject.outbox_object_attachments).options(
+                        joinedload(models.OutboxObjectAttachment.upload)
+                    )
+                )
+                .where(
+                    models.OutboxObject.public_id.like(f"{short_id}%"),
+                    models.OutboxObject.slug == slug,
+                    models.OutboxObject.is_deleted.is_(False),
+                )
+            )
+        )
+        .unique()
+        .scalar_one_or_none()
+    )
+
+
 async def get_anybox_object_by_ap_id(
     db_session: AsyncSession, ap_id: str
 ) -> AnyboxObject | None:
@@ -1057,6 +1090,20 @@ async def get_anybox_object_by_ap_id(
         return await get_inbox_object_by_ap_id(db_session, ap_id)
 
 
+async def get_webmention_by_id(
+    db_session: AsyncSession, webmention_id: int
+) -> models.Webmention | None:
+    return (
+        await db_session.execute(
+            select(models.Webmention)
+            .where(models.Webmention.id == webmention_id)
+            .options(
+                joinedload(models.Webmention.outbox_object),
+            )
+        )
+    ).scalar_one_or_none()  # type: ignore
+
+
 async def _handle_delete_activity(
     db_session: AsyncSession,
     from_actor: models.Actor,
@@ -1124,6 +1171,23 @@ async def _handle_delete_activity(
             logger.info("Removing actor from follower")
             await db_session.delete(follower)
 
+            # Also mark Follow activities for this actor as deleted
+            follow_activities = (
+                await db_session.scalars(
+                    select(models.OutboxObject).where(
+                        models.OutboxObject.ap_type == "Follow",
+                        models.OutboxObject.relates_to_actor_id
+                        == ap_object_to_delete.id,
+                        models.OutboxObject.is_deleted.is_(False),
+                    )
+                )
+            ).all()
+            for follow_activity in follow_activities:
+                logger.info(
+                    f"Marking Follow activity {follow_activity.ap_id} as deleted"
+                )
+                follow_activity.is_deleted = True
+
         following = (
             await db_session.scalars(
                 select(models.Following).where(
@@ -1184,6 +1248,67 @@ async def _get_replies_count(
     )
 
 
+async def _get_outbox_replies_count(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> int:
+    return (await _get_replies_count(db_session, outbox_object.ap_id)) + (
+        await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.REPLY,
+            )
+        )
+    )
+
+
+async def _get_outbox_likes_count(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> int:
+    return (
+        await db_session.scalar(
+            select(func.count(models.InboxObject.id)).where(
+                models.InboxObject.ap_type == "Like",
+                models.InboxObject.relates_to_outbox_object_id == outbox_object.id,
+                models.InboxObject.is_deleted.is_(False),
+            )
+        )
+    ) + (
+        await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.LIKE,
+            )
+        )
+    )
+
+
+async def _get_outbox_announces_count(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> int:
+    return (
+        await db_session.scalar(
+            select(func.count(models.InboxObject.id)).where(
+                models.InboxObject.ap_type == "Announce",
+                models.InboxObject.relates_to_outbox_object_id == outbox_object.id,
+                models.InboxObject.is_deleted.is_(False),
+            )
+        )
+    ) + (
+        await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.REPOST,
+            )
+        )
+    )
+
+
 async def _revert_side_effect_for_deleted_object(
     db_session: AsyncSession,
     delete_activity: models.InboxObject | None,
@@ -1214,8 +1339,8 @@ async def _revert_side_effect_for_deleted_object(
                 # also needs to be forwarded
                 is_delete_needs_to_be_forwarded = True
 
-                new_replies_count = await _get_replies_count(
+                new_replies_count = await _get_outbox_replies_count(
-                    db_session, replied_object.ap_id
+                    db_session, replied_object  # type: ignore
                 )
 
                 await db_session.execute(
@@ -1245,12 +1370,13 @@ async def _revert_side_effect_for_deleted_object(
         )
         if related_object:
             if related_object.is_from_outbox:
+                likes_count = await _get_outbox_likes_count(db_session, related_object)
                 await db_session.execute(
                     update(models.OutboxObject)
                     .where(
                         models.OutboxObject.id == related_object.id,
                     )
-                    .values(likes_count=models.OutboxObject.likes_count - 1)
+                    .values(likes_count=likes_count - 1)
                 )
     elif (
         deleted_ap_object.ap_type == "Annouce"
@@ -1262,12 +1388,15 @@ async def _revert_side_effect_for_deleted_object(
         )
         if related_object:
             if related_object.is_from_outbox:
+                announces_count = await _get_outbox_announces_count(
+                    db_session, related_object
+                )
                 await db_session.execute(
                     update(models.OutboxObject)
                     .where(
                         models.OutboxObject.id == related_object.id,
                     )
-                    .values(announces_count=models.OutboxObject.announces_count - 1)
+                    .values(announces_count=announces_count - 1)
                 )
 
     # Delete any Like/Announce
@@ -1754,16 +1883,30 @@ async def _process_note_object(
 
     is_from_following = ro.actor.ap_id in {f.ap_actor_id for f in following}
     is_reply = bool(ro.in_reply_to)
-    is_local_reply = (
+    is_local_reply = bool(
         ro.in_reply_to
         and ro.in_reply_to.startswith(BASE_URL)
         and ro.content  # Hide votes from Question
     )
     is_mention = False
+    hashtags = []
     tags = ro.ap_object.get("tag", [])
     for tag in ap.as_list(tags):
         if tag.get("name") == LOCAL_ACTOR.handle or tag.get("href") == LOCAL_ACTOR.url:
             is_mention = True
+        if tag.get("type") == "Hashtag":
+            if tag_name := tag.get("name"):
+                hashtags.append(tag_name)
+
+    object_info = ObjectInfo(
+        is_reply=is_reply,
+        is_local_reply=is_local_reply,
+        is_mention=is_mention,
+        is_from_following=is_from_following,
+        hashtags=hashtags,
+        actor_handle=ro.actor.handle,
+        remote_object=ro,
+    )
 
     inbox_object = models.InboxObject(
         server=urlparse(ro.ap_id).hostname,
@@ -1781,9 +1924,7 @@ async def _process_note_object(
         activity_object_ap_id=ro.activity_object_ap_id,
         og_meta=await opengraph.og_meta_from_note(db_session, ro),
         # Hide replies from the stream
-        is_hidden_from_stream=not (
+        is_hidden_from_stream=not stream_visibility_callback(object_info),
-            (not is_reply and is_from_following) or is_mention or is_local_reply
-        ),
         # We may already have some replies in DB
         replies_count=await _get_replies_count(db_session, ro.ap_id),
     )
@@ -1809,8 +1950,8 @@ async def _process_note_object(
                         replied_object,  # type: ignore  # outbox check below
                     )
                 else:
-                    new_replies_count = await _get_replies_count(
+                    new_replies_count = await _get_outbox_replies_count(
-                        db_session, replied_object.ap_id
+                        db_session, replied_object  # type: ignore
                     )
 
                     await db_session.execute(
@@ -2056,7 +2197,10 @@ async def _handle_like_activity(
         )
         await db_session.delete(like_activity)
     else:
-        relates_to_outbox_object.likes_count = models.OutboxObject.likes_count + 1
+        relates_to_outbox_object.likes_count = await _get_outbox_likes_count(
+            db_session,
+            relates_to_outbox_object,
+        )
 
         notif = models.Notification(
             notification_type=models.NotificationType.LIKE,
@@ -2467,11 +2611,21 @@ async def fetch_actor_collection(db_session: AsyncSession, url: str) -> list[Act
 
 @dataclass
 class ReplyTreeNode:
-    ap_object: AnyboxObject
+    ap_object: AnyboxObject | None
+    wm_reply: WebmentionReply | None
     children: list["ReplyTreeNode"]
     is_requested: bool = False
     is_root: bool = False
 
+    @property
+    def published_at(self) -> datetime.datetime:
+        if self.ap_object:
+            return self.ap_object.ap_published_at  # type: ignore
+        elif self.wm_reply:
+            return self.wm_reply.published_at
+        else:
+            raise ValueError(f"Should never happen: {self}")
+
 
 async def get_replies_tree(
     db_session: AsyncSession,
@@ -2545,6 +2699,7 @@ async def get_replies_tree(
         for child in index.get(node.ap_object.ap_id, []):  # type: ignore
             child_node = ReplyTreeNode(
                 ap_object=child,
+                wm_reply=None,
                 is_requested=child.ap_id == requested_object.ap_id,  # type: ignore
                 children=[],
             )
@@ -2553,7 +2708,7 @@ async def get_replies_tree(
 
         return sorted(
             children,
-            key=lambda node: node.ap_object.ap_published_at,  # type: ignore
+            key=lambda node: node.published_at,
         )
 
     if None in nodes_by_in_reply_to:
@@ -2566,6 +2721,7 @@ async def get_replies_tree(
 
     root_node = ReplyTreeNode(
         ap_object=root_ap_object,
+        wm_reply=None,
         is_root=True,
         is_requested=root_ap_object.ap_id == requested_object.ap_id,
         children=[],

app/config.py

@@ -16,6 +16,8 @@ from loguru import logger
 from mistletoe import markdown  # type: ignore
 
 from app.customization import _CUSTOM_ROUTES
+from app.customization import _StreamVisibilityCallback
+from app.customization import default_stream_visibility_callback
 from app.utils.emoji import _load_emojis
 from app.utils.version import get_version_commit
 
@@ -91,7 +93,7 @@ class Config(pydantic.BaseModel):
     name: str
     summary: str
     https: bool
-    icon_url: str
+    icon_url: str | None = None
     image_url: str | None = None
     secret: str
     debug: bool = False
@@ -116,6 +118,8 @@ class Config(pydantic.BaseModel):
     sqlalchemy_database: str | None = None
     key_path: str | None = None
 
+    session_timeout: int = 3600 * 24 * 3  # in seconds, 3 days by default
+
     # Only set when the app is served on a non-root path
     id: str | None = None
 
@@ -171,6 +175,7 @@ ALSO_KNOWN_AS = CONFIG.also_known_as
 CUSTOM_CONTENT_SECURITY_POLICY = CONFIG.custom_content_security_policy
 
 INBOX_RETENTION_DAYS = CONFIG.inbox_retention_days
+SESSION_TIMEOUT = CONFIG.session_timeout
 CUSTOM_FOOTER = (
     markdown(CONFIG.custom_footer.replace("{version}", VERSION))
     if CONFIG.custom_footer
@@ -257,5 +262,16 @@ def verify_csrf_token(
     return None
 
 
-def hmac_sha256():
+def hmac_sha256() -> hmac.HMAC:
     return hmac.new(CONFIG.secret.encode(), digestmod=hashlib.sha256)
+
+
+stream_visibility_callback: _StreamVisibilityCallback
+try:
+    from data.stream import (  # type: ignore  # noqa: F401, E501
+        custom_stream_visibility_callback,
+    )
+
+    stream_visibility_callback = custom_stream_visibility_callback
+except ImportError:
+    stream_visibility_callback = default_stream_visibility_callback

app/customization.py

@@ -1,12 +1,19 @@
+from dataclasses import dataclass
 from pathlib import Path
+from typing import TYPE_CHECKING
 from typing import Any
 from typing import Callable
 
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import Request
+from loguru import logger
 from starlette.responses import JSONResponse
 
+if TYPE_CHECKING:
+    from app.ap_object import RemoteObject
+
+
 _DATA_DIR = Path().parent.resolve() / "data"
 _Handler = Callable[..., Any]
 
@@ -110,3 +117,38 @@ def get_custom_router() -> APIRouter | None:
             router.add_api_route(path, handler.handler)
 
     return router
+
+
+@dataclass
+class ObjectInfo:
+    # Is it a reply?
+    is_reply: bool
+
+    # Is it a reply to an outbox object
+    is_local_reply: bool
+
+    # Is the object mentioning the local actor
+    is_mention: bool
+
+    # Is it from someone the local actor is following
+    is_from_following: bool
+
+    # List of hashtags, e.g. #microblogpub
+    hashtags: list[str]
+
+    # @dev@microblog.pub
+    actor_handle: str
+
+    remote_object: "RemoteObject"
+
+
+_StreamVisibilityCallback = Callable[[ObjectInfo], bool]
+
+
+def default_stream_visibility_callback(object_info: ObjectInfo) -> bool:
+    logger.info(f"{object_info=}")
+    return (
+        (not object_info.is_reply and object_info.is_from_following)
+        or object_info.is_mention
+        or object_info.is_local_reply
+    )

app/httpsig.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+import json
 import typing
 from dataclasses import dataclass
 from datetime import datetime
@@ -198,6 +199,32 @@ async def httpsig_checker(
             server=server,
         )
 
+    # Try to drop Delete activity spams early on, this prevent making an extra
+    # HTTP requests trying to fetch an unavailable actor to verify the HTTP sig
+    try:
+        if request.method == "POST" and request.url.path.endswith("/inbox"):
+            from app import models  # TODO: solve this circular import
+
+            activity = json.loads(body)
+            actor_id = ap.get_id(activity["actor"])
+            if (
+                ap.as_list(activity["type"])[0] == "Delete"
+                and actor_id == ap.get_id(activity["object"])
+                and not (
+                    await db_session.scalars(
+                        select(models.Actor).where(
+                            models.Actor.ap_id == actor_id,
+                        )
+                    )
+                ).one_or_none()
+            ):
+                logger.info(f"Dropping Delete activity early for {body=}")
+                raise fastapi.HTTPException(status_code=202)
+    except fastapi.HTTPException as http_exc:
+        raise http_exc
+    except Exception:
+        logger.exception("Failed to check for Delete spam")
+
     # logger.debug(f"hsig={hsig}")
     signed_string, signature_date = _build_signed_string(
         hsig["headers"],

app/main.py

@@ -73,6 +73,9 @@ from app.templates import is_current_user_admin
 from app.uploads import UPLOAD_DIR
 from app.utils import pagination
 from app.utils.emoji import EMOJIS_BY_NAME
+from app.utils.facepile import Face
+from app.utils.facepile import WebmentionReply
+from app.utils.facepile import merge_faces
 from app.utils.highlight import HIGHLIGHT_CSS_HASH
 from app.utils.url import check_url
 from app.webfinger import get_remote_follow_template
@@ -724,7 +727,7 @@ async def _fetch_webmentions(
                 models.Webmention.outbox_object_id == outbox_object.id,
                 models.Webmention.is_deleted.is_(False),
             )
-            .limit(10)
+            .limit(50)
         )
     ).all()
 
@@ -774,23 +777,90 @@ async def outbox_by_public_id(
         is_current_user_admin=is_current_user_admin(request),
     )
 
+    webmentions = await _fetch_webmentions(db_session, maybe_object)
     likes = await _fetch_likes(db_session, maybe_object)
     shares = await _fetch_shares(db_session, maybe_object)
-    webmentions = await _fetch_webmentions(db_session, maybe_object)
     return await templates.render_template(
         db_session,
         request,
         "object.html",
         {
-            "replies_tree": replies_tree,
+            "replies_tree": _merge_replies(replies_tree, webmentions),
             "outbox_object": maybe_object,
-            "likes": likes,
+            "likes": _merge_faces_from_inbox_object_and_webmentions(
-            "shares": shares,
+                likes,
-            "webmentions": webmentions,
+                webmentions,
+                models.WebmentionType.LIKE,
+            ),
+            "shares": _merge_faces_from_inbox_object_and_webmentions(
+                shares,
+                webmentions,
+                models.WebmentionType.REPOST,
+            ),
+            "webmentions": _filter_webmentions(webmentions),
         },
     )
 
 
+def _filter_webmentions(
+    webmentions: list[models.Webmention],
+) -> list[models.Webmention]:
+    return [
+        wm
+        for wm in webmentions
+        if wm.webmention_type
+        not in [
+            models.WebmentionType.LIKE,
+            models.WebmentionType.REPOST,
+            models.WebmentionType.REPLY,
+        ]
+    ]
+
+
+def _merge_faces_from_inbox_object_and_webmentions(
+    inbox_objects: list[models.InboxObject],
+    webmentions: list[models.Webmention],
+    webmention_type: models.WebmentionType,
+) -> list[Face]:
+    wm_faces = []
+    for wm in webmentions:
+        if wm.webmention_type != webmention_type:
+            continue
+        if face := Face.from_webmention(wm):
+            wm_faces.append(face)
+
+    return merge_faces(
+        [Face.from_inbox_object(obj) for obj in inbox_objects] + wm_faces
+    )
+
+
+def _merge_replies(
+    reply_tree_node: boxes.ReplyTreeNode,
+    webmentions: list[models.Webmention],
+) -> boxes.ReplyTreeNode:
+    # TODO: return None as we update the object in place
+    webmention_replies = []
+    for wm in [
+        wm for wm in webmentions if wm.webmention_type == models.WebmentionType.REPLY
+    ]:
+        if rep := WebmentionReply.from_webmention(wm):
+            webmention_replies.append(
+                boxes.ReplyTreeNode(
+                    ap_object=None,
+                    wm_reply=rep,
+                    is_requested=False,
+                    children=[],
+                )
+            )
+
+    reply_tree_node.children = sorted(
+        reply_tree_node.children + webmention_replies,
+        key=lambda node: node.published_at,
+        reverse=True,
+    )
+    return reply_tree_node
+
+
 @app.get("/articles/{short_id}/{slug}")
 async def article_by_slug(
     short_id: str,
@@ -799,24 +869,8 @@ async def article_by_slug(
     db_session: AsyncSession = Depends(get_db_session),
     httpsig_info: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse | RedirectResponse:
-    maybe_object = (
+    maybe_object = await boxes.get_outbox_object_by_slug_and_short_id(
-        (
+        db_session, slug, short_id
-            await db_session.execute(
-                select(models.OutboxObject)
-                .options(
-                    joinedload(models.OutboxObject.outbox_object_attachments).options(
-                        joinedload(models.OutboxObjectAttachment.upload)
-                    )
-                )
-                .where(
-                    models.OutboxObject.public_id.like(f"{short_id}%"),
-                    models.OutboxObject.slug == slug,
-                    models.OutboxObject.is_deleted.is_(False),
-                )
-            )
-        )
-        .unique()
-        .scalar_one_or_none()
     )
     if not maybe_object:
         raise HTTPException(status_code=404)
@@ -840,11 +894,19 @@ async def article_by_slug(
         request,
         "object.html",
         {
-            "replies_tree": replies_tree,
+            "replies_tree": _merge_replies(replies_tree, webmentions),
             "outbox_object": maybe_object,
-            "likes": likes,
+            "likes": _merge_faces_from_inbox_object_and_webmentions(
-            "shares": shares,
+                likes,
-            "webmentions": webmentions,
+                webmentions,
+                models.WebmentionType.LIKE,
+            ),
+            "shares": _merge_faces_from_inbox_object_and_webmentions(
+                shares,
+                webmentions,
+                models.WebmentionType.REPOST,
+            ),
+            "webmentions": _filter_webmentions(webmentions),
         },
     )
 
@@ -1118,7 +1180,11 @@ async def nodeinfo(
     )
 
 
-proxy_client = httpx.AsyncClient(follow_redirects=True, http2=True)
+proxy_client = httpx.AsyncClient(
+    http2=True,
+    follow_redirects=True,
+    timeout=httpx.Timeout(timeout=10.0),
+)
 
 
 async def _proxy_get(
@@ -1398,7 +1464,7 @@ async def json_feed(
                 ],
             }
         )
-    return {
+    result = {
         "version": "https://jsonfeed.org/version/1",
         "title": f"{LOCAL_ACTOR.display_name}'s microblog'",
         "home_page_url": LOCAL_ACTOR.url,
@@ -1406,10 +1472,12 @@ async def json_feed(
         "author": {
             "name": LOCAL_ACTOR.display_name,
             "url": LOCAL_ACTOR.url,
-            "avatar": LOCAL_ACTOR.icon_url,
         },
         "items": data,
     }
+    if LOCAL_ACTOR.icon_url:
+        result["author"]["avatar"] = LOCAL_ACTOR.icon_url  # type: ignore
+    return result
 
 
 async def _gen_rss_feed(
@@ -1421,7 +1489,8 @@ async def _gen_rss_feed(
     fg.description(f"{LOCAL_ACTOR.display_name}'s microblog")
     fg.author({"name": LOCAL_ACTOR.display_name})
     fg.link(href=LOCAL_ACTOR.url, rel="alternate")
-    fg.logo(LOCAL_ACTOR.icon_url)
+    if LOCAL_ACTOR.icon_url:
+        fg.logo(LOCAL_ACTOR.icon_url)
     fg.language("en")
 
     outbox_objects = await _get_outbox_for_feed(db_session)

app/models.py

@@ -468,6 +468,14 @@ class IndieAuthAccessToken(Base):
     is_revoked = Column(Boolean, nullable=False, default=False)
 
 
+@enum.unique
+class WebmentionType(str, enum.Enum):
+    UNKNOWN = "unknown"
+    LIKE = "like"
+    REPLY = "reply"
+    REPOST = "repost"
+
+
 class Webmention(Base):
     __tablename__ = "webmention"
     __table_args__ = (UniqueConstraint("source", "target", name="uix_source_target"),)
@@ -484,6 +492,8 @@ class Webmention(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
     outbox_object = relationship(OutboxObject, uselist=False)
 
+    webmention_type = Column(Enum(WebmentionType), nullable=True)
+
     @property
     def as_facepile_item(self) -> webmentions.Webmention | None:
         if not self.source_microformats:
@@ -493,6 +503,7 @@ class Webmention(Base):
                 self.source_microformats["items"], self.source
             )
         except Exception:
+            # TODO: return a facepile with the unknown image
             logger.warning(
                 f"Failed to generate facefile item for Webmention id={self.id}"
             )

app/scss/main.scss

@@ -467,6 +467,9 @@ a.label-btn {
   span {
     color: $muted-color;
   }
+  span.new {
+    color: $secondary-color;
+  }
 }
 .actor-metadata {
     color: $muted-color;
@@ -541,3 +544,7 @@ a.label-btn {
     content: ': ';
   }
 }
+
+.margin-top-20 {
+  margin-top: 20px;
+}

app/templates.py

@@ -27,6 +27,7 @@ from app.ap_object import Object
 from app.config import BASE_URL
 from app.config import CUSTOM_FOOTER
 from app.config import DEBUG
+from app.config import SESSION_TIMEOUT
 from app.config import VERSION
 from app.config import generate_csrf_token
 from app.config import session_serializer
@@ -69,10 +70,10 @@ def is_current_user_admin(request: Request) -> bool:
         try:
             loaded_session = session_serializer.loads(
                 session_cookie,
-                max_age=3600 * 12,
+                max_age=SESSION_TIMEOUT,
             )
         except Exception:
-            pass
+            logger.exception("Failed to validate session timeout")
         else:
             is_admin = loaded_session.get("is_logged_in")
 
@@ -335,6 +336,14 @@ def _clean_html(html: str, note: Object) -> str:
         raise
 
 
+def _clean_html_wm(html: str) -> str:
+    return bleach.clean(
+        html,
+        attributes=ALLOWED_ATTRIBUTES,
+        strip=True,
+    )
+
+
 def _timeago(original_dt: datetime) -> str:
     dt = original_dt
     if dt.tzinfo:
@@ -411,6 +420,7 @@ def _poll_item_pct(item: ap.RawObject, voters_count: int) -> int:
 _templates.env.filters["domain"] = _filter_domain
 _templates.env.filters["media_proxy_url"] = _media_proxy_url
 _templates.env.filters["clean_html"] = _clean_html
+_templates.env.filters["clean_html_wm"] = _clean_html_wm
 _templates.env.filters["timeago"] = _timeago
 _templates.env.filters["format_date"] = _format_date
 _templates.env.filters["has_media_type"] = _has_media_type

app/templates/followers.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>{{ local_actor.display_name }}'s followers</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/following.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>{{ local_actor.display_name }}'s follows</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/login.html

@@ -1,5 +1,8 @@
 {%- import "utils.html" as utils with context -%}
 {% extends "layout.html" %}
+{% block head %}
+<meta name="robots" content="noindex, nofollow">
+{% endblock %}
 {% block main_tag %} class="main-flex"{% endblock %}
 {% block content %}
 <div class="centered">

app/templates/notifications.html

@@ -10,6 +10,9 @@
         <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.actor.ap_id }}">
             {% if with_icon %}{{ utils.display_tiny_actor_icon(notif.actor) }}{% endif %} {{ notif.actor.display_name | clean_html(notif.actor) | safe }}</a> {{ text }}
         <span title="{{ notif.created_at.isoformat() }}">{{ notif.created_at | timeago }}</span>
+        {% if notif.is_new %}
+            <span class="new">new</span>
+        {% endif %}
     </div>
 {% endmacro %}
 
@@ -66,8 +69,8 @@
                 {{ notif_actor_action(notif, "shared a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
            {% elif notif.notification_type.value == "undo_announce" %}
-                {{ notif_actor_action(notif, "unshared a post") }}
+                {{ notif_actor_action(notif, "unshared a post", with_icon=True) }}
-                {{ utils.display_object(notif.outbox_object, with_icon=True) }}
+                {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "mention" %}
                 {{ notif_actor_action(notif, "mentioned you") }}
                 {{ utils.display_object(notif.inbox_object) }}

app/templates/object.html

@@ -31,9 +31,16 @@
 {% macro display_replies_tree(replies_tree_node) %}
 
 {% if replies_tree_node.is_requested %}
-{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root, is_object_page=True) }}
+{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root, is_object_page=True, is_h_entry=False) }}
 {% else %}
-    {{ utils.display_object(replies_tree_node.ap_object) }}
+    {% if replies_tree_node.wm_reply %}
+        {# u-comment h-cite is displayed by default for webmention #}
+        {{ utils.display_webmention_reply(replies_tree_node.wm_reply) }}
+    {% else %}
+        <div class="u-comment h-cite">
+            {{ utils.display_object(replies_tree_node.ap_object, is_h_entry=False) }}
+        </div>
+    {% endif %}
 {% endif %}
 
 {% for child in replies_tree_node.children %}
@@ -42,6 +49,8 @@
 
 {% endmacro %}
 
+<div class="h-entry">
 {{ display_replies_tree(replies_tree) }}
+</div>
 
 {% endblock %}

app/templates/remote_follow.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>Remote follow {{ local_actor.display_name }}</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/remote_interact.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>Interact from your instance</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/utils.html

@@ -142,6 +142,17 @@
 {% endblock %}
 {% endmacro %}
 
+{% macro admin_force_delete_webmention_button(webmention_id, permalink_id=None) %}
+{% block admin_force_delete_webmention_button scoped %}
+<form action="{{ request.url_for("admin_actions_force_delete_webmention") }}" class="object-delete-form" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url(permalink_id) }}
+    <input type="hidden" name="webmention_id" value="{{ webmention_id }}">
+    <input type="submit" value="local delete">
+</form>
+{% endblock %}
+{% endmacro %}
+
 {% macro admin_announce_button(ap_object_id, permalink_id=None) %}
 {% block admin_announce_button scoped %}
 <form action="{{ request.url_for("admin_actions_announce") }}" method="POST">
@@ -413,7 +424,9 @@
 
     {% if attachment.type == "Image" or (attachment | has_media_type("image")) %}
         {% if attachment.url not in object.inlined_images %}
+        <a class="media-link" href="{{ attachment.proxied_url }}" target="_blank">
             <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment">
+        </a>
         {% endif %}
     {% elif attachment.type == "Video" or (attachment | has_media_type("video")) %}
     <video controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %}></video>
@@ -439,11 +452,55 @@
 {% endblock %}
 {% endmacro %}
 
-{% macro display_object(object, likes=[], shares=[], webmentions=[], expanded=False, actors_metadata={}, is_object_page=False) %}
+{% macro display_webmention_reply(wm_reply) %}
+{% block display_webmention_reply scoped %}
+
+<div class="ap-object u-comment h-cite">
+<div class="actor-box h-card p-author">
+    <div class="icon-box">
+        <img src="{{ wm_reply.face.picture_url }}" alt="{{ wm_reply.face.name }}'s avatar" class="actor-icon u-photo">
+    </div>
+    <a href="{{ wm_reply.face.url }}" class="u-url">
+        <div><strong class="p-name">{{ wm_reply.face.name | clean_html_wm | safe }}</strong></div>
+        <div class="actor-handle">{{ wm_reply.face.url | truncate(64, True) }}</div>
+    </a>
+</div>
+
+    <p class="in-reply-to">in reply to <a href="{{ wm_reply.in_reply_to }}" title="{{ wm_reply.in_reply_to }}" rel="nofollow">
+        this note
+    </a></p>
+
+    <div class="obj-content margin-top-20">
+    <div class="e-content">
+        {{ wm_reply.content | clean_html_wm | safe }}
+    </div>
+    </div>
+
+    <nav class="flexbox activity-bar margin-top-20">
+    <ul>
+        <li>
+            <div><a href="{{ wm_reply.url }}" rel="nofollow" class="object-permalink u-url u-uid">permalink</a></div>
+        </li>
+        <li>
+            <time class="dt-published" datetime="{{ wm_reply.published_at.replace(microsecond=0).isoformat() }}" title="{{ wm_reply.published_at.replace(microsecond=0).isoformat() }}">{{ wm_reply.published_at | timeago }}</time>
+        </li>
+        {% if is_admin %}
+            <li>
+                {{ admin_force_delete_webmention_button(wm_reply.webmention_id) }}
+            </li>
+        {% endif %}
+    </ul>
+    </nav>
+</div>
+
+{% endblock %}
+{% endmacro %}
+
+{% macro display_object(object, likes=[], shares=[], webmentions=[], expanded=False, actors_metadata={}, is_object_page=False, is_h_entry=True) %}
 {% block display_object scoped %}
 {% set is_article_mode = object.is_from_outbox and object.ap_type == "Article" and is_object_page %}
 {% if object.ap_type in ["Note", "Article", "Video", "Page", "Question", "Event"] %}
-<div class="ap-object {% if expanded %}ap-object-expanded {% endif %}h-entry" id="{{ object.permalink_id }}">
+<div class="ap-object {% if expanded %}ap-object-expanded {% endif %}{% if is_h_entry %}h-entry{% endif %}" id="{{ object.permalink_id }}">
 
     {% if is_article_mode %}
     <data class="h-card">
@@ -693,7 +750,7 @@
                 {{ admin_expand_button(object) }}
             </li>
         {% endif %}
-        {% if object.is_from_inbox %}
+        {% if object.is_from_inbox and not object.announced_via_outbox_object_ap_id %}
             <li>
                 {{ admin_force_delete_button(object.ap_id) }}
             </li>
@@ -709,8 +766,8 @@
             <div class="interactions-block">Likes
                 <div class="facepile-wrapper">
                 {% for like in likes %}
-                    <a href="{% if is_admin %}{{ url_for("admin_profile") }}?actor_id={{ like.actor.ap_id }}{% else %}{{ like.actor.url }}{% endif %}" title="{{ like.actor.handle }}" rel="noreferrer">
+                    <a href="{% if is_admin and like.ap_actor_id %}{{ url_for("admin_profile") }}?actor_id={{ like.ap_actor_id }}{% else %}{{ like.url }}{% endif %}" title="{{ like.name }}" rel="noreferrer">
-                        <img src="{{ like.actor.resized_icon_url }}" alt="{{ like.actor.handle}}">
+                        <img src="{{ like.picture_url }}" alt="{{ like.name }}">
                     </a>
                 {% endfor %}
                 {% if object.likes_count > likes | length %}
@@ -726,8 +783,8 @@
             <div class="interactions-block">Shares
                 <div class="facepile-wrapper">
                 {% for share in shares %}
-                    <a href="{% if is_admin %}{{ url_for("admin_profile") }}?actor_id={{ share.actor.ap_id }}{% else %}{{ share.actor.url }}{% endif %}" title="{{ share.actor.handle }}" rel="noreferrer">
+                    <a href="{% if is_admin and share.ap_actor_id %}{{ url_for("admin_profile") }}?actor_id={{ share.ap_actor_id }}{% else %}{{ share.url }}{% endif %}" title="{{ share.name }}" rel="noreferrer">
-                        <img src="{{ share.actor.resized_icon_url }}" alt="{{ share.actor.handle}}">
+                        <img src="{{ share.picture_url }}" alt="{{ share.name }}">
                     </a>
                 {% endfor %}
                 {% if object.announces_count > shares | length %}

app/utils/emoji.py

@@ -23,6 +23,8 @@ def _load_emojis(root_dir: Path, base_url: str) -> None:
             mt = mimetypes.guess_type(emoji.name)[0]
             if mt and mt.startswith("image/"):
                 name = emoji.name.split(".")[0]
+                if not re.match(EMOJI_REGEX, f":{name}:"):
+                    continue
                 ap_emoji: "RawObject" = {
                     "type": "Emoji",
                     "name": f":{name}:",

app/utils/facepile.py

@@ -0,0 +1,159 @@
+import datetime
+from dataclasses import dataclass
+from typing import Any
+from typing import Optional
+
+from loguru import logger
+
+from app import media
+from app.models import InboxObject
+from app.models import Webmention
+from app.utils.datetime import parse_isoformat
+from app.utils.url import make_abs
+
+
+@dataclass
+class Face:
+    ap_actor_id: str | None
+    url: str
+    name: str
+    picture_url: str
+    created_at: datetime.datetime
+
+    @classmethod
+    def from_inbox_object(cls, like: InboxObject) -> "Face":
+        return cls(
+            ap_actor_id=like.actor.ap_id,
+            url=like.actor.url,  # type: ignore
+            name=like.actor.handle,  # type: ignore
+            picture_url=like.actor.resized_icon_url,
+            created_at=like.created_at,  # type: ignore
+        )
+
+    @classmethod
+    def from_webmention(cls, webmention: Webmention) -> Optional["Face"]:
+        items = webmention.source_microformats.get("items", [])  # type: ignore
+        for item in items:
+            if item["type"][0] == "h-card":
+                try:
+                    return cls(
+                        ap_actor_id=None,
+                        url=(
+                            item["properties"]["url"][0]
+                            if item["properties"].get("url")
+                            else webmention.source
+                        ),
+                        name=item["properties"]["name"][0],
+                        picture_url=media.resized_media_url(
+                            make_abs(
+                                item["properties"]["photo"][0], webmention.source
+                            ),  # type: ignore
+                            50,
+                        ),
+                        created_at=webmention.created_at,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+            elif item["type"][0] == "h-entry":
+                author = item["properties"]["author"][0]
+                try:
+                    return cls(
+                        ap_actor_id=None,
+                        url=webmention.source,
+                        name=author["properties"]["name"][0],
+                        picture_url=media.resized_media_url(
+                            make_abs(
+                                author["properties"]["photo"][0], webmention.source
+                            ),  # type: ignore
+                            50,
+                        ),
+                        created_at=webmention.created_at,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+
+        return None
+
+
+def merge_faces(faces: list[Face]) -> list[Face]:
+    return sorted(
+        faces,
+        key=lambda f: f.created_at,
+        reverse=True,
+    )[:10]
+
+
+def _parse_face(webmention: Webmention, items: list[dict[str, Any]]) -> Face | None:
+    for item in items:
+        if item["type"][0] == "h-card":
+            try:
+                return Face(
+                    ap_actor_id=None,
+                    url=(
+                        item["properties"]["url"][0]
+                        if item["properties"].get("url")
+                        else webmention.source
+                    ),
+                    name=item["properties"]["name"][0],
+                    picture_url=media.resized_media_url(
+                        make_abs(
+                            item["properties"]["photo"][0], webmention.source
+                        ),  # type: ignore
+                        50,
+                    ),
+                    created_at=webmention.created_at,  # type: ignore
+                )
+            except Exception:
+                logger.exception(
+                    f"Failed to build Face for webmention id={webmention.id}"
+                )
+                break
+
+    return None
+
+
+@dataclass
+class WebmentionReply:
+    face: Face
+    content: str
+    url: str
+    published_at: datetime.datetime
+    in_reply_to: str
+    webmention_id: int
+
+    @classmethod
+    def from_webmention(cls, webmention: Webmention) -> Optional["WebmentionReply"]:
+        items = webmention.source_microformats.get("items", [])  # type: ignore
+        for item in items:
+            if item["type"][0] == "h-entry":
+                try:
+                    face = _parse_face(webmention, item["properties"].get("author", []))
+                    if not face:
+                        logger.info(
+                            "Failed to build WebmentionReply/Face for "
+                            f"webmention id={webmention.id}"
+                        )
+                        break
+                    return cls(
+                        face=face,
+                        content=item["properties"]["content"][0]["html"],
+                        url=item["properties"]["url"][0],
+                        published_at=parse_isoformat(
+                            item["properties"]["published"][0]
+                        ).replace(tzinfo=None),
+                        in_reply_to=webmention.target,  # type: ignore
+                        webmention_id=webmention.id,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+
+        return None

app/webmentions.py

@@ -1,3 +1,5 @@
+from urllib.parse import urlparse
+
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from fastapi import APIRouter
@@ -6,13 +8,20 @@ from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
 from loguru import logger
+from sqlalchemy import func
 from sqlalchemy import select
 
 from app import models
+from app.boxes import _get_outbox_announces_count
+from app.boxes import _get_outbox_likes_count
+from app.boxes import _get_outbox_replies_count
 from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import get_outbox_object_by_slug_and_short_id
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.utils import microformats
+from app.utils.facepile import Face
+from app.utils.facepile import WebmentionReply
 from app.utils.url import check_url
 from app.utils.url import is_url_valid
 
@@ -47,6 +56,7 @@ async def webmention_endpoint(
 
         check_url(source)
         check_url(target)
+        parsed_target_url = urlparse(target)
     except Exception:
         logger.exception("Invalid webmention request")
         raise HTTPException(status_code=400, detail="Invalid payload")
@@ -65,6 +75,16 @@ async def webmention_endpoint(
         logger.info("Found existing Webmention, will try to update or delete")
 
     mentioned_object = await get_outbox_object_by_ap_id(db_session, target)
+
+    if not mentioned_object and parsed_target_url.path.startswith("/articles/"):
+        try:
+            _, _, short_id, slug = parsed_target_url.path.split("/")
+            mentioned_object = await get_outbox_object_by_slug_and_short_id(
+                db_session, slug, short_id
+            )
+        except Exception:
+            logger.exception(f"Failed to match {target}")
+
     if not mentioned_object:
         logger.info(f"Invalid target {target=}")
 
@@ -90,8 +110,13 @@ async def webmention_endpoint(
         logger.warning(f"target {target=} not found in source")
         if existing_webmention_in_db:
             logger.info("Deleting existing Webmention")
-            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
             existing_webmention_in_db.is_deleted = True
+            await db_session.flush()
+
+            # Revert side effects
+            await _handle_webmention_side_effects(
+                db_session, existing_webmention_in_db, mentioned_object
+            )
 
             notif = models.Notification(
                 notification_type=models.NotificationType.DELETED_WEBMENTION,
@@ -110,10 +135,14 @@ async def webmention_endpoint(
         else:
             return JSONResponse(content={}, status_code=200)
 
+    webmention_type = models.WebmentionType.UNKNOWN
+    webmention: models.Webmention
     if existing_webmention_in_db:
         # Undelete if needed
         existing_webmention_in_db.is_deleted = False
         existing_webmention_in_db.source_microformats = data
+        await db_session.flush()
+        webmention = existing_webmention_in_db
 
         notif = models.Notification(
             notification_type=models.NotificationType.UPDATED_WEBMENTION,
@@ -127,9 +156,11 @@ async def webmention_endpoint(
             target=target,
             source_microformats=data,
             outbox_object_id=mentioned_object.id,
+            webmention_type=webmention_type,
         )
         db_session.add(new_webmention)
         await db_session.flush()
+        webmention = new_webmention
 
         notif = models.Notification(
             notification_type=models.NotificationType.NEW_WEBMENTION,
@@ -138,8 +169,60 @@ async def webmention_endpoint(
         )
         db_session.add(notif)
 
-        mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
+    # Determine the webmention type
+    for item in data.get("items", []):
+        if target in item.get("properties", {}).get(
+            "in-reply-to", []
+        ) and WebmentionReply.from_webmention(webmention):
+            webmention_type = models.WebmentionType.REPLY
+            break
+        elif target in item.get("properties", {}).get(
+            "like-of", []
+        ) and Face.from_webmention(webmention):
+            webmention_type = models.WebmentionType.LIKE
+            break
+        elif target in item.get("properties", {}).get(
+            "repost-of", []
+        ) and Face.from_webmention(webmention):
+            webmention_type = models.WebmentionType.REPOST
+            break
 
+    if webmention_type != models.WebmentionType.UNKNOWN:
+        webmention.webmention_type = webmention_type
+        await db_session.flush()
+
+    # Handle side effect
+    await _handle_webmention_side_effects(db_session, webmention, mentioned_object)
     await db_session.commit()
 
     return JSONResponse(content={}, status_code=200)
+
+
+async def _handle_webmention_side_effects(
+    db_session: AsyncSession,
+    webmention: models.Webmention,
+    mentioned_object: models.OutboxObject,
+) -> None:
+    if webmention.webmention_type == models.WebmentionType.UNKNOWN:
+        # TODO: recount everything
+        mentioned_object.webmentions_count = await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == mentioned_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.UNKNOWN,
+            )
+        )
+    elif webmention.webmention_type == models.WebmentionType.LIKE:
+        mentioned_object.likes_count = await _get_outbox_likes_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPOST:
+        mentioned_object.announces_count = await _get_outbox_announces_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPLY:
+        mentioned_object.replies_count = await _get_outbox_replies_count(
+            db_session, mentioned_object
+        )
+    else:
+        raise ValueError(f"Unhandled {webmention.webmention_type} webmention")

data/_theme.scss

@@ -1 +0,0 @@
-// override vars for theming here

docs/developer_guide.md

@@ -58,3 +58,24 @@ And check out the result by starting a static server using Python standard libra
 cd docs/dist
 python -m http.server 8001
 ```
+
+## Contributing
+
+Contributions/patches are welcome, but please start a discussion in a [ticket](https://todo.sr.ht/~tsileo/microblog.pub) or a [thread in the mailing list](https://lists.sr.ht/~tsileo/microblog.pub-devel) before working on anything consequent.
+
+### Patches
+
+Please ensure your code passes the code quality checks:
+
+```bash
+inv autoformat
+inv lint
+```
+
+And that the tests suite is passing:
+
+```bash
+inv tests
+```
+
+Please also consider adding new test cases if needed.

docs/install.md

@@ -193,4 +193,8 @@ http {
 
 ## YunoHost edition
 
-[YunoHost](https://yunohost.org/) support is a work in progress.
+[YunoHost](https://yunohost.org/) support is available (although it is not an official package for now): <https://git.sr.ht/~tsileo/microblog.pub_ynh>.
+
+## Available tutorial/guides
+
+ - [Opalstack](https://community.opalstack.com/d/1055-howto-install-and-run-microblogpub-on-opalstack), thanks to [@defulmere@mastodon.social](https://mastodon.online/@defulmere).

docs/user_guide.md

@@ -113,6 +113,7 @@ You can copy/paste them from [getemoji.com](https://getemoji.com/).
 #### Custom emoji
 
 You can add custom emoji in the `data/custom_emoji` directory and they will be picked automatically.
+Do not use exotic characters in filename - only letters, numbers, and underscore symbol `_` are allowed.
 
 #### Custom CSS
 
@@ -465,6 +466,7 @@ make self-destruct
 
 If the server is not (re)starting, you can:
 
- - [Ensure that the configuration is valid](/user_guide.html#configuration-checking)
+ - [Ensure that the configuration is valid](/user_guide.html#configuration-checking).
- - [Verify if you haven't any syntax error in the custom theme by recompiling the CSS](/user_guide.html#recompiling-css-files)
+ - [Verify if you haven't any syntax error in the custom theme by recompiling the CSS](/user_guide.html#recompiling-css-files).
- - Look at the log files
+ - Look at the log files (in `data/uvicorn.log`, `data/incoming.log` and `data/outgoing.log`).
+ - If the CSS is not working, ensure your reverse proxy is serving the static file correctly.

scripts/config_wizard.py

@@ -25,6 +25,10 @@ def _(event):
 
 
 def main() -> None:
+    theme_file = Path("data/_theme.scss")
+    if not theme_file.exists():
+        theme_file.write_text("// override vars for theming here")
+
     print("Welcome to microblog.pub setup wizard\n")
     print("Generating key...")
     if _KEY_PATH.exists():
@@ -75,9 +79,10 @@ def main() -> None:
         proto = "http"
 
     print("Note that you can put your icon/avatar in the static/ directory")
-    dat["icon_url"] = prompt(
+    if icon_url := prompt(
         "icon URL: ", default=f'{proto}://{dat["domain"]}/static/nopic.png'
-    )
+    ):
+        dat["icon_url"] = icon_url
     dat["secret"] = os.urandom(16).hex()
 
     with config_file.open("w") as f:

tasks.py

@@ -46,16 +46,16 @@ def compile_scss(ctx, watch=False):
     # type: (Context, bool) -> None
     from app.utils.favicon import build_favicon
 
+    theme_file = Path("data/_theme.scss")
+    if not theme_file.exists():
+        theme_file.write_text("// override vars for theming here")
+
     favicon_file = Path("data/favicon.ico")
     if not favicon_file.exists():
         build_favicon()
     else:
         shutil.copy2(favicon_file, "app/static/favicon.ico")
 
-    theme_file = Path("data/_theme.scss")
-    if not theme_file.exists():
-        theme_file.write_text("// override vars for theming here")
-
     if watch:
         run("boussole watch", echo=True)
     else: