Quick hotfix for retries

Also expand documentation about custom templates

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim as python-base
+FROM python:3.11-slim as python-base
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     POETRY_HOME="/opt/poetry" \

README.md

@@ -10,6 +10,7 @@ Instances in the wild:
  - [microblog.pub](https://microblog.pub/) (follow to get updated about the project)
  - [hexa.ninja](https://hexa.ninja) (theme customization example)
  - [testing.microblog.pub](https://testing.microblog.pub/)
+ - [Irish Left Archive](https://posts.leftarchive.ie/) (another theme customization example)
 
 There are still some rough edges, but the server is mostly functional.
 

app/admin.py

@@ -189,8 +189,11 @@ async def admin_new(
             content += f"{in_reply_to_object.actor.handle} "
         for tag in in_reply_to_object.tags:
             if tag.get("type") == "Mention" and tag["name"] != LOCAL_ACTOR.handle:
-                mentioned_actor = await fetch_actor(db_session, tag["href"])
-                content += f"{mentioned_actor.handle} "
+                try:
+                    mentioned_actor = await fetch_actor(db_session, tag["href"])
+                    content += f"{mentioned_actor.handle} "
+                except Exception:
+                    logger.exception(f"Failed to lookup {mentioned_actor}")
 
         # Copy the content warning if any
         if in_reply_to_object.summary:

app/config.py

@@ -124,6 +124,7 @@ class Config(pydantic.BaseModel):
     key_path: str | None = None
 
     session_timeout: int = 3600 * 24 * 3  # in seconds, 3 days by default
+    csrf_token_exp: int = 3600
 
     disabled_notifications: list[str] = []
 
@@ -263,7 +264,7 @@ def verify_csrf_token(
     if redirect_url:
         please_try_again = f'<a href="{redirect_url}">please try again</a>'
     try:
-        csrf_serializer.loads(csrf_token, max_age=1800)
+        csrf_serializer.loads(csrf_token, max_age=CONFIG.csrf_token_exp)
     except (itsdangerous.BadData, itsdangerous.SignatureExpired):
         logger.exception("Failed to verify CSRF token")
         raise HTTPException(

app/incoming_activities.py

@@ -60,7 +60,7 @@ def _set_next_try(
     if not outgoing_activity.tries:
         raise ValueError("Should never happen")
 
-    if outgoing_activity.tries == _MAX_RETRIES:
+    if outgoing_activity.tries >= _MAX_RETRIES:
         outgoing_activity.is_errored = True
         outgoing_activity.next_try = None
     else:

app/indieauth.py

@@ -10,6 +10,8 @@ from fastapi import Form
 from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
+from fastapi.security import HTTPBasic
+from fastapi.security import HTTPBasicCredentials
 from loguru import logger
 from pydantic import BaseModel
 from sqlalchemy import select
@@ -26,6 +28,8 @@ from app.redirect import redirect
 from app.utils import indieauth
 from app.utils.datetime import now
 
+basic_auth = HTTPBasic()
+
 router = APIRouter()
 
 
@@ -41,6 +45,7 @@ async def well_known_authorization_server(
         "revocation_endpoint": request.url_for("indieauth_revocation_endpoint"),
         "revocation_endpoint_auth_methods_supported": ["none"],
         "registration_endpoint": request.url_for("oauth_registration_endpoint"),
+        "introspection_endpoint": request.url_for("oauth_introspection_endpoint"),
     }
 
 
@@ -378,6 +383,8 @@ async def _check_access_token(
 class AccessTokenInfo:
     scopes: list[str]
     client_id: str | None
+    access_token: str
+    exp: int
 
 
 async def verify_access_token(
@@ -409,6 +416,13 @@ async def verify_access_token(
             if access_token.indieauth_authorization_request
             else None
         ),
+        access_token=access_token.access_token,
+        exp=int(
+            (
+                access_token.created_at.replace(tzinfo=timezone.utc)
+                + timedelta(seconds=access_token.expires_in)
+            ).timestamp()
+        ),
     )
 
 
@@ -434,6 +448,13 @@ async def check_access_token(
             if access_token.indieauth_authorization_request
             else None
         ),
+        access_token=access_token.access_token,
+        exp=int(
+            (
+                access_token.created_at.replace(tzinfo=timezone.utc)
+                + timedelta(seconds=access_token.expires_in)
+            ).timestamp()
+        ),
     )
 
     logger.info(
@@ -474,3 +495,58 @@ async def indieauth_revocation_endpoint(
         content={},
         status_code=200,
     )
+
+
+@router.post("/token_introspection")
+async def oauth_introspection_endpoint(
+    request: Request,
+    credentials: HTTPBasicCredentials = Depends(basic_auth),
+    db_session: AsyncSession = Depends(get_db_session),
+    token: str = Form(),
+) -> JSONResponse:
+    registered_client = (
+        await db_session.scalars(
+            select(models.OAuthClient).where(
+                models.OAuthClient.client_id == credentials.username,
+                models.OAuthClient.client_secret == credentials.password,
+            )
+        )
+    ).one_or_none()
+    if not registered_client:
+        raise HTTPException(status_code=401, detail="unauthenticated")
+
+    access_token = (
+        await db_session.scalars(
+            select(models.IndieAuthAccessToken)
+            .where(models.IndieAuthAccessToken.access_token == token)
+            .join(
+                models.IndieAuthAuthorizationRequest,
+                models.IndieAuthAccessToken.indieauth_authorization_request_id
+                == models.IndieAuthAuthorizationRequest.id,
+            )
+            .where(
+                models.IndieAuthAuthorizationRequest.client_id == credentials.username
+            )
+        )
+    ).one_or_none()
+    if not access_token:
+        return JSONResponse(content={"active": False})
+
+    is_token_valid, _ = await _check_access_token(db_session, token)
+    if not is_token_valid:
+        return JSONResponse(content={"active": False})
+
+    return JSONResponse(
+        content={
+            "active": True,
+            "client_id": credentials.username,
+            "scope": access_token.scope,
+            "exp": int(
+                (
+                    access_token.created_at.replace(tzinfo=timezone.utc)
+                    + timedelta(seconds=access_token.expires_in)
+                ).timestamp()
+            ),
+        },
+        status_code=200,
+    )

app/ldsig.py

@@ -23,6 +23,13 @@ requests_loader = pyld.documentloader.requests.requests_document_loader()
 def _loader(url, options={}):
     # See https://github.com/digitalbazaar/pyld/issues/133
     options["headers"]["Accept"] = "application/ld+json"
+
+    # XXX: temp fix/hack is it seems to be down for now
+    if url == "https://w3id.org/identity/v1":
+        url = (
+            "https://raw.githubusercontent.com/web-payments/web-payments.org"
+            "/master/contexts/identity-v1.jsonld"
+        )
     return requests_loader(url, options)
 
 
@@ -34,7 +41,7 @@ def _options_hash(doc: ap.RawObject) -> str:
     for k in ["type", "id", "signatureValue"]:
         if k in doc:
             del doc[k]
-    doc["@context"] = "https://w3id.org/identity/v1"
+    doc["@context"] = "https://w3id.org/security/v1"
     normalized = jsonld.normalize(
         doc, {"algorithm": "URDNA2015", "format": "application/nquads"}
     )

app/main.py

@@ -1256,7 +1256,11 @@ async def post_remote_interaction(
 @app.get("/.well-known/webfinger")
 async def wellknown_webfinger(resource: str) -> JSONResponse:
     """Exposes/servers WebFinger data."""
-    if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]:
+    if resource not in [
+        f"acct:{USERNAME}@{WEBFINGER_DOMAIN}",
+        ID,
+        f"acct:{USERNAME}@{DOMAIN}",
+    ]:
         logger.info(f"Got invalid req for {resource}")
         raise HTTPException(status_code=404)
 
@@ -1409,6 +1413,7 @@ async def serve_proxy_media(
             _filter_proxy_resp_headers(
                 proxy_resp,
                 [
+                    "content-encoding",
                     "content-length",
                     "content-type",
                     "content-range",
@@ -1690,9 +1695,9 @@ async def _gen_rss_feed(
 
         fe = fg.add_entry()
         fe.id(outbox_object.url)
-
-        # Atom feeds require a title
-        if not is_rss:
+        if outbox_object.name is not None:
+            fe.title(outbox_object.name)
+        elif not is_rss:  # Atom feeds require a title
             fe.title(outbox_object.url)
 
         fe.link(href=outbox_object.url)

app/micropub.py

@@ -132,7 +132,7 @@ async def post_micropub_endpoint(
             h = form_data["h"]
         entry_type = f"h-{h}"
 
-    logger.info(f"Creating {entry_type}")
+    logger.info(f"Creating {entry_type=} with {access_token_info=}")
 
     if entry_type != "h-entry":
         return JSONResponse(
@@ -150,7 +150,7 @@ async def post_micropub_endpoint(
     else:
         content = form_data["content"]
 
-    public_id = await send_create(
+    public_id, _ = await send_create(
         db_session,
         "Note",
         content,

app/models.py

@@ -1,4 +1,5 @@
 import enum
+from datetime import datetime
 from typing import Any
 from typing import Optional
 from typing import Union
@@ -436,7 +437,7 @@ class OutboxObjectAttachment(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
 
     upload_id = Column(Integer, ForeignKey("upload.id"), nullable=False)
-    upload = relationship(Upload, uselist=False)
+    upload: Mapped["Upload"] = relationship(Upload, uselist=False)
 
 
 class IndieAuthAuthorizationRequest(Base):
@@ -459,7 +460,9 @@ class IndieAuthAccessToken(Base):
     __tablename__ = "indieauth_access_token"
 
     id = Column(Integer, primary_key=True, index=True)
-    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+    created_at: Mapped[datetime] = Column(
+        DateTime(timezone=True), nullable=False, default=now
+    )
 
     # Will be null for personal access tokens
     indieauth_authorization_request_id = Column(
@@ -470,9 +473,9 @@ class IndieAuthAccessToken(Base):
         uselist=False,
     )
 
-    access_token = Column(String, nullable=False, unique=True, index=True)
+    access_token: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
     refresh_token = Column(String, nullable=True, unique=True, index=True)
-    expires_in = Column(Integer, nullable=False)
+    expires_in: Mapped[int] = Column(Integer, nullable=False)
     scope = Column(String, nullable=False)
     is_revoked = Column(Boolean, nullable=False, default=False)
     was_refreshed = Column(Boolean, nullable=False, default=False, server_default="0")

app/outgoing_activities.py

@@ -151,7 +151,7 @@ def _set_next_try(
     if not outgoing_activity.tries:
         raise ValueError("Should never happen")
 
-    if outgoing_activity.tries == _MAX_RETRIES:
+    if outgoing_activity.tries >= _MAX_RETRIES:
         outgoing_activity.is_errored = True
         outgoing_activity.next_try = None
     else:

app/prune.py

@@ -102,6 +102,8 @@ async def _prune_old_inbox_objects(
                     models.InboxObject.ap_type.in_(["Note"]),
                 )
             ),
+            # Keep Move object as they are linked to notifications
+            models.InboxObject.ap_type.not_in(["Move"]),
             # Filter by retention days
             models.InboxObject.ap_published_at
             < now() - timedelta(days=INBOX_RETENTION_DAYS),

app/scss/main.scss

@@ -432,8 +432,7 @@ a.label-btn {
     .activity-attachment {
         margin: 30px 0 20px 0;
         img, audio, video {
-            width: 100%;
-            max-width: 740px;
+            max-width: calc(min(740px, 100%));
         }
     }
     img.inline-img {

app/source.py

@@ -3,12 +3,12 @@ import typing
 
 from loguru import logger
 from mistletoe import Document  # type: ignore
+from mistletoe.block_token import CodeFence  # type: ignore
 from mistletoe.html_renderer import HTMLRenderer  # type: ignore
 from mistletoe.span_token import SpanToken  # type: ignore
-from pygments import highlight  # type: ignore
 from pygments.formatters import HtmlFormatter  # type: ignore
 from pygments.lexers import get_lexer_by_name as get_lexer  # type: ignore
-from pygments.lexers import guess_lexer  # type: ignore
+from pygments.util import ClassNotFound  # type: ignore
 from sqlalchemy import select
 
 from app import webfinger
@@ -104,10 +104,16 @@ class CustomRenderer(HTMLRenderer):
         )
         return link
 
-    def render_block_code(self, token: typing.Any) -> str:
+    def render_block_code(self, token: CodeFence) -> str:
+        lexer_attr = ""
+        try:
+            lexer = get_lexer(token.language)
+            lexer_attr = f' data-microblogpub-lexer="{lexer.aliases[0]}"'
+        except ClassNotFound:
+            pass
+
         code = token.children[0].content
-        lexer = get_lexer(token.language) if token.language else guess_lexer(code)
-        return highlight(code, lexer, _FORMATTER)
+        return f"<pre><code{lexer_attr}>\n{code}\n</code></pre>"
 
 
 async def _prefetch_mentioned_actors(

app/templates/articles.html

@@ -11,8 +11,8 @@
 <ul class="h-feed" id="articles">
 <data class="p-name" value="{{ local_actor.display_name}}'s articles"></data>
 {% for outbox_object in objects %}
-    <li>
-        <span class="muted">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</span> <a href="{{ outbox_object.url }}">{{ outbox_object.name }}</a>
+    <li class="h-entry">
+        <time class="muted dt-published" datetime="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</time> <a href="{{ outbox_object.url }}" class="u-url u-uid p-name">{{ outbox_object.name }}</a>
     </li>
 {% endfor %}
 </ul>

app/templates/notifications.html

@@ -51,7 +51,7 @@
             {% elif notif.notification_type.value == "unblock" %}
                 {{ notif_actor_action(notif, "was unblocked") }}
                 {{ utils.display_actor(notif.actor, actors_metadata) }}
-            {%- elif notif.notification_type.value == "move" %}
+            {%- elif notif.notification_type.value == "move" and notif.inbox_object %}
                 {# for move notif, the actor is the target and the inbox object the Move activity #}
                 <div class="actor-action">
                     <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.inbox_object.actor.ap_id }}">

app/uploads.py

@@ -60,7 +60,7 @@ async def save_upload(db_session: AsyncSession, f: UploadFile) -> models.Upload:
             destination_image.putdata(original_image.getdata())
             destination_image.save(
                 dest_filename,
-                format=_original_image.format,
+                format=_original_image.format,  # type: ignore
             )
 
             with open(dest_filename, "rb") as dest_f:

app/utils/facepile.py

@@ -1,5 +1,6 @@
 import datetime
 from dataclasses import dataclass
+from datetime import timezone
 from typing import Any
 from typing import Optional
 
@@ -9,7 +10,7 @@ from app import media
 from app.models import InboxObject
 from app.models import Webmention
 from app.utils.datetime import parse_isoformat
-from app.utils.url import make_abs
+from app.utils.url import must_make_abs
 
 
 @dataclass
@@ -39,13 +40,15 @@ class Face:
                     return cls(
                         ap_actor_id=None,
                         url=(
-                            item["properties"]["url"][0]
+                            must_make_abs(
+                                item["properties"]["url"][0], webmention.source
+                            )
                             if item["properties"].get("url")
                             else webmention.source
                         ),
                         name=item["properties"]["name"][0],
                         picture_url=media.resized_media_url(
-                            make_abs(
+                            must_make_abs(
                                 item["properties"]["photo"][0], webmention.source
                             ),  # type: ignore
                             50,
@@ -65,7 +68,7 @@ class Face:
                         url=webmention.source,
                         name=author["properties"]["name"][0],
                         picture_url=media.resized_media_url(
-                            make_abs(
+                            must_make_abs(
                                 author["properties"]["photo"][0], webmention.source
                             ),  # type: ignore
                             50,
@@ -96,13 +99,13 @@ def _parse_face(webmention: Webmention, items: list[dict[str, Any]]) -> Face | N
                 return Face(
                     ap_actor_id=None,
                     url=(
-                        item["properties"]["url"][0]
+                        must_make_abs(item["properties"]["url"][0], webmention.source)
                         if item["properties"].get("url")
                         else webmention.source
                     ),
                     name=item["properties"]["name"][0],
                     picture_url=media.resized_media_url(
-                        make_abs(
+                        must_make_abs(
                             item["properties"]["photo"][0], webmention.source
                         ),  # type: ignore
                         50,
@@ -140,13 +143,23 @@ class WebmentionReply:
                             f"webmention id={webmention.id}"
                         )
                         break
+
+                    if "published" in item["properties"]:
+                        published_at = (
+                            parse_isoformat(item["properties"]["published"][0])
+                            .astimezone(timezone.utc)
+                            .replace(tzinfo=None)
+                        )
+                    else:
+                        published_at = webmention.created_at  # type: ignore
+
                     return cls(
                         face=face,
                         content=item["properties"]["content"][0]["html"],
-                        url=item["properties"]["url"][0],
-                        published_at=parse_isoformat(
-                            item["properties"]["published"][0]
-                        ).replace(tzinfo=None),
+                        url=must_make_abs(
+                            item["properties"]["url"][0], webmention.source
+                        ),
+                        published_at=published_at,
                         in_reply_to=webmention.target,  # type: ignore
                         webmention_id=webmention.id,  # type: ignore
                     )

app/utils/highlight.py

@@ -32,23 +32,22 @@ def highlight(html: str) -> str:
 
         # If this comes from a microblog.pub instance we may have the language
         # in the class name
-        if "class" in code.attrs and code.attrs["class"][0].startswith("language-"):
+        if "data-microblogpub-lexer" in code.attrs:
             try:
-                lexer = get_lexer_by_name(
-                    code.attrs["class"][0].removeprefix("language-")
-                )
+                lexer = get_lexer_by_name(code.attrs["data-microblogpub-lexer"])
             except Exception:
                 lexer = guess_lexer(code_content)
-        else:
-            lexer = guess_lexer(code_content)
 
-        # Replace the code with Pygment output
-        # XXX: the HTML escaping causes issue with Python type annotations
-        code_content = code_content.replace(") -> ", ") -> ")
-        code.parent.replaceWith(
-            BeautifulSoup(
-                phighlight(code_content, lexer, _FORMATTER), "html5lib"
-            ).body.next
-        )
+            # Replace the code with Pygment output
+            # XXX: the HTML escaping causes issue with Python type annotations
+            code_content = code_content.replace(") -> ", ") -> ")
+            code.parent.replaceWith(
+                BeautifulSoup(
+                    phighlight(code_content, lexer, _FORMATTER), "html5lib"
+                ).body.next
+            )
+        else:
+            code.name = "div"
+            code["class"] = code.get("class", []) + ["highlight"]
 
     return soup.body.encode_contents().decode()

app/utils/url.py

@@ -21,6 +21,13 @@ def make_abs(url: str | None, parent: str) -> str | None:
     )
 
 
+def must_make_abs(url: str | None, parent: str) -> str:
+    abs_url = make_abs(url, parent)
+    if not abs_url:
+        raise ValueError("missing URL")
+    return abs_url
+
+
 class InvalidURLError(Exception):
     pass
 

docs/install.md

@@ -191,6 +191,29 @@ http {
 }
 ```
 
+
+## (Advanced) Running on a subdomain
+
+It is possible to run microblogpub on a subdomain (`sub.domain.tld`) while being reachable from the root root domain (`domain.tld`) using the `name@domain.tld` handle.
+
+This requires forwarding/proxying requests from the root domain to the subdomain, for example using NGINX:
+
+```nginx
+location /.well-known/webfinger {
+  add_header Access-Control-Allow-Origin '*';
+  return 301 https://sub.domain.tld$request_uri;
+}
+```
+
+And updating `data/profile.toml` to specify the root domain as the webfinger domain:
+
+```toml
+webfinger_domain = "domain.tld"
+```
+
+Once configured correctly, people will be able to follow you using `name@domain.tld`, while using `sub.domain.tld` for the web interface.
+
+
 ## (Advanced) Running from subpath
 
 It is possible to configure microblogpub to run from subpath.

docs/user_guide.md

@@ -25,9 +25,10 @@ As these two config items define your ActivityPub handle `@handle@domain`.
 
 You can tweak your profile by tweaking these items:
 
- - `name`
- - `summary` (using Markdown)
- - `icon_url`
+ - `name`: The name shown with your profile.
+ - `summary`: The summary or 'bio' part of your profile, written in Markdown.
+ - `icon_url`: Your profile image or avatar.
+ - `image_url`: This provides a 'header' or 'banner' image. Note that it is not shown by the default Microblog.pub templates. It will be used by Mastodon (which uses a 3:1 ratio image) and Pleroma. Pixelfed and Peertube, for example, don't show these images by default.
 
 Whenever one of these config items is updated, an `Update` activity will be sent to all known servers to update your remote profile.
 
@@ -35,6 +36,15 @@ The server will need to be restarted for taking changes into account.
 
 Before restarting the server, you can ensure you haven't made any mistakes by running the [configuration checking task](/user_guide.html#configuration-checking).
 
+Note that currently `image_url` is not used anywhere in microblog.pub itself, but other clients/servers do occasionally use it when showing remote profiles as a background image.
+Also, this image _can_ be used in microblog.pub - just add this:
+
+```html
+<img src="{{ local_actor.image_url | media_proxy_url }}">
+```
+
+to an appropriate place of your template (most likely, `header.html`).
+For more information, see a section about [custom templates](/user_guide.html#custom-templates) further in this document.
 
 ### Profile metadata
 
@@ -161,10 +171,35 @@ $secondary-color: #32cd32;
 
 See `app/scss/main.scss` to see what variables can be overridden.
 
+You will need to [recompile CSS](#recompiling-css-files) after doing any CSS changes (for actual css files to be updates) and restart microblog.pub (for css link in HTML documents to be updated with a new checksum - otherwise, browsers that downloaded old CSS will keep using it).
+
+#### Custom favicon
+
+By default, microblog.pub favicon is a square of `$primary-color` CSS color (see above section on how to redefine CSS colors).
+You can change it to any icon you like - just save a desired file as `data/favicon.ico`.
+After that, run the "[recompile CSS](#recompiling-css-files)" task to copy it to `app/static/favicon.ico`.
+
 #### Custom templates
 
 If you'd like to customize your instance's theme beyond CSS, you can modify the app's HTML by placing templates in `data/templates` which overwrite the defaults in `app/templates`.
 
+Templates are written using [Jinja](https://jinja.palletsprojects.com/en/latest/templates/) templating language.
+Moreover, `utils.html` has scoped blocks around the body of every macro.
+This allows macros to be overridden individually in `data/templates/utils.html`, without copying the whole file.
+For example, to only override the display of a specific actor's name/icon, you can create `data/templates/utils.html` file with following content:
+
+```jinja
+{% extends "app/utils.html" %}
+
+{% block display_actor %}
+	{% if actor.ap_id == "https://me.example.com" %}
+		<!-- custom actor display -->
+	{% else %}
+		{{ super() }}
+	{% endif %}
+{% endblock %}
+```
+
 #### Custom Content Security Policy (CSP)
 
 You can override the default Content Security Policy by adding a line in `data/profile.toml`:
@@ -320,7 +355,7 @@ First you need to grab the "ActivityPub actor URL" for your existing account:
 
 ```bash
 # For a Python install
-poetry run inv webfinger username@domain.tld
+poetry run inv webfinger username@instance-you-want-to-move-from.tld
 ```
 
 Edit the config.
@@ -329,7 +364,7 @@ Edit the config.
 
 ```bash
 # For a Docker install
-make account=username@domain.tld webfinger
+make account=username@instance-you-want-to-move-from.tld webfinger
 ```
 
 Edit the config.
@@ -339,11 +374,13 @@ Edit the config.
 And add a reference to your old/existing account in `profile.toml`:
 
 ```toml
-also_known_as = "my@old-account.com"
+also_known_as = "https://instance-you-want-to-move-form.tld/users/username"
 ```
 
 Restart the server, and you should be able to complete the move from your existing account.
 
+Note that if you already have a redirect in place on Mastodon, you may have to remove it before initiating the migration.
+
 ## Import follows from Mastodon
 
 You can import the list of follows/following accounts from Mastodon.