From 0f6915fdbb228a60b895fde2bf16e6731594b667 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 19 Jul 2022 20:38:32 +0200 Subject: [PATCH] Webmention improvements - Tweak design for IndieAuth login flow - Webmentions notifications support - Refactor webmentions processing --- alembic/env.py | 7 +- .../2b51ae7047cb_webmention_notifications.py | 32 +++++++++ app/admin.py | 1 + app/main.py | 2 + app/models.py | 66 ++++++++++-------- app/templates/indieauth_flow.html | 67 +++++++++---------- app/templates/notifications.html | 31 ++++++++- app/utils/microformats.py | 31 ++++++--- app/webmentions.py | 57 ++++++++++++---- 9 files changed, 204 insertions(+), 90 deletions(-) create mode 100644 alembic/versions/2b51ae7047cb_webmention_notifications.py diff --git a/alembic/env.py b/alembic/env.py index f0bf9cd..ca982b3 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -49,6 +49,7 @@ def run_migrations_offline() -> None: target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, + render_as_batch=True, ) with context.begin_transaction(): @@ -69,7 +70,11 @@ def run_migrations_online() -> None: ) with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) with context.begin_transaction(): context.run_migrations() diff --git a/alembic/versions/2b51ae7047cb_webmention_notifications.py b/alembic/versions/2b51ae7047cb_webmention_notifications.py new file mode 100644 index 0000000..a72eb87 --- /dev/null +++ b/alembic/versions/2b51ae7047cb_webmention_notifications.py @@ -0,0 +1,32 @@ +"""Webmention notifications + +Revision ID: 2b51ae7047cb +Revises: e58c1ffadf2e +Create Date: 2022-07-19 20:22:06.968951 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2b51ae7047cb' +down_revision = 'e58c1ffadf2e' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('notifications', schema=None) as batch_op: + batch_op.create_foreign_key('fk_webmention_id', 'webmention', ['webmention_id'], ['id']) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('notifications', schema=None) as batch_op: + batch_op.drop_constraint('fk_webmention_id', type_='foreignkey') + + # ### end Alembic commands ### diff --git a/app/admin.py b/app/admin.py index f939f5e..57aaa38 100644 --- a/app/admin.py +++ b/app/admin.py @@ -435,6 +435,7 @@ async def get_notifications( models.OutboxObject.outbox_object_attachments ).options(joinedload(models.OutboxObjectAttachment.upload)), ), + joinedload(models.Notification.webmention), ) .order_by(models.Notification.created_at.desc()) ) diff --git a/app/main.py b/app/main.py index eca9b27..ed540db 100644 --- a/app/main.py +++ b/app/main.py @@ -76,6 +76,8 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac # TODO(ts): # # Next: +# - Webmention notification +# - Page support # - Article support # - indieauth tweaks # - API for posting notes diff --git a/app/models.py b/app/models.py index 1958561..80ec2d6 100644 --- a/app/models.py +++ b/app/models.py @@ -300,35 +300,6 @@ class Following(Base): ap_actor_id = Column(String, nullable=False, unique=True) -@enum.unique -class NotificationType(str, enum.Enum): - NEW_FOLLOWER = "new_follower" - UNFOLLOW = "unfollow" - LIKE = "like" - UNDO_LIKE = "undo_like" - ANNOUNCE = "announce" - UNDO_ANNOUNCE = "undo_announce" - MENTION = "mention" - - -class Notification(Base): - __tablename__ = "notifications" - - id = Column(Integer, primary_key=True, index=True) - created_at = Column(DateTime(timezone=True), nullable=False, default=now) - notification_type = Column(Enum(NotificationType), nullable=True) - is_new = Column(Boolean, nullable=False, default=True) - - actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True) - actor = relationship(Actor, uselist=False) - - outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True) - outbox_object = relationship(OutboxObject, uselist=False) - - inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True) - inbox_object = relationship(InboxObject, uselist=False) - - class IncomingActivity(Base): __tablename__ = "incoming_activity" @@ -503,6 +474,43 @@ class Webmention(Base): return None +@enum.unique +class NotificationType(str, enum.Enum): + NEW_FOLLOWER = "new_follower" + UNFOLLOW = "unfollow" + LIKE = "like" + UNDO_LIKE = "undo_like" + ANNOUNCE = "announce" + UNDO_ANNOUNCE = "undo_announce" + MENTION = "mention" + NEW_WEBMENTION = "new_webmention" + UPDATED_WEBMENTION = "updated_webmention" + DELETED_WEBMENTION = "deleted_webmention" + + +class Notification(Base): + __tablename__ = "notifications" + + id = Column(Integer, primary_key=True, index=True) + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + notification_type = Column(Enum(NotificationType), nullable=True) + is_new = Column(Boolean, nullable=False, default=True) + + actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True) + actor = relationship(Actor, uselist=False) + + outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True) + outbox_object = relationship(OutboxObject, uselist=False) + + inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True) + inbox_object = relationship(InboxObject, uselist=False) + + webmention_id = Column( + Integer, ForeignKey("webmention.id", name="fk_webmention_id"), nullable=True + ) + webmention = relationship(Webmention, uselist=False) + + outbox_fts = Table( "outbox_fts", metadata_obj, diff --git a/app/templates/indieauth_flow.html b/app/templates/indieauth_flow.html index 6af8071..819ee45 100644 --- a/app/templates/indieauth_flow.html +++ b/app/templates/indieauth_flow.html @@ -2,41 +2,40 @@ {% extends "layout.html" %} {% block content %}
-
-{% if client.logo %} -
- -
-{% endif %} -
-
-{{ client.name }} -

wants you to login as {{ me }} with the following redirect URI: {{ redirect_uri }}.

-
-
-
+
+ {% if client.logo %} +
+ {{ client.name }} logo +
+ {% endif %} +
+
+ {{ client.name }} +

wants you to login as {{ me }} with the following redirect URI: {{ redirect_uri }}.

-
- - {% if scopes %} -

Scopes

-
    - {% for scope in scopes %} -
  • -
    • - {% endfor %} -
- {% endif %} - - - - - - - - -
-
+
+ {{ utils.embed_csrf_token() }} + {% if scopes %} +

Scopes

+
    + {% for scope in scopes %} +
  • +
    • + {% endfor %} +
+ {% endif %} + + + + + + + + +
+
+
+
{% endblock %} diff --git a/app/templates/notifications.html b/app/templates/notifications.html index 02ac007..7ad0b6d 100644 --- a/app/templates/notifications.html +++ b/app/templates/notifications.html @@ -47,7 +47,36 @@ {{ notif.actor.display_name }} mentioned you {{ utils.display_object(notif.inbox_object) }} - + {% elif notif.notification_type.value == "new_webmention" %} +
+ new webmention from + {% set facepile_item = notif.webmention.as_facepile_item %} + {% if facepile_item %} + {{ facepile_item.actor_name }} + {% endif %} + {{ notif.webmention.source }} +
+ {{ utils.display_object(notif.outbox_object) }} + {% elif notif.notification_type.value == "updated_webmention" %} +
+ updated webmention from + {% set facepile_item = notif.webmention.as_facepile_item %} + {% if facepile_item %} + {{ facepile_item.actor_name }} + {% endif %} + {{ notif.webmention.source }} +
+ {{ utils.display_object(notif.outbox_object) }} + {% elif notif.notification_type.value == "deleted_webmention" %} +
+ deleted webmention from + {% set facepile_item = notif.webmention.as_facepile_item %} + {% if facepile_item %} + {{ facepile_item.actor_name }} + {% endif %} + {{ notif.webmention.source }} +
+ {{ utils.display_object(notif.outbox_object) }} {% else %}
Implement {{ notif.notification_type }} diff --git a/app/utils/microformats.py b/app/utils/microformats.py index 937e4e8..7e45c6b 100644 --- a/app/utils/microformats.py +++ b/app/utils/microformats.py @@ -7,19 +7,28 @@ from loguru import logger from app import config -async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str] | None: +class URLNotFoundOrGone(Exception): + pass + + +async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str]: async with httpx.AsyncClient() as client: + resp = await client.get( + url, + headers={ + "User-Agent": config.USER_AGENT, + }, + follow_redirects=True, + ) + if resp.status_code in [404, 410]: + raise URLNotFoundOrGone + try: - resp = await client.get( - url, - headers={ - "User-Agent": config.USER_AGENT, - }, - follow_redirects=True, - ) resp.raise_for_status() - except (httpx.HTTPError, httpx.HTTPStatusError): - logger.exception(f"Failed to discover webmention endpoint for {url}") - return None + except httpx.HTTPStatusError: + logger.error( + f"Failed to parse microformats for {url}: " f"got {resp.status_code}" + ) + raise return mf2py.parse(doc=resp.text), resp.text diff --git a/app/webmentions.py b/app/webmentions.py index 972e98a..20c4808 100644 --- a/app/webmentions.py +++ b/app/webmentions.py @@ -1,3 +1,4 @@ +import httpx from bs4 import BeautifulSoup # type: ignore from fastapi import APIRouter from fastapi import Depends @@ -73,33 +74,53 @@ async def webmention_endpoint( await db_session.commit() raise HTTPException(status_code=400, detail="Invalid target") - maybe_data_and_html = await microformats.fetch_and_parse(source) - if not maybe_data_and_html: - logger.info("failed to fetch source") + is_webmention_deleted = False + try: + data_and_html = await microformats.fetch_and_parse(source) + except microformats.URLNotFoundOrGone: + is_webmention_deleted = True + except httpx.HTTPError: + raise HTTPException(status_code=500, detail=f"Fetch to process {source}") + data, html = data_and_html + is_target_found_in_source = is_source_containing_target(html, target) + + data, html = data_and_html + if is_webmention_deleted or not is_target_found_in_source: + logger.warning(f"target {target=} not found in source") if existing_webmention_in_db: logger.info("Deleting existing Webmention") mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1 existing_webmention_in_db.is_deleted = True - await db_session.commit() - raise HTTPException(status_code=400, detail="failed to fetch source") - data, html = maybe_data_and_html + notif = models.Notification( + notification_type=models.NotificationType.DELETED_WEBMENTION, + outbox_object_id=mentioned_object.id, + webmention_id=existing_webmention_in_db.id, + ) + db_session.add(notif) - if not is_source_containing_target(html, target): - logger.warning("target not found in source") - - if existing_webmention_in_db: - logger.info("Deleting existing Webmention") - mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1 - existing_webmention_in_db.is_deleted = True await db_session.commit() - raise HTTPException(status_code=400, detail="target not found in source") + if not is_target_found_in_source: + raise HTTPException( + status_code=400, + detail="target not found in source", + ) + else: + return JSONResponse(content={}, status_code=200) if existing_webmention_in_db: + # Undelete if needed existing_webmention_in_db.is_deleted = False existing_webmention_in_db.source_microformats = data + + notif = models.Notification( + notification_type=models.NotificationType.UPDATED_WEBMENTION, + outbox_object_id=mentioned_object.id, + webmention_id=existing_webmention_in_db.id, + ) + db_session.add(notif) else: new_webmention = models.Webmention( source=source, @@ -108,6 +129,14 @@ async def webmention_endpoint( outbox_object_id=mentioned_object.id, ) db_session.add(new_webmention) + await db_session.flush() + + notif = models.Notification( + notification_type=models.NotificationType.NEW_WEBMENTION, + outbox_object_id=mentioned_object.id, + webmention_id=new_webmention.id, + ) + db_session.add(notif) mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1