Bootstrap Micropub support, and start support for Update activities

alembic/versions/e58c1ffadf2e_update_support.py

@@ -0,0 +1,28 @@
+"""Update support
+
+Revision ID: e58c1ffadf2e
+Revises: fd23d95e5c16
+Create Date: 2022-07-17 18:19:42.362542
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'e58c1ffadf2e'
+down_revision = 'fd23d95e5c16'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('outbox', sa.Column('revisions', sa.JSON(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('outbox', 'revisions')
+    # ### end Alembic commands ###

app/activitypub.py

@@ -251,16 +251,30 @@ async def get_object(activity: RawObject) -> RawObject:
 
 
 def wrap_object(activity: RawObject) -> RawObject:
-    return {
+    # TODO(ts): improve Create VS Update
-        "@context": AS_EXTENDED_CTX,
+    if "updated" in activity:
-        "actor": config.ID,
+        return {
-        "to": activity.get("to", []),
+            "@context": AS_EXTENDED_CTX,
-        "cc": activity.get("cc", []),
+            "actor": config.ID,
-        "id": activity["id"] + "/activity",
+            "to": activity.get("to", []),
-        "object": remove_context(activity),
+            "cc": activity.get("cc", []),
-        "published": activity["published"],
+            "id": activity["id"] + "/update_activity/" + activity["updated"],
-        "type": "Create",
+            "object": remove_context(activity),
-    }
+            "published": activity["published"],
+            "updated": activity["updated"],
+            "type": "Update",
+        }
+    else:
+        return {
+            "@context": AS_EXTENDED_CTX,
+            "actor": config.ID,
+            "to": activity.get("to", []),
+            "cc": activity.get("cc", []),
+            "id": activity["id"] + "/activity",
+            "object": remove_context(activity),
+            "published": activity["published"],
+            "type": "Create",
+        }
 
 
 def wrap_object_if_needed(raw_object: RawObject) -> RawObject:

app/ap_object.py

@@ -147,6 +147,10 @@ class Object:
     def summary(self) -> str | None:
         return self.ap_object.get("summary")
 
+    @property
+    def name(self) -> str | None:
+        return self.ap_object.get("name")
+
     @cached_property
     def permalink_id(self) -> str:
         return (

app/boxes.py

@@ -392,6 +392,77 @@ async def send_create(
     return note_id
 
 
+async def send_update(
+    db_session: AsyncSession,
+    ap_id: str,
+    source: str,
+) -> str:
+    outbox_object = await get_outbox_object_by_ap_id(db_session, ap_id)
+    if not outbox_object:
+        raise ValueError(f"{ap_id} not found")
+
+    revisions = outbox_object.revisions or []
+    revisions.append(
+        {
+            "ap_object": outbox_object.ap_object,
+            "source": outbox_object.source,
+            "updated": (
+                outbox_object.ap_object.get("updated")
+                or outbox_object.ap_object.get("published")
+            ),
+        }
+    )
+
+    updated = now().replace(microsecond=0).isoformat().replace("+00:00", "Z")
+    content, tags, mentioned_actors = await markdownify(db_session, source)
+
+    note = {
+        "@context": ap.AS_EXTENDED_CTX,
+        "type": outbox_object.ap_type,
+        "id": outbox_object.ap_id,
+        "attributedTo": ID,
+        "content": content,
+        "to": outbox_object.ap_object["to"],
+        "cc": outbox_object.ap_object["cc"],
+        "published": outbox_object.ap_object["published"],
+        "context": outbox_object.ap_context,
+        "conversation": outbox_object.ap_context,
+        "url": outbox_object.url,
+        "tag": tags,
+        "summary": outbox_object.summary,
+        "inReplyTo": outbox_object.in_reply_to,
+        "sensitive": outbox_object.sensitive,
+        "attachment": outbox_object.ap_object["attachment"],
+        "updated": updated,
+    }
+
+    outbox_object.ap_object = note
+    outbox_object.source = source
+    outbox_object.revisions = revisions
+    await db_session.commit()
+
+    recipients = await _compute_recipients(db_session, note)
+    for rcp in recipients:
+        await new_outgoing_activity(db_session, rcp, outbox_object.id)
+
+    # If the note is public, check if we need to send any webmentions
+    if outbox_object.visibility == ap.VisibilityEnum.PUBLIC:
+        possible_targets = opengraph._urls_from_note(note)
+        logger.info(f"webmentions possible targert {possible_targets}")
+        for target in possible_targets:
+            webmention_endpoint = await webmentions.discover_webmention_endpoint(target)
+            logger.info(f"{target=} {webmention_endpoint=}")
+            if webmention_endpoint:
+                await new_outgoing_activity(
+                    db_session,
+                    webmention_endpoint,
+                    outbox_object_id=outbox_object.id,
+                    webmention_target=target,
+                )
+
+    return outbox_object.public_id  # type: ignore
+
+
 async def _compute_recipients(
     db_session: AsyncSession, ap_object: ap.RawObject
 ) -> set[str]:

app/database.py

@@ -1,6 +1,7 @@
 from typing import Any
 from typing import AsyncGenerator
 
+from sqlalchemy import MetaData
 from sqlalchemy import create_engine
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.asyncio import create_async_engine
@@ -20,6 +21,7 @@ async_engine = create_async_engine(DATABASE_URL, future=True, echo=False)
 async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False)
 
 Base: Any = declarative_base()
+metadata_obj = MetaData()
 
 
 async def get_db_session() -> AsyncGenerator[AsyncSession, None]:

app/incoming_activities.py

@@ -100,12 +100,12 @@ async def process_next_incoming_activity(db_session: AsyncSession) -> bool:
     next_activity.last_try = now()
 
     try:
-        async with db_session.begin_nested():
+        # async with db_session.begin_nested():
-            await save_to_inbox(
+        await save_to_inbox(
-                db_session,
+            db_session,
-                next_activity.ap_object,
+            next_activity.ap_object,
-                next_activity.sent_by_ap_actor_id,
+            next_activity.sent_by_ap_actor_id,
-            )
+        )
     except Exception:
         logger.exception("Failed")
         next_activity.error = traceback.format_exc()

app/indieauth.py

@@ -292,6 +292,13 @@ async def verify_access_token(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> AccessTokenInfo:
     token = request.headers.get("Authorization", "").removeprefix("Bearer ")
+
+    # Check if the token is within the form data
+    if not token:
+        form_data = await request.form()
+        if "access_token" in form_data:
+            token = form_data.get("access_token")
+
     is_token_valid, access_token = await _check_access_token(db_session, token)
     if not is_token_valid:
         raise HTTPException(

app/main.py

@@ -44,6 +44,7 @@ from app import boxes
 from app import config
 from app import httpsig
 from app import indieauth
+from app import micropub
 from app import models
 from app import templates
 from app import webmentions
@@ -177,6 +178,7 @@ app.mount("/static", StaticFiles(directory="app/static"), name="static")
 app.include_router(admin.router, prefix="/admin")
 app.include_router(admin.unauthenticated_router, prefix="/admin")
 app.include_router(indieauth.router)
+app.include_router(micropub.router)
 app.include_router(webmentions.router)
 app.add_middleware(ProxyHeadersMiddleware)
 app.add_middleware(CustomMiddleware)

app/micropub.py

@@ -0,0 +1,109 @@
+from typing import Any
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from fastapi.responses import RedirectResponse
+
+from app import activitypub as ap
+from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import send_create
+from app.boxes import send_delete
+from app.database import AsyncSession
+from app.database import get_db_session
+from app.indieauth import AccessTokenInfo
+from app.indieauth import verify_access_token
+
+router = APIRouter()
+
+
+@router.get("/micropub")
+async def micropub_endpoint(
+    request: Request,
+    access_token_info: AccessTokenInfo = Depends(verify_access_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> dict[str, Any] | JSONResponse:
+    if request.query_params.get("q") == "config":
+        return {}
+
+    elif request.query_params.get("q") == "source":
+        url = request.query_params.get("url")
+        outbox_object = await get_outbox_object_by_ap_id(db_session, url)
+        if not outbox_object:
+            return JSONResponse(
+                content={
+                    "error": "invalid_request",
+                    "error_description": "No post with this URL",
+                },
+                status_code=400,
+            )
+
+        extra_props: dict[str, list[str]] = {}
+
+        return {
+            "type": ["h-entry"],
+            "properties": {
+                "published": [
+                    outbox_object.ap_published_at.isoformat()  # type: ignore
+                ],
+                "content": [outbox_object.source],
+                **extra_props,
+            },
+        }
+
+    return {}
+
+
+@router.post("/micropub")
+async def post_micropub_endpoint(
+    request: Request,
+    access_token_info: AccessTokenInfo = Depends(verify_access_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse | JSONResponse:
+    form_data = await request.form()
+    if "action" in form_data:
+        if form_data["action"] == "delete":
+            outbox_object = await get_outbox_object_by_ap_id(
+                db_session, form_data["url"]
+            )
+            if not outbox_object:
+                return JSONResponse(
+                    content={
+                        "error": "invalid_request",
+                        "error_description": "No post with this URL",
+                    },
+                    status_code=400,
+                )
+            await send_delete(db_session, outbox_object.ap_id)  # type: ignore
+            return JSONResponse(content={}, status_code=200)
+
+    h = "entry"
+    if "h" in form_data:
+        h = form_data["h"]
+
+    if h != "entry":
+        return JSONResponse(
+            content={
+                "error": "invalid_request",
+                "error_description": "Only h-entry are supported",
+            },
+            status_code=400,
+        )
+
+    content = form_data["content"]
+    public_id = await send_create(
+        db_session,
+        content,
+        uploads=[],
+        in_reply_to=None,
+        visibility=ap.VisibilityEnum.PUBLIC,
+    )
+
+    return JSONResponse(
+        content={},
+        status_code=201,
+        headers={
+            "Location": request.url_for("outbox_by_public_id", public_id=public_id)
+        },
+    )

app/models.py

@@ -3,6 +3,7 @@ from typing import Any
 from typing import Optional
 from typing import Union
 
+import pydantic
 from loguru import logger
 from sqlalchemy import JSON
 from sqlalchemy import Boolean
@@ -12,6 +13,7 @@ from sqlalchemy import Enum
 from sqlalchemy import ForeignKey
 from sqlalchemy import Integer
 from sqlalchemy import String
+from sqlalchemy import Table
 from sqlalchemy import UniqueConstraint
 from sqlalchemy.orm import Mapped
 from sqlalchemy.orm import relationship
@@ -23,10 +25,17 @@ from app.ap_object import Attachment
 from app.ap_object import Object as BaseObject
 from app.config import BASE_URL
 from app.database import Base
+from app.database import metadata_obj
 from app.utils import webmentions
 from app.utils.datetime import now
 
 
+class ObjectRevision(pydantic.BaseModel):
+    ap_object: ap.RawObject
+    source: str
+    updated_at: str
+
+
 class Actor(Base, BaseActor):
     __tablename__ = "actor"
 
@@ -147,6 +156,7 @@ class OutboxObject(Base, BaseObject):
 
     # Source content for activities (like Notes)
     source = Column(String, nullable=True)
+    revisions: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
 
     ap_published_at = Column(DateTime(timezone=True), nullable=False, default=now)
     visibility = Column(Enum(ap.VisibilityEnum), nullable=False)
@@ -491,3 +501,18 @@ class Webmention(Base):
                 f"Failed to generate facefile item for Webmention id={self.id}"
             )
             return None
+
+
+outbox_fts = Table(
+    "outbox_fts",
+    metadata_obj,
+    Column("rowid", Integer),
+    Column("outbox_fts", String),
+    Column("summary", String, nullable=True),
+    Column("name", String, nullable=True),
+    Column("source", String),
+)
+
+# db.execute(select(outbox_fts.c.rowid).where(outbox_fts.c.outbox_fts.op("MATCH")("toto AND omg"))).all()  # noqa
+# db.execute(select(models.OutboxObject).join(outbox_fts, outbox_fts.c.rowid == models.OutboxObject.id).where(outbox_fts.c.outbox_fts.op("MATCH")("toto2"))).scalars()  # noqa
+# db.execute(insert(outbox_fts).values({"outbox_fts": "delete", "rowid": 1, "source": dat[0].source}))  # noqa

app/templates/index.html

@@ -6,6 +6,7 @@
 <link rel="indieauth-metadata" href="{{ url_for("well_known_authorization_server") }}">
 <link rel="authorization_endpoint" href="{{ url_for("indieauth_authorization_endpoint") }}">
 <link rel="token_endpoint" href="{{ url_for("indieauth_token_endpoint") }}">
+<link rel="micropub" href="{{ url_for("micropub_endpoint") }}">
 <link rel="alternate" href="{{ local_actor.url }}" title="ActivityPub profile"  type="application/activity+json">
 <meta content="profile" property="og:type" />
 <meta content="{{ local_actor.url }}" property="og:url" />