diff --git a/app/actor.py b/app/actor.py
index c5dd0c3..1bbe1e2 100644
--- a/app/actor.py
+++ b/app/actor.py
@@ -199,8 +199,11 @@ async def fetch_actor(
             )
             try:
                 ap_actor = await ap.fetch(actor_id)
-                existing_actor.ap_actor = ap_actor
-                existing_actor.updated_at = now()
+                await update_actor_if_needed(
+                    db_session,
+                    existing_actor,
+                    RemoteActor(ap_actor),
+                )
                 return existing_actor
             except Exception:
                 logger.exception(f"Failed to refresh {actor_id}")
@@ -223,8 +226,11 @@ async def fetch_actor(
             ).one_or_none()
             if existing_actor_by_url:
                 # Update the actor as we had to fetch it anyway
-                existing_actor_by_url.ap_actor = ap_actor
-                existing_actor_by_url.updated_at = now()
+                await update_actor_if_needed(
+                    db_session,
+                    existing_actor_by_url,
+                    RemoteActor(ap_actor),
+                )
                 return existing_actor_by_url
 
         return await save_actor(db_session, ap_actor)
@@ -232,6 +238,20 @@ async def fetch_actor(
         raise ap.ObjectNotFoundError(actor_id)
 
 
+async def update_actor_if_needed(
+    db_session: AsyncSession,
+    actor_in_db: "ActorModel",
+    ra: RemoteActor,
+) -> None:
+    # Check if we actually need to udpte the actor in DB
+    if _actor_hash(ra) != _actor_hash(actor_in_db):
+        actor_in_db.ap_actor = ra.ap_actor
+        actor_in_db.handle = ra.handle
+        actor_in_db.ap_type = ra.ap_type
+        actor_in_db.updated_at = now()
+        await db_session.flush()
+
+
 @dataclass
 class ActorMetadata:
     ap_actor_id: str
diff --git a/app/boxes.py b/app/boxes.py
index 9c72b6f..dd27a19 100644
--- a/app/boxes.py
+++ b/app/boxes.py
@@ -24,6 +24,7 @@ from app.actor import Actor
 from app.actor import RemoteActor
 from app.actor import fetch_actor
 from app.actor import save_actor
+from app.actor import update_actor_if_needed
 from app.ap_object import RemoteObject
 from app.config import BASE_URL
 from app.config import BLOCKED_SERVERS
@@ -1499,8 +1500,7 @@ async def _handle_update_activity(
             )
 
         # Update the actor
-        from_actor.ap_actor = updated_actor.ap_actor
-        from_actor.updated_at = now()
+        await update_actor_if_needed(db_session, from_actor, updated_actor)
     elif (ap_type := wrapped_object["type"]) in [
         "Question",
         "Note",
diff --git a/app/httpsig.py b/app/httpsig.py
index e0b3926..daac37b 100644
--- a/app/httpsig.py
+++ b/app/httpsig.py
@@ -116,7 +116,7 @@ async def _get_public_key(
     # Fetch it
     from app import activitypub as ap
     from app.actor import RemoteActor
-    from app.actor import _actor_hash
+    from app.actor import update_actor_if_needed
 
     # Without signing the request as if it's the first contact, the 2 servers
     # might race to fetch each other key
@@ -140,16 +140,10 @@ async def _get_public_key(
             f"failed to fetch requested key {key_id}: got {actor['publicKey']}"
         )
 
-    if (
-        should_skip_cache
-        and actor["type"] != "Key"
-        and existing_actor
-        and _actor_hash(RemoteActor(actor)) != _actor_hash(existing_actor)
-    ):
+    if should_skip_cache and actor["type"] != "Key" and existing_actor:
         # We had to skip the cache, which means the actor key probably changed
         # and we want to update our cached version
-        existing_actor.ap_actor = actor
-        existing_actor.updated_at = now()
+        await update_actor_if_needed(db_session, existing_actor, RemoteActor(actor))
         await db_session.commit()
 
     _KEY_CACHE[key_id] = k