remove fk from fetch_current_batch_end_id

an attempt at fixing #584

migrations/versions/7b0e9b8e0887_remove_fetch_batch_fk.py

@@ -0,0 +1,64 @@
+"""remove fetch batch fk
+
+things are real bad if the associated post is deleted and this is nulled
+keeping an opaque ID and associated date should work fine
+see GH-584
+
+Revision ID: 7b0e9b8e0887
+Revises: 740fe24a7712
+Create Date: 2022-02-27 11:48:55.107299
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '7b0e9b8e0887'
+down_revision = '740fe24a7712'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('accounts', sa.Column('fetch_current_batch_end_date', sa.DateTime(timezone=True), nullable=True))
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_date = posts.created_at
+            FROM posts WHERE accounts.fetch_current_batch_end_id == posts.id;
+            ''')
+
+    # update ids from "mastodon:69420@chitter.xyz" format to just "69420"
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_id =
+            split_part(
+                split_part(fetch_current_batch_end_id, ':', 2)
+                    '@', 1);
+        ''')
+
+    op.drop_constraint('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', type_='foreignkey')
+
+
+def downgrade():
+    # converts ids like "69420" back to "mastodon:69420@chitter.xyz"
+    # i sure hope there isn't a mastodon-compatible out there that can have
+    # : or @ in its post ids
+    op.execute('''
+        WITH accounts_exploded_ids AS (
+            SELECT
+                id,
+                split_part(id, ':', 1) || ':' AS service,
+                CASE WHEN position('@' IN id) != 0
+                    THEN '@' || split_part(id, @, 2)
+                    ELSE ''
+                END as instance
+            FROM accounts
+        )
+        UPDATE accounts SET fetch_current_batch_end_id = e.service || fetch_current_batch_end_id || e.instance
+        FROM accounts_exploded_ids AS e WHERE e.id = accounts.id AND fetch_current_batch_end_id IS NOT NULL;
+    ''')
+    op.execute('''
+        UPDATE accounts SET fetch_current_batch_end_id = NULL
+            WHERE NOT EXISTS (SELECT 1 FROM posts WHERE fetch_current_batch_end_id = posts.id);
+            ''')
+    op.create_foreign_key('fk_accounts_fetch_current_batch_end_id_posts', 'accounts', 'posts', ['fetch_current_batch_end_id'], ['id'], ondelete='SET NULL')
+    op.drop_column('accounts', 'fetch_current_batch_end_date')

model.py

@@ -146,15 +146,8 @@ class Account(TimestampMixin, RemoteIDMixin):
     fetch_history_complete = db.Column(db.Boolean, server_default='FALSE',
                                        nullable=False)
 
-    @declared_attr
-    def fetch_current_batch_end_id(cls):
-        return db.Column(db.String, db.ForeignKey('posts.id', ondelete='SET NULL'))
-    @declared_attr
-    def fetch_current_batch_end(cls):
-        return db.relationship("Post", foreign_keys=(cls.fetch_current_batch_end_id,))
-    # the declared_attr is necessary because of the foreign key
-    # and because this class is technically one big mixin
-    # https://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/mixins.html#mixing-in-relationships
+    fetch_current_batch_end_id = db.Column(db.String)
+    fetch_current_batch_end_date = db.Column(db.DateTime(timezone=True))
 
     reason = db.Column(db.String)
     dormant = db.Column(db.Boolean, server_default='FALSE', nullable=False)

tasks.py

@@ -119,10 +119,10 @@ def fetch_acc(id_):
             else:
                 max_id = None
             since_id = None
-        elif account.fetch_current_batch_end:
+        elif account.fetch_current_batch_end_date:
             oldest = (db.session.query(Post)
                       .with_parent(account, 'posts')
-                      .filter(Post.created_at > account.fetch_current_batch_end.created_at)
+                      .filter(Post.created_at > account.fetch_current_batch_end_date)
                       .order_by(db.asc(Post.created_at))
                       .first())
             # ^ None if this is our first fetch of this batch, otherwise oldest of this batch
@@ -130,7 +130,7 @@ def fetch_acc(id_):
                 max_id = oldest.remote_id
             else:
                 max_id = None
-            since_id = account.fetch_current_batch_end.remote_id
+            since_id = account.fetch_current_batch_end_id
         else:
             # we shouldn't get here unless the user had no posts on the service last time we fetched
             max_id = None
@@ -171,7 +171,7 @@ def fetch_acc(id_):
             batch_end = (Post.query.with_parent(account, 'posts').order_by(
                 db.desc(Post.created_at)).first())
             if batch_end:
-                account.fetch_current_batch_end_id = batch_end.id
+                account.fetch_current_batch_end_id = batch_end.remote_id
             else:
                 account.fetch_current_batch_end_id = None
             db.session.commit()