[feature] add support for receiving federated status edits (#3597)

* add support for extracting Updated field from Statusable implementers

* add support for status edits in the database, and update status dereferencer to handle them

* remove unused AdditionalInfo{}.CreatedAt

* remove unused AdditionalEmojiInfo{}.CreatedAt

* update new mention creation to use status.UpdatedAt

* remove mention.UpdatedAt, fixes related to NewULIDFromTime() change

* add migration to remove Mention{}.UpdatedAt field

* add migration to add the StatusEdit{} table

* start adding tests, add delete function for status edits

* add more of status edit migrations, fill in more of the necessary edit delete functionality

* remove unused function

* allow generating gotosocial compatible ulid via CLI with `go run ./cmd/gen-ulid`

* add StatusEdit{} test models

* fix new statusedits sql

* use model instead of table name

* actually remove the Mention.UpdatedAt field...

* fix tests now new models are added, add more status edit DB tests

* fix panic wording

* add test for deleting status edits

* don't automatically set `updated_at` field on updated statuses

* flesh out more of the dereferencer status edit tests, ensure updated at field set on outgoing AS statuses

* remove media_attachments.updated_at column

* fix up more tests, further complete the dereferencer status edit tests

* update more status serialization tests not expecting 'updated' AS property

* gah!! json serialization tests!!

* undo some gtscontext wrapping changes

* more serialization test fixing 🥲

* more test fixing, ensure the edit.status_id field is actually set 🤦

* fix status edit test

* grrr linter

* add edited_at field to apimodel status

* remove the choice of paging on the timeline public filtered test (otherwise it needs updating every time you add statuses ...)

* ensure that status.updated_at always fits chronologically

* fix more serialization tests ...

* add more code comments

* fix envparsing

* update swagger file

* properly handle media description changes during status edits

* slight formatting tweak

* code comment

internal/processing/account/rss_test.go

@@ -70,7 +70,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
 func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
 	getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(context.Background(), "the_mighty_zork")
 	suite.NoError(err)
-	suite.EqualValues(1704878640, lastModified.Unix())
+	suite.EqualValues(1730451600, lastModified.Unix())
 
 	feed, err := getFeed()
 	suite.NoError(err)
@@ -79,13 +79,23 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
     <title>Posts from @the_mighty_zork@localhost:8080</title>
     <link>http://localhost:8080/@the_mighty_zork</link>
     <description>Posts from @the_mighty_zork@localhost:8080</description>
-    <pubDate>Wed, 10 Jan 2024 09:24:00 +0000</pubDate>
-    <lastBuildDate>Wed, 10 Jan 2024 09:24:00 +0000</lastBuildDate>
+    <pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate>
+    <lastBuildDate>Fri, 01 Nov 2024 09:00:00 +0000</lastBuildDate>
     <image>
       <url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp</url>
       <title>Avatar for @the_mighty_zork@localhost:8080</title>
       <link>http://localhost:8080/@the_mighty_zork</link>
     </image>
+    <item>
+      <title>edited status</title>
+      <link>http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</link>
+      <description>@the_mighty_zork@localhost:8080 made a new post: &#34;this is the latest revision of the status, with a content-warning&#34;</description>
+      <content:encoded><![CDATA[<p>this is the latest revision of the status, with a content-warning</p>]]></content:encoded>
+      <author>@the_mighty_zork@localhost:8080</author>
+      <guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</guid>
+      <pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate>
+      <source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
+    </item>
     <item>
       <title>HTML in post</title>
       <link>http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40</link>

internal/processing/media/getfile.go

@@ -177,9 +177,7 @@ func (p *Processor) getAttachmentContent(
 	}
 
 	// Start preparing API content model.
-	apiContent := &apimodel.Content{
-		ContentUpdated: attach.UpdatedAt,
-	}
+	apiContent := &apimodel.Content{}
 
 	// Retrieve appropriate
 	// size file from storage.

internal/processing/media/unattach_test.go

@@ -20,7 +20,6 @@ package media_test
 import (
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/suite"
 )
@@ -42,8 +41,6 @@ func (suite *UnattachTestSuite) TestUnattachMedia() {
 
 	dbAttachment, errWithCode := suite.db.GetAttachmentByID(ctx, a.ID)
 	suite.NoError(errWithCode)
-
-	suite.WithinDuration(dbAttachment.UpdatedAt, time.Now(), 1*time.Minute)
 	suite.Empty(dbAttachment.StatusID)
 }
 

internal/processing/status/get.go

@@ -67,7 +67,6 @@ func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account
 	if errWithCode != nil {
 		return nil, errWithCode
 	}
-
 	return p.c.GetAPIStatus(ctx, requestingAccount, targetStatus)
 }
 
@@ -106,5 +105,6 @@ func (p *Processor) SourceGet(ctx context.Context, requestingAccount *gtsmodel.A
 		err = gtserror.Newf("error converting status: %w", err)
 		return nil, gtserror.NewErrorInternalError(err)
 	}
+
 	return statusSource, nil
 }

internal/processing/stream/notification_test.go

@@ -79,8 +79,8 @@ func (suite *NotificationTestSuite) TestStreamNotification() {
     "header_description": "Flat gray background (default header).",
     "followers_count": 0,
     "following_count": 0,
-    "statuses_count": 3,
-    "last_status_at": "2021-09-11",
+    "statuses_count": 4,
+    "last_status_at": "2024-11-01",
     "emojis": [],
     "fields": []
   }

internal/processing/stream/statusupdate_test.go

@@ -54,6 +54,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() {
 	suite.Equal(`{
   "id": "01FVW7JHQFSFK166WWKR8CBA6M",
   "created_at": "2021-09-20T10:40:37.000Z",
+  "edited_at": null,
   "in_reply_to_id": null,
   "in_reply_to_account_id": null,
   "sensitive": false,
@@ -90,8 +91,8 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() {
     "header_description": "Flat gray background (default header).",
     "followers_count": 0,
     "following_count": 0,
-    "statuses_count": 3,
-    "last_status_at": "2021-09-11",
+    "statuses_count": 4,
+    "last_status_at": "2024-11-01",
     "emojis": [],
     "fields": []
   },

internal/processing/timeline/public_test.go

@@ -102,8 +102,8 @@ func (suite *PublicTestSuite) TestPublicTimelineGetHideFiltered() {
 		requester           = suite.testAccounts["local_account_1"]
 		maxID               = ""
 		sinceID             = ""
-		minID               = "01F8MHAAY43M6RJ473VQFCVH36" // 1 before filteredStatus
-		limit               = 10
+		minID               = ""
+		limit               = 100
 		local               = false
 		filteredStatus      = suite.testStatuses["admin_account_status_2"]
 		filteredStatusFound = false

internal/processing/workers/util.go

@@ -75,6 +75,21 @@ func (u *utils) wipeStatus(
 		}
 	}
 
+	// Before handling media, ensure
+	// historic edits are populated.
+	if !status.EditsPopulated() {
+		var err error
+
+		// Fetch all historical edits of status from database.
+		status.Edits, err = u.state.DB.GetStatusEditsByIDs(
+			gtscontext.SetBarebones(ctx),
+			status.EditIDs,
+		)
+		if err != nil {
+			errs.Appendf("error getting status edits from database: %w", err)
+		}
+	}
+
 	// Either delete all attachments for this status,
 	// or simply detach + clean them separately later.
 	//
@@ -83,20 +98,27 @@ func (u *utils) wipeStatus(
 	// status immediately (in case of delete + redraft).
 	if deleteAttachments {
 		// todo:u.state.DB.DeleteAttachmentsForStatus
-		for _, id := range status.AttachmentIDs {
+		for _, id := range status.AllAttachmentIDs() {
 			if err := u.media.Delete(ctx, id); err != nil {
 				errs.Appendf("error deleting media: %w", err)
 			}
 		}
 	} else {
 		// todo:u.state.DB.UnattachAttachmentsForStatus
-		for _, id := range status.AttachmentIDs {
+		for _, id := range status.AllAttachmentIDs() {
 			if _, err := u.media.Unattach(ctx, status.Account, id); err != nil {
 				errs.Appendf("error unattaching media: %w", err)
 			}
 		}
 	}
 
+	// Delete all historical edits of status.
+	if ids := status.EditIDs; len(ids) > 0 {
+		if err := u.state.DB.DeleteStatusEdits(ctx, ids); err != nil {
+			errs.Appendf("error deleting status edits: %w", err)
+		}
+	}
+
 	// Delete all mentions generated by this status.
 	// todo:u.state.DB.DeleteMentionsForStatus
 	for _, id := range status.MentionIDs {
@@ -120,19 +142,20 @@ func (u *utils) wipeStatus(
 		errs.Appendf("error deleting status faves: %w", err)
 	}
 
-	if pollID := status.PollID; pollID != "" {
+	if id := status.PollID; id != "" {
 		// Delete this poll by ID from the database.
-		if err := u.state.DB.DeletePollByID(ctx, pollID); err != nil {
+		if err := u.state.DB.DeletePollByID(ctx, id); err != nil {
 			errs.Appendf("error deleting status poll: %w", err)
 		}
 
 		// Cancel any scheduled expiry task for poll.
-		_ = u.state.Workers.Scheduler.Cancel(pollID)
+		_ = u.state.Workers.Scheduler.Cancel(id)
 	}
 
 	// Get all boost of this status so that we can
 	// delete those boosts + remove them from timelines.
 	boosts, err := u.state.DB.GetStatusBoosts(
+
 		// We MUST set a barebones context here,
 		// as depending on where it came from the
 		// original BoostOf may already be gone.
@@ -537,11 +560,7 @@ func (u *utils) requestFave(
 	}
 
 	// Create + store new interaction request.
-	req, err = typeutils.StatusFaveToInteractionRequest(ctx, fave)
-	if err != nil {
-		return gtserror.Newf("error creating interaction request: %w", err)
-	}
-
+	req = typeutils.StatusFaveToInteractionRequest(fave)
 	if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil {
 		return gtserror.Newf("db error storing interaction request: %w", err)
 	}
@@ -584,11 +603,7 @@ func (u *utils) requestReply(
 	}
 
 	// Create + store interaction request.
-	req, err = typeutils.StatusToInteractionRequest(ctx, reply)
-	if err != nil {
-		return gtserror.Newf("error creating interaction request: %w", err)
-	}
-
+	req = typeutils.StatusToInteractionRequest(reply)
 	if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil {
 		return gtserror.Newf("db error storing interaction request: %w", err)
 	}
@@ -631,11 +646,7 @@ func (u *utils) requestAnnounce(
 	}
 
 	// Create + store interaction request.
-	req, err = typeutils.StatusToInteractionRequest(ctx, boost)
-	if err != nil {
-		return gtserror.Newf("error creating interaction request: %w", err)
-	}
-
+	req = typeutils.StatusToInteractionRequest(boost)
 	if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil {
 		return gtserror.Newf("db error storing interaction request: %w", err)
 	}