Text duplication fix (#137)

* start testing text duplication

* tests

* fixes + tests
This commit is contained in:
Tobi Smethurst
2021-08-11 16:54:54 +02:00
committed by GitHub
parent 8330263965
commit 329a5e8144
13 changed files with 696 additions and 42 deletions

View File

@@ -39,39 +39,39 @@ func (p *processor) Create(account *gtsmodel.Account, application *gtsmodel.Appl
}
// check if replyToID is ok
if err := p.processReplyToID(form, account.ID, newStatus); err != nil {
if err := p.ProcessReplyToID(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// check if mediaIDs are ok
if err := p.processMediaIDs(form, account.ID, newStatus); err != nil {
if err := p.ProcessMediaIDs(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// check if visibility settings are ok
if err := p.processVisibility(form, account.Privacy, newStatus); err != nil {
if err := p.ProcessVisibility(form, account.Privacy, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// handle language settings
if err := p.processLanguage(form, account.Language, newStatus); err != nil {
if err := p.ProcessLanguage(form, account.Language, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// handle mentions
if err := p.processMentions(form, account.ID, newStatus); err != nil {
if err := p.ProcessMentions(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
if err := p.processTags(form, account.ID, newStatus); err != nil {
if err := p.ProcessTags(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
if err := p.processEmojis(form, account.ID, newStatus); err != nil {
if err := p.ProcessEmojis(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
if err := p.processContent(form, account.ID, newStatus); err != nil {
if err := p.ProcessContent(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}

View File

@@ -34,6 +34,19 @@ type Processor interface {
Unfave(account *gtsmodel.Account, targetStatusID string) (*apimodel.Status, gtserror.WithCode)
// Context returns the context (previous and following posts) from the given status ID
Context(account *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode)
/*
PROCESSING UTILS
*/
ProcessVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error
ProcessReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error
ProcessMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error
ProcessLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error
ProcessMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
ProcessTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
ProcessEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
ProcessContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
}
type processor struct {

View File

@@ -0,0 +1,54 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package status_test
import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// nolint
type StatusStandardTestSuite struct {
suite.Suite
config *config.Config
db db.DB
log *logrus.Logger
typeConverter typeutils.TypeConverter
fromClientAPIChan chan gtsmodel.FromClientAPI
// standard suite models
testTokens map[string]*oauth.Token
testClients map[string]*oauth.Client
testApplications map[string]*gtsmodel.Application
testUsers map[string]*gtsmodel.User
testAccounts map[string]*gtsmodel.Account
testAttachments map[string]*gtsmodel.MediaAttachment
testStatuses map[string]*gtsmodel.Status
testTags map[string]*gtsmodel.Tag
testMentions map[string]*gtsmodel.Mention
// module being tested
status status.Processor
}

View File

@@ -12,7 +12,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
func (p *processor) ProcessVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
// by default all flags are set to true
gtsAdvancedVis := &gtsmodel.VisibilityAdvanced{
Federated: true,
@@ -83,7 +83,7 @@ func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, a
return nil
}
func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
func (p *processor) ProcessReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
if form.InReplyToID == "" {
return nil
}
@@ -132,7 +132,7 @@ func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, th
return nil
}
func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
func (p *processor) ProcessMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
if form.MediaIDs == nil {
return nil
}
@@ -161,7 +161,7 @@ func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thi
return nil
}
func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
func (p *processor) ProcessLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
if form.Language != "" {
status.Language = form.Language
} else {
@@ -173,7 +173,7 @@ func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, acc
return nil
}
func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
func (p *processor) ProcessMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
menchies := []string{}
gtsMenchies, err := p.db.MentionStringsToMentions(util.DeriveMentionsFromStatus(form.Status), accountID, status.ID)
if err != nil {
@@ -198,7 +198,7 @@ func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, acc
return nil
}
func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
func (p *processor) ProcessTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
tags := []string{}
gtsTags, err := p.db.TagStringsToTags(util.DeriveHashtagsFromStatus(form.Status), accountID, status.ID)
if err != nil {
@@ -217,7 +217,7 @@ func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, account
return nil
}
func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
func (p *processor) ProcessEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
emojis := []string{}
gtsEmojis, err := p.db.EmojiStringsToEmojis(util.DeriveEmojisFromStatus(form.Status), accountID, status.ID)
if err != nil {
@@ -233,7 +233,7 @@ func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accou
return nil
}
func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
func (p *processor) ProcessContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
// if there's nothing in the status at all we can just return early
if form.Status == "" {
status.Content = ""
@@ -249,15 +249,16 @@ func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, acco
content := text.RemoveHTML(form.Status)
// parse content out of the status depending on what format has been submitted
var formatted string
switch form.Format {
case apimodel.StatusFormatPlain:
content = p.formatter.FromPlain(content, status.GTSMentions, status.GTSTags)
formatted = p.formatter.FromPlain(content, status.GTSMentions, status.GTSTags)
case apimodel.StatusFormatMarkdown:
content = p.formatter.FromMarkdown(content, status.GTSMentions, status.GTSTags)
formatted = p.formatter.FromMarkdown(content, status.GTSMentions, status.GTSTags)
default:
return fmt.Errorf("format %s not recognised as a valid status format", form.Format)
}
status.Content = content
status.Content = formatted
return nil
}

View File

@@ -0,0 +1,349 @@
package status_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
"github.com/superseriousbusiness/gotosocial/testrig"
)
const statusText1 = `Another test @foss_satan@fossbros-anonymous.io
#Hashtag
Text`
const statusText1ExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/>Text</p>`
const statusText1ExpectedPartial = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/>#Hashtag<br/><br/>Text</p>`
const statusText2 = `Another test @foss_satan@fossbros-anonymous.io
#Hashtag
#hashTAG`
const status2TextExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>hashTAG</span></a></p>`
type UtilTestSuite struct {
StatusStandardTestSuite
}
func (suite *UtilTestSuite) SetupSuite() {
suite.testTokens = testrig.NewTestTokens()
suite.testClients = testrig.NewTestClients()
suite.testApplications = testrig.NewTestApplications()
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.testAttachments = testrig.NewTestAttachments()
suite.testStatuses = testrig.NewTestStatuses()
suite.testTags = testrig.NewTestTags()
suite.testMentions = testrig.NewTestMentions()
}
func (suite *UtilTestSuite) SetupTest() {
suite.config = testrig.NewTestConfig()
suite.db = testrig.NewTestDB()
suite.log = testrig.NewTestLog()
suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
suite.fromClientAPIChan = make(chan gtsmodel.FromClientAPI, 100)
suite.status = status.New(suite.db, suite.typeConverter, suite.config, suite.fromClientAPIChan, suite.log)
testrig.StandardDBSetup(suite.db, nil)
}
func (suite *UtilTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
}
func (suite *UtilTestSuite) TestProcessMentions1() {
creatingAccount := suite.testAccounts["local_account_1"]
mentionedAccount := suite.testAccounts["remote_account_1"]
form := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: statusText1,
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatPlain,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
status := &gtsmodel.Status{
ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
}
err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Len(suite.T(), status.GTSMentions, 1)
newMention := status.GTSMentions[0]
assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
assert.Equal(suite.T(), status.ID, newMention.StatusID)
assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
assert.Equal(suite.T(), mentionedAccount.URI, newMention.MentionedAccountURI)
assert.Equal(suite.T(), mentionedAccount.URL, newMention.MentionedAccountURL)
assert.NotNil(suite.T(), newMention.GTSAccount)
assert.Len(suite.T(), status.Mentions, 1)
assert.Equal(suite.T(), newMention.ID, status.Mentions[0])
}
func (suite *UtilTestSuite) TestProcessContentFull1() {
/*
TEST PREPARATION
*/
// we need to partially process the status first since processContent expects a status with some stuff already set on it
creatingAccount := suite.testAccounts["local_account_1"]
form := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: statusText1,
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatPlain,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
status := &gtsmodel.Status{
ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
}
err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Empty(suite.T(), status.Content) // shouldn't be set yet
err = suite.status.ProcessTags(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Empty(suite.T(), status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), statusText1ExpectedFull, status.Content)
}
func (suite *UtilTestSuite) TestProcessContentPartial1() {
/*
TEST PREPARATION
*/
// we need to partially process the status first since processContent expects a status with some stuff already set on it
creatingAccount := suite.testAccounts["local_account_1"]
form := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: statusText1,
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatPlain,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
status := &gtsmodel.Status{
ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
}
err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Empty(suite.T(), status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), statusText1ExpectedPartial, status.Content)
}
func (suite *UtilTestSuite) TestProcessMentions2() {
creatingAccount := suite.testAccounts["local_account_1"]
mentionedAccount := suite.testAccounts["remote_account_1"]
form := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: statusText2,
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatPlain,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
status := &gtsmodel.Status{
ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
}
err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Len(suite.T(), status.GTSMentions, 1)
newMention := status.GTSMentions[0]
assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
assert.Equal(suite.T(), status.ID, newMention.StatusID)
assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
assert.Equal(suite.T(), mentionedAccount.URI, newMention.MentionedAccountURI)
assert.Equal(suite.T(), mentionedAccount.URL, newMention.MentionedAccountURL)
assert.NotNil(suite.T(), newMention.GTSAccount)
assert.Len(suite.T(), status.Mentions, 1)
assert.Equal(suite.T(), newMention.ID, status.Mentions[0])
}
func (suite *UtilTestSuite) TestProcessContentFull2() {
/*
TEST PREPARATION
*/
// we need to partially process the status first since processContent expects a status with some stuff already set on it
creatingAccount := suite.testAccounts["local_account_1"]
form := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: statusText2,
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatPlain,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
status := &gtsmodel.Status{
ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
}
err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Empty(suite.T(), status.Content) // shouldn't be set yet
err = suite.status.ProcessTags(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Empty(suite.T(), status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), status2TextExpectedFull, status.Content)
}
func (suite *UtilTestSuite) TestProcessContentPartial2() {
/*
TEST PREPARATION
*/
// we need to partially process the status first since processContent expects a status with some stuff already set on it
creatingAccount := suite.testAccounts["local_account_1"]
form := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: statusText2,
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatPlain,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
status := &gtsmodel.Status{
ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
}
err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
assert.Empty(suite.T(), status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(form, creatingAccount.ID, status)
assert.NoError(suite.T(), err)
fmt.Println(status.Content)
// assert.Equal(suite.T(), statusText2ExpectedPartial, status.Content)
}
func TestUtilTestSuite(t *testing.T) {
suite.Run(t, new(UtilTestSuite))
}