[feature] Allow full BCP 47 in language inputs (#2067)

* Allow full BCP 47 in language inputs

Fixes #2066

* Fuse validation and normalization for languages

* Remove outdated comment line

* Move post language canonicalization test
This commit is contained in:
Vyr Cossont 2023-08-07 01:25:54 -07:00 committed by GitHub
parent 303a6a6b1d
commit 0f812746b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 132 additions and 70 deletions

View File

@ -87,7 +87,7 @@ func (m *Module) AccountCreatePOSTHandler(c *gin.Context) {
return
}
if err := validateCreateAccount(form); err != nil {
if err := validateNormalizeCreateAccount(form); err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1)
return
}
@ -110,9 +110,10 @@ func (m *Module) AccountCreatePOSTHandler(c *gin.Context) {
c.JSON(http.StatusOK, ti)
}
// validateCreateAccount checks through all the necessary prerequisites for creating a new account,
// validateNormalizeCreateAccount checks through all the necessary prerequisites for creating a new account,
// according to the provided account create request. If the account isn't eligible, an error will be returned.
func validateCreateAccount(form *apimodel.AccountCreateRequest) error {
// Side effect: normalizes the provided language tag for the user's locale.
func validateNormalizeCreateAccount(form *apimodel.AccountCreateRequest) error {
if form == nil {
return errors.New("form was nil")
}
@ -137,9 +138,11 @@ func validateCreateAccount(form *apimodel.AccountCreateRequest) error {
return errors.New("agreement to terms and conditions not given")
}
if err := validate.Language(form.Locale); err != nil {
locale, err := validate.Language(form.Locale)
if err != nil {
return err
}
form.Locale = locale
return validate.SignUpReason(form.Reason, config.GetAccountsReasonRequired())
}

View File

@ -98,7 +98,7 @@ func (m *Module) StatusCreatePOSTHandler(c *gin.Context) {
// }
// form.Status += "\n\nsent from " + user + "'s iphone\n"
if err := validateCreateStatus(form); err != nil {
if err := validateNormalizeCreateStatus(form); err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1)
return
}
@ -112,7 +112,9 @@ func (m *Module) StatusCreatePOSTHandler(c *gin.Context) {
c.JSON(http.StatusOK, apiStatus)
}
func validateCreateStatus(form *apimodel.AdvancedStatusCreateForm) error {
// validateNormalizeCreateStatus checks the form for disallowed combinations of attachments and overlength inputs.
// Side effect: normalizes the post's language tag.
func validateNormalizeCreateStatus(form *apimodel.AdvancedStatusCreateForm) error {
hasStatus := form.Status != ""
hasMedia := len(form.MediaIDs) != 0
hasPoll := form.Poll != nil
@ -162,9 +164,11 @@ func validateCreateStatus(form *apimodel.AdvancedStatusCreateForm) error {
}
if form.Language != "" {
if err := validate.Language(form.Language); err != nil {
language, err := validate.Language(form.Language)
if err != nil {
return err
}
form.Language = language
}
return nil

View File

@ -391,6 +391,42 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
suite.Equal(statusResponse.ID, gtsAttachment.StatusID)
}
// Post a new status with a language tag that is not in canonical format
func (suite *StatusCreateTestSuite) TestPostNewStatusWithNoncanonicalLanguageTag() {
t := suite.testTokens["local_account_1"]
oauthToken := oauth.DBTokenToToken(t)
// setup
recorder := httptest.NewRecorder()
ctx, _ := testrig.CreateGinTestContext(recorder, nil)
ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", statuses.BasePath), nil) // the endpoint we're hitting
ctx.Request.Header.Set("accept", "application/json")
ctx.Request.Form = url.Values{
"status": {"English? what's English? i speak American"},
"language": {"en-us"},
}
suite.statusModule.StatusCreatePOSTHandler(ctx)
suite.EqualValues(http.StatusOK, recorder.Code)
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
suite.NoError(err)
statusReply := &apimodel.Status{}
err = json.Unmarshal(b, statusReply)
suite.NoError(err)
suite.Equal("<p>English? what's English? i speak American</p>", statusReply.Content)
suite.NotNil(statusReply.Language)
suite.Equal("en-US", *statusReply.Language)
}
func TestStatusCreateTestSuite(t *testing.T) {
suite.Run(t, new(StatusCreateTestSuite))
}

View File

@ -34,8 +34,7 @@ import (
// Create processes the given form for creating a new account,
// returning an oauth token for that account if successful.
//
// Fields on the form should have already been validated by the
// caller, before this function is called.
// Precondition: the form's fields should have already been validated and normalized by the caller.
func (p *Processor) Create(
ctx context.Context,
appToken oauth2.TokenInfo,

View File

@ -222,10 +222,11 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form
if form.Source != nil {
if form.Source.Language != nil {
if err := validate.Language(*form.Source.Language); err != nil {
language, err := validate.Language(*form.Source.Language)
if err != nil {
return nil, gtserror.NewErrorBadRequest(err)
}
account.Language = *form.Source.Language
account.Language = language
}
if form.Source.Sensitive != nil {

View File

@ -37,6 +37,8 @@ import (
)
// Create processes the given form to create a new status, returning the api model representation of that status if it's OK.
//
// Precondition: the form's fields should have already been validated and normalized by the caller.
func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, application *gtsmodel.Application, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, gtserror.WithCode) {
accountURIs := uris.GenerateURIsForAccount(account.Username)
thisStatusID := id.NewULID()
@ -55,7 +57,6 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, appli
ContentWarning: text.SanitizePlaintext(form.SpoilerText),
ActivityStreamsType: ap.ObjectNote,
Sensitive: &sensitive,
Language: form.Language,
CreatedWithApplicationID: application.ID,
Text: form.Status,
}

View File

@ -208,6 +208,40 @@ func (suite *StatusCreateTestSuite) TestProcessMediaDescriptionTooShort() {
suite.Nil(apiStatus)
}
func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() {
ctx := context.Background()
creatingAccount := suite.testAccounts["local_account_1"]
creatingApplication := suite.testApplications["application_1"]
statusCreateForm := &apimodel.AdvancedStatusCreateForm{
StatusCreateRequest: apimodel.StatusCreateRequest{
Status: "你好世界", // hello world
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
SpoilerText: "",
Visibility: apimodel.VisibilityPublic,
ScheduledAt: "",
Language: "zh-Hans",
ContentType: apimodel.StatusContentTypePlain,
},
AdvancedVisibilityFlagsForm: apimodel.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm)
suite.NoError(err)
suite.NotNil(apiStatus)
suite.Equal("zh-Hans", *apiStatus.Language)
}
func TestStatusCreateTestSuite(t *testing.T) {
suite.Run(t, new(StatusCreateTestSuite))
}

View File

@ -99,14 +99,19 @@ func Email(email string) error {
return err
}
// Language checks that the given language string is a 2- or 3-letter ISO 639 code.
// Returns an error if the language cannot be parsed. See: https://pkg.go.dev/golang.org/x/text/language
func Language(lang string) error {
// Language checks that the given language string is a valid, if not necessarily canonical, BCP 47 language tag.
// Returns a canonicalized version of the tag if the language can be parsed.
// Returns an error if the language cannot be parsed.
// See: https://pkg.go.dev/golang.org/x/text/language
func Language(lang string) (string, error) {
if lang == "" {
return errors.New("no language provided")
return "", errors.New("no language provided")
}
_, err := language.ParseBase(lang)
return err
parsed, err := language.Parse(lang)
if err != nil {
return "", err
}
return parsed.String(), err
}
// SignUpReason checks that a sufficient reason is given for a server signup request

View File

@ -159,60 +159,39 @@ func (suite *ValidationTestSuite) TestValidateEmail() {
}
func (suite *ValidationTestSuite) TestValidateLanguage() {
empty := ""
notALanguage := "this isn't a language at all!"
english := "en"
capitalEnglish := "EN"
arabic3Letters := "ara"
mixedCapsEnglish := "eN"
englishUS := "en-us"
dutch := "nl"
german := "de"
var err error
err = validate.Language(empty)
if suite.Error(err) {
suite.Equal(errors.New("no language provided"), err)
testCases := []struct {
name, input, expected, err string
}{
{name: "empty", err: "no language provided"},
{name: "notALanguage", input: "this isn't a language at all!", err: "language: tag is not well-formed"},
{name: "english", input: "en", expected: "en"},
// Should be all lowercase
{name: "capitalEnglish", input: "EN", expected: "en"},
// Overlong, should be in ISO 639-1 format
{name: "arabic3Letters", input: "ara", expected: "ar"},
// Should be all lowercase
{name: "mixedCapsEnglish", input: "eN", expected: "en"},
// Region should be capitalized
{name: "englishUS", input: "en-us", expected: "en-US"},
{name: "dutch", input: "nl", expected: "nl"},
{name: "german", input: "de", expected: "de"},
{name: "chinese", input: "zh", expected: "zh"},
{name: "chineseSimplified", input: "zh-Hans", expected: "zh-Hans"},
{name: "chineseTraditional", input: "zh-Hant", expected: "zh-Hant"},
}
err = validate.Language(notALanguage)
if suite.Error(err) {
suite.Equal(errors.New("language: tag is not well-formed"), err)
}
err = validate.Language(english)
if suite.NoError(err) {
suite.Equal(nil, err)
}
err = validate.Language(capitalEnglish)
if suite.NoError(err) {
suite.Equal(nil, err)
}
err = validate.Language(arabic3Letters)
if suite.NoError(err) {
suite.Equal(nil, err)
}
err = validate.Language(mixedCapsEnglish)
if suite.NoError(err) {
suite.Equal(nil, err)
}
err = validate.Language(englishUS)
if suite.Error(err) {
suite.Equal(errors.New("language: tag is not well-formed"), err)
}
err = validate.Language(dutch)
if suite.NoError(err) {
suite.Equal(nil, err)
}
err = validate.Language(german)
if suite.NoError(err) {
suite.Equal(nil, err)
for _, testCase := range testCases {
testCase := testCase
suite.Run(testCase.name, func() {
actual, actualErr := validate.Language(testCase.input)
if testCase.err == "" {
suite.Equal(testCase.expected, actual)
suite.NoError(actualErr)
} else {
suite.Empty(actual)
suite.EqualError(actualErr, testCase.err)
}
})
}
}