Text duplication fix (#137)

* start testing text duplication

* tests

* fixes + tests
This commit is contained in:
Tobi Smethurst
2021-08-11 16:54:54 +02:00
committed by GitHub
parent 8330263965
commit 329a5e8144
13 changed files with 696 additions and 42 deletions

View File

@@ -46,7 +46,7 @@ func DeriveHashtagsFromStatus(status string) []string {
for _, m := range HashtagFinderRegex.FindAllStringSubmatch(status, -1) {
tags = append(tags, strings.TrimPrefix(m[1], "#"))
}
return uniqueLower(tags)
return unique(tags)
}
// DeriveEmojisFromStatus takes a plaintext (ie., not html-formatted) status,
@@ -92,17 +92,3 @@ func unique(s []string) []string {
}
return list
}
// uniqueLower returns a deduplicated version of a given string slice, with all entries converted to lowercase
func uniqueLower(s []string) []string {
keys := make(map[string]bool)
list := []string{}
for _, entry := range s {
eLower := strings.ToLower(entry)
if _, value := keys[eLower]; !value {
keys[eLower] = true
list = append(list, eLower)
}
}
return list
}