From af298e5281c14336c9087f79e10ddc583881209a Mon Sep 17 00:00:00 2001 From: Levi Bard Date: Fri, 19 Oct 2018 17:44:46 +0200 Subject: [PATCH] Account for underscores when tokenizing mentions for autocompletion (#888) * Account for underscores when tokenizing mentions for autocompletion Fixes #743 * Migrate MentionTokenizer to kotlin * Add tests for mention tokenizer --- .../tusky/util/MentionTokenizer.java | 67 ----------------- .../tusky/util/MentionTokenizer.kt | 72 +++++++++++++++++++ .../tusky/MentionTokenizerTest.kt | 61 ++++++++++++++++ 3 files changed, 133 insertions(+), 67 deletions(-) delete mode 100644 app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java create mode 100644 app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt create mode 100644 app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt diff --git a/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java b/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java deleted file mode 100644 index fd8748e32..000000000 --- a/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright 2017 Andrew Dawson - * - * This file is a part of Tusky. - * - * This program is free software; you can redistribute it and/or modify it under the terms of the - * GNU General Public License as published by the Free Software Foundation; either version 3 of the - * License, or (at your option) any later version. - * - * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even - * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General - * Public License for more details. - * - * You should have received a copy of the GNU General Public License along with Tusky; if not, - * see . */ - -package com.keylesspalace.tusky.util; - -import android.text.SpannableString; -import android.text.Spanned; -import android.text.TextUtils; -import android.widget.MultiAutoCompleteTextView; - -public class MentionTokenizer implements MultiAutoCompleteTextView.Tokenizer { - @Override - public int findTokenStart(CharSequence text, int cursor) { - int i = cursor; - while (i > 0 && text.charAt(i - 1) != '@') { - if (!Character.isLetterOrDigit(text.charAt(i - 1))) return cursor; - i--; - } - if (i < 1 || text.charAt(i - 1) != '@') { - return cursor; - } - return i; - } - - @Override - public int findTokenEnd(CharSequence text, int cursor) { - int i = cursor; - int length = text.length(); - while (i < length) { - if (text.charAt(i) == ' ') { - return i; - } else { - i++; - } - } - return length; - } - - @Override - public CharSequence terminateToken(CharSequence text) { - int i = text.length(); - while (i > 0 && text.charAt(i - 1) == ' ') { - i--; - } - if (i > 0 && text.charAt(i - 1) == ' ') { - return text; - } else if (text instanceof Spanned) { - SpannableString s = new SpannableString(text + " "); - TextUtils.copySpansFrom((Spanned) text, 0, text.length(), Object.class, s, 0); - return s; - } else { - return text + " "; - } - } -} diff --git a/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt b/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt new file mode 100644 index 000000000..cb996c0e6 --- /dev/null +++ b/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt @@ -0,0 +1,72 @@ +/* Copyright 2017 Andrew Dawson + * + * This file is a part of Tusky. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Tusky; if not, + * see . */ + +package com.keylesspalace.tusky.util + +import android.text.SpannableString +import android.text.Spanned +import android.text.TextUtils +import android.widget.MultiAutoCompleteTextView + +class MentionTokenizer : MultiAutoCompleteTextView.Tokenizer { + override fun findTokenStart(text: CharSequence, cursor: Int): Int { + if (cursor == 0) { + return cursor + } + var i = cursor + var character = text[i - 1] + while (i > 0 && character != '@') { + // See SpanUtils.MENTION_REGEX + if (!Character.isLetterOrDigit(character) && character != '_') { + return cursor + } + i-- + character = if (i == 0) ' ' else text[i - 1] + } + if (i < 1 || character != '@') { + return cursor + } + return i + } + + override fun findTokenEnd(text: CharSequence, cursor: Int): Int { + var i = cursor + val length = text.length + while (i < length) { + if (text[i] == ' ') { + return i + } else { + i++ + } + } + return length + } + + override fun terminateToken(text: CharSequence): CharSequence { + var i = text.length + while (i > 0 && text[i - 1] == ' ') { + i-- + } + return if (i > 0 && text[i - 1] == ' ') { + text + } else if (text is Spanned) { + val s = SpannableString(text.toString() + " ") + TextUtils.copySpansFrom(text, 0, text.length, Object::class.java, s, 0) + s + } else { + text.toString() + " " + } + } +} diff --git a/app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt b/app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt new file mode 100644 index 000000000..dbcdc53f5 --- /dev/null +++ b/app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt @@ -0,0 +1,61 @@ +/* Copyright 2018 Levi Bard + * + * This file is a part of Tusky. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Tusky; if not, + * see . */ + +package com.keylesspalace.tusky + +import com.keylesspalace.tusky.util.MentionTokenizer +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized + +@RunWith(Parameterized::class) +class MentionTokenizerTest(private val text: CharSequence, + private val expectedStartIndex: Int, + private val expectedEndIndex: Int) { + + companion object { + @Parameterized.Parameters(name = "{0}") + @JvmStatic + fun data(): Iterable { + return listOf( + arrayOf("@mention", 1, 8), + arrayOf("@ment10n", 1, 8), + arrayOf("@ment10n_", 1, 9), + arrayOf("@ment10n_n", 1, 10), + arrayOf("@ment10n_9", 1, 10), + arrayOf(" @mention", 2, 9), + arrayOf(" @ment10n", 2, 9), + arrayOf(" @ment10n_", 2, 10), + arrayOf(" @ment10n_ @", 12, 12), + arrayOf(" @ment10n_ @ment20n", 12, 19), + arrayOf(" @ment10n_ @ment20n_", 12, 20), + arrayOf(" @ment10n_ @ment20n_n", 12, 21), + arrayOf(" @ment10n_ @ment20n_9", 12, 21), + arrayOf("mention", 7, 7), + arrayOf("ment10n", 7, 7), + arrayOf("mentio_", 7, 7) + ) + } + } + + private val tokenizer = MentionTokenizer() + + @Test + fun tokenIndices_matchExpectations() { + Assert.assertEquals(expectedStartIndex, tokenizer.findTokenStart(text, text.length)) + Assert.assertEquals(expectedEndIndex, tokenizer.findTokenEnd(text, text.length)) + } +} \ No newline at end of file