Funkwhale/Clementine-audio-player-Mac-Linux-Windows
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Tidy unicode length detection & add Cyrillic FTS test.

Browse Source
This commit is contained in:
John Maguire 2010-06-21 12:07:56 +00:00
parent f544e28051
commit b91a61c260
3 changed files with 23 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/core/database.cpp
View File

@ -129,15 +129,17 @@ int Database::FTSOpen(
for (int i = 0; i < str.length(); ++i) { for (int i = 0; i < str.length(); ++i) {
QChar c = data[i]; QChar c = data[i];
ushort unicode = c.unicode(); ushort unicode = c.unicode();
if (unicode >= 0x00 && unicode <= 0x007f) { if (unicode <= 0x007f) {
offset += 1; offset += 1;
} else if (unicode >= 0x0080 && unicode <= 0x07ff) { } else if (unicode >= 0x0080 && unicode <= 0x07ff) {
offset += 2; offset += 2;
} else if (unicode >= 0x0800 && unicode <= 0xffff) { } else if (unicode >= 0x0800) {
offset += 3; offset += 3;
} else if (unicode >= 0x010000 && unicode <= 0x10ffff) {
offset += 4;
} }
// Unicode astral planes unsupported in Qt?
/*else if (unicode >= 0x010000 && unicode <= 0x10ffff) {
offset += 4;
}*/
if (!data[i].isLetterOrNumber()) { if (!data[i].isLetterOrNumber()) {
// Token finished. // Token finished.

1
src/core/database.h
View File

@ -78,6 +78,7 @@ class Database : public QObject {
FRIEND_TEST(DatabaseTest, FTSOpenParsesUTF8Input); FRIEND_TEST(DatabaseTest, FTSOpenParsesUTF8Input);
FRIEND_TEST(DatabaseTest, FTSOpenParsesMultipleTokens); FRIEND_TEST(DatabaseTest, FTSOpenParsesMultipleTokens);
FRIEND_TEST(DatabaseTest, FTSCursorWorks); FRIEND_TEST(DatabaseTest, FTSCursorWorks);
FRIEND_TEST(DatabaseTest, FTSOpenLeavesCyrillicQueries);
// Do static initialisation like loading sqlite functions. // Do static initialisation like loading sqlite functions.
static void StaticInit(); static void StaticInit();

16
tests/database_test.cpp
View File

@ -181,6 +181,22 @@ TEST_F(DatabaseTest, FTSOpenParsesMultipleTokens) {
EXPECT_EQ(13, tokens[1].end_offset); EXPECT_EQ(13, tokens[1].end_offset);
} }
TEST_F(DatabaseTest, FTSOpenLeavesCyrillicQueries) {
sqlite3_tokenizer_cursor* cursor = NULL;
const char* query = "Снег";
Database::FTSOpen(NULL, query, strlen(query), &cursor);
ASSERT_TRUE(cursor);
Database::UnicodeTokenizerCursor* real_cursor = reinterpret_cast<Database::UnicodeTokenizerCursor*>(cursor);
QList<Database::Token> tokens = real_cursor->tokens;
ASSERT_EQ(1, tokens.length());
EXPECT_EQ(0, real_cursor->position);
EXPECT_TRUE(real_cursor->current_utf8.isEmpty());
EXPECT_EQ(QString::fromUtf8("снег"), tokens[0].token);
EXPECT_EQ(0, tokens[0].start_offset);
EXPECT_EQ(strlen(query), tokens[0].end_offset);
}
TEST_F(DatabaseTest, FTSCursorWorks) { TEST_F(DatabaseTest, FTSCursorWorks) {
sqlite3_tokenizer_cursor* cursor = NULL; sqlite3_tokenizer_cursor* cursor = NULL;
Database::FTSOpen(NULL, "Röyksopp foo", 13, &cursor); Database::FTSOpen(NULL, "Röyksopp foo", 13, &cursor);