diff --git a/src/core/database.cpp b/src/core/database.cpp index 5571689a4..4abed97ef 100644 --- a/src/core/database.cpp +++ b/src/core/database.cpp @@ -129,15 +129,17 @@ int Database::FTSOpen( for (int i = 0; i < str.length(); ++i) { QChar c = data[i]; ushort unicode = c.unicode(); - if (unicode >= 0x00 && unicode <= 0x007f) { + if (unicode <= 0x007f) { offset += 1; } else if (unicode >= 0x0080 && unicode <= 0x07ff) { offset += 2; - } else if (unicode >= 0x0800 && unicode <= 0xffff) { + } else if (unicode >= 0x0800) { offset += 3; - } else if (unicode >= 0x010000 && unicode <= 0x10ffff) { - offset += 4; } + // Unicode astral planes unsupported in Qt? + /*else if (unicode >= 0x010000 && unicode <= 0x10ffff) { + offset += 4; + }*/ if (!data[i].isLetterOrNumber()) { // Token finished. diff --git a/src/core/database.h b/src/core/database.h index f89278eec..c04a133fc 100644 --- a/src/core/database.h +++ b/src/core/database.h @@ -78,6 +78,7 @@ class Database : public QObject { FRIEND_TEST(DatabaseTest, FTSOpenParsesUTF8Input); FRIEND_TEST(DatabaseTest, FTSOpenParsesMultipleTokens); FRIEND_TEST(DatabaseTest, FTSCursorWorks); + FRIEND_TEST(DatabaseTest, FTSOpenLeavesCyrillicQueries); // Do static initialisation like loading sqlite functions. static void StaticInit(); diff --git a/tests/database_test.cpp b/tests/database_test.cpp index 87f9e7c8c..b5f9dedb0 100644 --- a/tests/database_test.cpp +++ b/tests/database_test.cpp @@ -181,6 +181,22 @@ TEST_F(DatabaseTest, FTSOpenParsesMultipleTokens) { EXPECT_EQ(13, tokens[1].end_offset); } +TEST_F(DatabaseTest, FTSOpenLeavesCyrillicQueries) { + sqlite3_tokenizer_cursor* cursor = NULL; + const char* query = "Снег"; + Database::FTSOpen(NULL, query, strlen(query), &cursor); + ASSERT_TRUE(cursor); + Database::UnicodeTokenizerCursor* real_cursor = reinterpret_cast(cursor); + QList tokens = real_cursor->tokens; + ASSERT_EQ(1, tokens.length()); + EXPECT_EQ(0, real_cursor->position); + EXPECT_TRUE(real_cursor->current_utf8.isEmpty()); + + EXPECT_EQ(QString::fromUtf8("снег"), tokens[0].token); + EXPECT_EQ(0, tokens[0].start_offset); + EXPECT_EQ(strlen(query), tokens[0].end_offset); +} + TEST_F(DatabaseTest, FTSCursorWorks) { sqlite3_tokenizer_cursor* cursor = NULL; Database::FTSOpen(NULL, "Röyksopp foo", 13, &cursor);