Prefer ICU to transliterate characters when available

Fixes #1008

.github/workflows/ccpp.yml

@@ -40,6 +40,7 @@ jobs:
             gstreamer-plugins-base-devel
             vlc-devel
             taglib-devel
+            libicu-devel
             libQt5Core-devel
             libQt5Gui-devel
             libQt5Gui-private-headers-devel
@@ -116,6 +117,7 @@ jobs:
             vlc-devel
             taglib-devel
             tagparser-devel
+            libicu-devel
             libQt5Core-devel
             libQt5Gui-devel
             libQt5Gui-private-headers-devel
@@ -202,6 +204,7 @@ jobs:
             vlc-devel
             taglib-devel
             tagparser-devel
+            libicu-devel
             qt6-core-devel
             qt6-gui-devel
             qt6-gui-private-devel
@@ -284,6 +287,7 @@ jobs:
             gstreamer-plugins-base-devel
             vlc-devel
             taglib-devel
+            libicu-devel
             libQt5Core-devel
             libQt5Gui-devel
             libQt5Gui-private-headers-devel
@@ -365,6 +369,7 @@ jobs:
             gstreamer-plugins-base-devel
             vlc-devel
             taglib-devel
+            libicu-devel
             qt6-core-devel
             qt6-gui-devel
             qt6-gui-private-devel
@@ -452,6 +457,7 @@ jobs:
             vlc-devel
             taglib-devel
             tagparser-devel
+            libicu-devel
             libQt5Core-devel
             libQt5Gui-devel
             libQt5Gui-private-headers-devel
@@ -538,6 +544,7 @@ jobs:
             vlc-devel
             taglib-devel
             tagparser-devel
+            libicu-devel
             qt6-core-devel
             qt6-gui-devel
             qt6-gui-private-devel
@@ -624,6 +631,7 @@ jobs:
             pulseaudio-libs-devel
             libnotify-devel
             gnutls-devel
+            libicu-devel
             qt6-qtbase-devel
             qt6-qtbase-private-devel
             qt6-qttools-devel
@@ -704,6 +712,7 @@ jobs:
             pulseaudio-libs-devel
             libnotify-devel
             gnutls-devel
+            libicu-devel
             qt6-qtbase-devel
             qt6-qtbase-private-devel
             qt6-qttools-devel
@@ -786,6 +795,7 @@ jobs:
           taglib-devel
           chromaprint-devel
           fftw-devel
+          icu-devel
           libcdio-devel
           libgpod-devel
           libmtp-devel
@@ -862,6 +872,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qtbase5-dev
             qtbase5-dev-tools
             qttools5-dev
@@ -924,6 +935,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qtbase5-dev
             qtbase5-dev-tools
             qttools5-dev
@@ -985,6 +997,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qt6-base-dev
             qt6-base-dev-tools
             qt6-tools-dev
@@ -1051,6 +1064,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qtbase5-dev
             qtbase5-dev-tools
             qttools5-dev
@@ -1118,6 +1132,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qtbase5-dev
             qtbase5-dev-tools
             qttools5-dev
@@ -1184,6 +1199,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qt6-base-dev
             qt6-base-dev-tools
             qt6-tools-dev
@@ -1250,6 +1266,7 @@ jobs:
             libasound2-dev
             libpulse-dev
             libtag1-dev
+            libicu-dev
             qt6-base-dev
             qt6-base-dev-tools
             qt6-tools-dev
@@ -1355,7 +1372,8 @@ jobs:
                      -DProtobuf_INCLUDE_DIRS=${{github.workspace}}/../../strawberry-macos-dependencies/strawberry-macos-dependencies/usr/include \
                      -DProtobuf_LIBRARY=${{github.workspace}}/../../strawberry-macos-dependencies/strawberry-macos-dependencies/usr/lib/libprotobuf.dylib \
                      -DProtobuf_PROTOC_EXECUTABLE=${{github.workspace}}/../../strawberry-macos-dependencies/strawberry-macos-dependencies/usr/bin/protoc \
-                     -DFFTW3_DIR=${{github.workspace}}/../../strawberry-macos-dependencies/strawberry-macos-dependencies/usr
+                     -DFFTW3_DIR=${{github.workspace}}/../../strawberry-macos-dependencies/strawberry-macos-dependencies/usr \
+                     -DICU_ROOT=${{github.workspace}}/../../strawberry-macos-dependencies/strawberry-macos-dependencies/usr
 
       - name: Build
         working-directory: build
@@ -2169,7 +2187,7 @@ jobs:
       with:
         usesh: true
         mem: 4096
-        prepare: pkg install -y git cmake pkgconf gettext-tools boost-libs glib gnutls qt5-core qt5-concurrent qt5-network qt5-sql qt5-dbus qt5-gui qt5-widgets qt5-buildtools qt5-linguisttools qt5-qmake qt5-sqldrivers-sqlite3 qt5-testlib sqlite gstreamer1 gstreamer1-plugins chromaprint protobuf protobuf-c taglib libcdio libmtp gdk-pixbuf2 libgpod fftw3 googletest iconv
+        prepare: pkg install -y git cmake pkgconf gettext-tools boost-libs glib gnutls qt5-core qt5-concurrent qt5-network qt5-sql qt5-dbus qt5-gui qt5-widgets qt5-buildtools qt5-linguisttools qt5-qmake qt5-sqldrivers-sqlite3 qt5-testlib sqlite gstreamer1 gstreamer1-plugins chromaprint protobuf protobuf-c taglib libcdio libmtp gdk-pixbuf2 libgpod fftw3 googletest iconv icu
         run: |
           git config --global --add safe.directory /__w/strawberry/strawberry
           cmake -E make_directory build

3rdparty/singleapplication/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 
 include(CheckIncludeFiles)
 include(CheckFunctionExists)

CMakeLists.txt

@@ -1,7 +1,10 @@
 project(strawberry)
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 cmake_policy(SET CMP0054 NEW)
+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12)
+  cmake_policy(SET CMP0074 NEW)
+endif()
 
 include(CheckCXXCompilerFlag)
 include(CheckCXXSourceRuns)
@@ -102,6 +105,10 @@ if(Backtrace_FOUND)
   set(HAVE_BACKTRACE ON)
 endif()
 find_package(Iconv)
+find_package(ICU COMPONENTS uc i18n)
+if(ICU_FOUND)
+  set(HAVE_ICU ON)
+endif()
 find_package(GnuTLS REQUIRED)
 find_package(Protobuf REQUIRED)
 if(NOT Protobuf_PROTOC_EXECUTABLE)

debian/control.in

@@ -17,6 +17,7 @@ Build-Depends: debhelper (>= 11),
                libasound2-dev,
                libpulse-dev,
                libtag1-dev,
+               libicu-devel,
                @DEBIAN_BUILD_DEPENDS_QT_PACKAGES@,
                libgstreamer1.0-dev,
                libgstreamer-plugins-base1.0-dev,

dist/unix/strawberry.spec.in

@@ -49,6 +49,8 @@ BuildRequires:  pkgconfig(sqlite3) >= 3.9
 BuildRequires:  pkgconfig(taglib)
 %endif
 BuildRequires:  pkgconfig(fftw3)
+BuildRequires:  pkgconfig(icu-uc)
+BuildRequires:  pkgconfig(icu-i18n)
 %if "@QT_VERSION_MAJOR@" == "5" && ( 0%{?fedora} || 0%{?rhel_version} || 0%{?centos} )
 BuildRequires:  pkgconfig(Qt@QT_VERSION_MAJOR@Core)
 BuildRequires:  pkgconfig(Qt@QT_VERSION_MAJOR@Gui)

ext/gstmoodbar/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 
 set(SOURCES gstfastspectrum.cpp gstmoodbarplugin.cpp)
 

ext/libstrawberry-common/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 
 set(SOURCES
   core/logging.cpp

ext/libstrawberry-tagreader/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 
 set(MESSAGES tagreadermessages.proto)
 set(SOURCES tagreaderbase.cpp)

ext/strawberry-tagreader/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 
 set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR})
 

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.7)
 
 if(HAVE_TRANSLATIONS)
   include(../cmake/Translations.cmake)
@@ -1145,6 +1145,11 @@ if(FREEBSD)
   target_link_libraries(strawberry_lib PRIVATE iconv)
 endif()
 
+if(HAVE_ICU)
+  target_include_directories(strawberry_lib SYSTEM PRIVATE ${ICU_INCLUDE_DIRS})
+  target_link_libraries(strawberry_lib PRIVATE ${ICU_LIBRARIES})
+endif()
+
 if(APPLE)
   target_link_libraries(strawberry_lib PRIVATE
     "-framework AppKit"

src/config.h.in

@@ -21,6 +21,8 @@
 #cmakedefine HAVE_MUSICBRAINZ
 #cmakedefine HAVE_GLOBALSHORTCUTS
 #cmakedefine HAVE_X11_GLOBALSHORTCUTS
+#cmakedefine HAVE_ICU
+
 #cmakedefine USE_INSTALL_PREFIX
 
 #cmakedefine HAVE_GSTREAMER

src/core/utilities.cpp

@@ -23,7 +23,12 @@
 
 #include <memory>
 #include <cstdlib>
-#include <iconv.h>
+
+#ifdef HAVE_ICU
+#  include <unicode/translit.h>
+#else
+#  include <iconv.h>
+#endif
 
 #include <QtGlobal>
 #include <QApplication>
@@ -786,16 +791,39 @@ QString DesktopEnvironment() {
 
 }
 
-QString UnicodeToAscii(const QString &unicode) {
+#ifdef HAVE_ICU
+
+QString Transliterate(const QString &accented_str) {
+
+  UErrorCode errorcode = U_ZERO_ERROR;
+  std::unique_ptr<icu::Transliterator> transliterator;
+  transliterator.reset(icu::Transliterator::createInstance("Any-Latin; Latin-ASCII;", UTRANS_FORWARD, errorcode));
+
+  if (!transliterator) return accented_str;
+
+  QByteArray accented_data = accented_str.toUtf8();
+  icu::UnicodeString ustring = icu::UnicodeString(accented_data.constData());
+  transliterator->transliterate(ustring);
+
+  std::string unaccented_str;
+  ustring.toUTF8String(unaccented_str);
+
+  return QString::fromStdString(unaccented_str);
+
+}
+
+#else
+
+QString Transliterate(const QString &accented_str) {
 
 #ifdef LC_ALL
   setlocale(LC_ALL, "");
 #endif
 
   iconv_t conv = iconv_open("ASCII//TRANSLIT", "UTF-8");
-  if (conv == reinterpret_cast<iconv_t>(-1)) return unicode;
+  if (conv == reinterpret_cast<iconv_t>(-1)) return accented_str;
 
-  QByteArray utf8 = unicode.toUtf8();
+  QByteArray utf8 = accented_str.toUtf8();
 
   size_t input_len = utf8.length() + 1;
   char *input_ptr = new char[input_len];
@@ -817,8 +845,11 @@ QString UnicodeToAscii(const QString &unicode) {
   delete[] output_ptr;
 
   return ret;
+
 }
 
+#endif
+
 QString MacAddress() {
 
   QString ret;

src/core/utilities.h

@@ -132,7 +132,7 @@ QString GetRandomString(const int len, const QString &UseCharacters);
 
 QString DesktopEnvironment();
 
-QString UnicodeToAscii(const QString &unicode);
+QString Transliterate(const QString &accented_str);
 
 QString MacAddress();
 

src/covermanager/albumcoverloader.cpp

@@ -110,7 +110,7 @@ QString AlbumCoverLoader::AlbumCoverFilename(QString artist, QString album, cons
   album.remove('/').remove('\\');
 
   QString filename = artist + "-" + album;
-  filename = Utilities::UnicodeToAscii(filename.toLower());
+  filename = Utilities::Transliterate(filename.toLower());
   filename = filename.replace(' ', '-')
                      .replace("--", "-")
                      .remove(OrganizeFormat::kInvalidFatCharacters)

src/organize/organizeformat.cpp

@@ -128,7 +128,7 @@ QString OrganizeFormat::GetFilenameForSong(const Song &song, QString extension)
   }
 
   if (remove_problematic_) filename = filename.remove(kProblematicCharacters);
-  if (remove_non_fat_ || (remove_non_ascii_ && !allow_ascii_ext_)) filename = Utilities::UnicodeToAscii(filename);
+  if (remove_non_fat_ || (remove_non_ascii_ && !allow_ascii_ext_)) filename = Utilities::Transliterate(filename);
   if (remove_non_fat_) filename = filename.remove(kInvalidFatCharacters);
 
   if (remove_non_ascii_) {

tests/src/organizeformat_test.cpp

@@ -176,8 +176,18 @@ TEST_F(OrganizeFormatTest, ReplaceNonAscii) {
   format_.set_remove_non_ascii(true);
   EXPECT_EQ("Royksopp", format_.GetFilenameForSong(song_));
 
+  song_.set_artist("");
+  EXPECT_EQ("", format_.GetFilenameForSong(song_));
+
+#ifdef HAVE_ICU
+
   song_.set_artist(QString::fromUtf8("Владимир Высоцкий"));
-  EXPECT_EQ("_________________", format_.GetFilenameForSong(song_));
+  EXPECT_EQ("Vladimir_Vysockij", format_.GetFilenameForSong(song_));
+
+  song_.set_artist(QString::fromUtf8("エックス・ジャパン"));
+  EXPECT_EQ("ekkusujapan", format_.GetFilenameForSong(song_));
+
+#endif
 
 }
 

tests/src/utilities_test.cpp

@@ -166,9 +166,9 @@ TEST(UtilitiesTest, Random) {
 
 }
 
-TEST(UtilitiesTest, UnicodeToAscii) {
+TEST(UtilitiesTest, Transliterate) {
 
-  ASSERT_EQ(Utilities::UnicodeToAscii("ÆØÅ"), "AEOA");
+  ASSERT_EQ(Utilities::Transliterate("ÆØÅ"), "AEOA");
 
 }