Add support for ignoring diacritics while searching and filtering lists

NEWS

@@ -30,7 +30,8 @@ ncmpcpp-0.8 (????-??-??)
 * Disable autocenter mode while searching and filtering.
 * Added '--quiet' comand line argument that supresses messages shown at startup.
 * Multiple songs in Media library are now added to playlist in the same order they are displayed.
-* Added configuration options 'media_library_albums_split_by_date' that determines whether albums in media library should be split by date.
+* Added configuration option 'media_library_albums_split_by_date' that determines whether albums in media library should be split by date.
+* Added configuration option 'ignore_diacritics' that allows for ignoring diacritics while searching (boost compiled with ICU support is required).
 
 ncmpcpp-0.7.7 (2016-10-31)
 * Fixed compilation on 32bit platforms.

configure.ac

@@ -108,7 +108,7 @@ dnl ====================
 dnl = checking for icu =
 dnl ====================
 AH_TEMPLATE([BOOST_REGEX_ICU], [boost.regex was compiled with ICU support])
-PKG_CHECK_MODULES([ICU], [icu-uc], [
+PKG_CHECK_MODULES([ICU], [icu-i18n icu-uc], [
 	old_CPPFLAGS="$CPPFLAGS"
 	old_LIBS="$LIBS"
 	AC_SUBST(ICU_CFLAGS)

doc/config

@@ -421,11 +421,18 @@
 #regular_expressions = perl
 #
 ##
-## Note: If below is enabled, ncmpcpp will ignore leading "The" word while
+## Note: if below is enabled, ncmpcpp will ignore leading "The" word while
 ## sorting items in browser, tags in media library, etc.
 ##
 #ignore_leading_the = no
 #
+##
+## Note: if below is enabled, ncmpcpp will ignore diacritics while searching and
+## filtering lists. This takes an effect only if boost was compiled with ICU
+## support.
+##
+#ignore_diacritics = no
+#
 #block_search_constraints_change_if_items_found = yes
 #
 #mouse_support = yes

doc/ncmpcpp.1

@@ -315,6 +315,9 @@ Type of currently used regular expressions.
 .B ignore_leading_the = yes/no
 If enabled, word "the" at the beginning of tags/filenames/sort format will be ignored while sorting items.
 .TP
+.B ignore_diacritics = yes/no
+If enabled, diacritics in strings will be ignored while searching and filtering lists.
+.TP
 .B block_search_constraints_change_if_items_found = yes/no
 If enabled, fields in Search engine above "Reset" button will be blocked after successful searching, otherwise they won't.
 .TP

src/regex_filter.h

@@ -25,6 +25,8 @@
 
 #ifdef BOOST_REGEX_ICU
 # include <boost/regex/icu.hpp>
+# include <unicode/errorcode.h>
+# include <unicode/translit.h>
 #else
 # include <boost/regex.hpp>
 #endif // BOOST_REGEX_ICU
@@ -32,6 +34,39 @@
 #include <cassert>
 #include <iostream>
 
+#include "utility/functional.h"
+
+namespace {
+
+#ifdef BOOST_REGEX_ICU
+
+struct StripDiacritics
+{
+	static void convert(UnicodeString &s)
+	{
+		if (m_converter == nullptr)
+		{
+			ErrorCode result;
+			m_converter = Transliterator::createInstance(
+				"NFD; [:M:] Remove; NFC", UTRANS_FORWARD, result);
+			if (result.isFailure())
+				throw std::runtime_error(
+					"instantiation of transliterator instance failed with "
+					+ std::string(result.errorName()));
+		}
+		m_converter->transliterate(s);
+	}
+
+private:
+	static Transliterator *m_converter;
+};
+
+Transliterator *StripDiacritics::m_converter;
+
+#endif // BOOST_REGEX_ICU
+
+}
+
 namespace Regex {
 
 typedef
@@ -43,31 +78,44 @@ typedef
 Regex;
 
 template <typename StringT>
-inline Regex make(StringT &&s, boost::regex_constants::syntax_option_type flags)
+inline Regex make(StringT &&s,
+                  boost::regex_constants::syntax_option_type flags)
 {
 	return
-#	ifdef BOOST_REGEX_ICU
+#ifdef BOOST_REGEX_ICU
 	boost::make_u32regex
-#	else
+#else
 	boost::regex
-#	endif // BOOST_REGEX_ICU
+#endif // BOOST_REGEX_ICU
 	(std::forward<StringT>(s), flags);
 }
 
-template <typename StringT>
-inline bool search(StringT &&s, const Regex &rx)
+template <typename CharT>
+inline bool search(const std::basic_string<CharT> &s,
+                   const Regex &rx,
+                   bool ignore_diacritics)
 {
 	try {
-		return
-#		ifdef BOOST_REGEX_ICU
-		boost::u32regex_search
-#		else
-		boost::regex_search
-#		endif // BOOST_REGEX_ICU
-		(std::forward<StringT>(s), rx);
+#ifdef BOOST_REGEX_ICU
+		if (ignore_diacritics)
+		{
+			auto us = UnicodeString::fromUTF8(
+				StringPiece(convertString<char, CharT>::apply(s)));
+			StripDiacritics::convert(us);
+			return boost::u32regex_search(us, rx);
+		}
+		else
+			return boost::u32regex_search(s, rx);
+#else
+		return boost::regex_search(s, rx);
+#endif // BOOST_REGEX_ICU
 	} catch (std::out_of_range &e) {
 		// Invalid UTF-8 sequence, ignore the string.
-		std::cerr << "Regex::search: error while processing \"" << s << "\": " << e.what() << "\n";
+		std::cerr << "Regex::search: error while processing \""
+		          << s
+		          << "\": "
+		          << e.what()
+		          << "\n";
 		return false;
 	}
 }

src/screens/browser.cpp

@@ -734,7 +734,7 @@ bool browserEntryMatcher(const Regex::Regex &rx, const MPD::Item &item, bool fil
 {
 	if (isItemParentDirectory(item))
 		return filter;
-	return Regex::search(itemToString(item), rx);
+	return Regex::search(itemToString(item), rx, Config.ignore_diacritics);
 }
 
 }

src/screens/media_library.cpp

@@ -1086,19 +1086,19 @@ std::string SongToString(const MPD::Song &s)
 
 bool TagEntryMatcher(const Regex::Regex &rx, const PrimaryTag &pt)
 {
-	return Regex::search(pt.tag(), rx);
+	return Regex::search(pt.tag(), rx, Config.ignore_diacritics);
 }
 
 bool AlbumEntryMatcher(const Regex::Regex &rx, const NC::Menu<AlbumEntry>::Item &item, bool filter)
 {
 	if (item.isSeparator() || item.value().isAllTracksEntry())
 		return filter;
-	return Regex::search(AlbumToString(item.value()), rx);
+	return Regex::search(AlbumToString(item.value()), rx, Config.ignore_diacritics);
 }
 
 bool SongEntryMatcher(const Regex::Regex &rx, const MPD::Song &s)
 {
-	return Regex::search(SongToString(s), rx);
+	return Regex::search(SongToString(s), rx, Config.ignore_diacritics);
 }
 
 bool MoveToTag(NC::Menu<PrimaryTag> &tags, const std::string &primary_tag)

src/screens/playlist.cpp

@@ -353,7 +353,7 @@ std::string songToString(const MPD::Song &s)
 
 bool playlistEntryMatcher(const Regex::Regex &rx, const MPD::Song &s)
 {
-	return Regex::search(songToString(s), rx);
+	return Regex::search(songToString(s), rx, Config.ignore_diacritics);
 }
 
 }

src/screens/playlist_editor.cpp

@@ -558,12 +558,12 @@ std::string SongToString(const MPD::Song &s)
 
 bool PlaylistEntryMatcher(const Regex::Regex &rx, const MPD::Playlist &playlist)
 {
-	return Regex::search(playlist.path(), rx);
+	return Regex::search(playlist.path(), rx, Config.ignore_diacritics);
 }
 
 bool SongEntryMatcher(const Regex::Regex &rx, const MPD::Song &s)
 {
-	return Regex::search(SongToString(s), rx);
+	return Regex::search(SongToString(s), rx, Config.ignore_diacritics);
 }
 
 boost::optional<size_t> GetSongIndexInPlaylist(MPD::Playlist playlist, const MPD::Song &song)

src/screens/search_engine.cpp

@@ -517,36 +517,36 @@ void SearchEngine::Search()
 		{
 			if (!rx[0].empty())
 				any_found =
-				   Regex::search(s->getArtist(), rx[0])
-				|| Regex::search(s->getAlbumArtist(), rx[0])
-				|| Regex::search(s->getTitle(), rx[0])
-				|| Regex::search(s->getAlbum(), rx[0])
-				|| Regex::search(s->getName(), rx[0])
-				|| Regex::search(s->getComposer(), rx[0])
-				|| Regex::search(s->getPerformer(), rx[0])
-				|| Regex::search(s->getGenre(), rx[0])
-				|| Regex::search(s->getDate(), rx[0])
-				|| Regex::search(s->getComment(), rx[0]);
+					   Regex::search(s->getArtist(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getAlbumArtist(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getTitle(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getAlbum(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getName(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getComposer(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getPerformer(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getGenre(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getDate(), rx[0], Config.ignore_diacritics)
+					|| Regex::search(s->getComment(), rx[0], Config.ignore_diacritics);
 			if (found && !rx[1].empty())
-				found = Regex::search(s->getArtist(), rx[1]);
+				found = Regex::search(s->getArtist(), rx[1], Config.ignore_diacritics);
 			if (found && !rx[2].empty())
-				found = Regex::search(s->getAlbumArtist(), rx[2]);
+				found = Regex::search(s->getAlbumArtist(), rx[2], Config.ignore_diacritics);
 			if (found && !rx[3].empty())
-				found = Regex::search(s->getTitle(), rx[3]);
+				found = Regex::search(s->getTitle(), rx[3], Config.ignore_diacritics);
 			if (found && !rx[4].empty())
-				found = Regex::search(s->getAlbum(), rx[4]);
+				found = Regex::search(s->getAlbum(), rx[4], Config.ignore_diacritics);
 			if (found && !rx[5].empty())
-				found = Regex::search(s->getName(), rx[5]);
+				found = Regex::search(s->getName(), rx[5], Config.ignore_diacritics);
 			if (found && !rx[6].empty())
-				found = Regex::search(s->getComposer(), rx[6]);
+				found = Regex::search(s->getComposer(), rx[6], Config.ignore_diacritics);
 			if (found && !rx[7].empty())
-				found = Regex::search(s->getPerformer(), rx[7]);
+				found = Regex::search(s->getPerformer(), rx[7], Config.ignore_diacritics);
 			if (found && !rx[8].empty())
-				found = Regex::search(s->getGenre(), rx[8]);
+				found = Regex::search(s->getGenre(), rx[8], Config.ignore_diacritics);
 			if (found && !rx[9].empty())
-				found = Regex::search(s->getDate(), rx[9]);
+				found = Regex::search(s->getDate(), rx[9], Config.ignore_diacritics);
 			if (found && !rx[10].empty())
-				found = Regex::search(s->getComment(), rx[10]);
+				found = Regex::search(s->getComment(), rx[10], Config.ignore_diacritics);
 		}
 		else // match only if values are equal
 		{
@@ -616,7 +616,7 @@ bool SEItemEntryMatcher(const Regex::Regex &rx, const NC::Menu<SEItem>::Item &it
 {
 	if (item.isSeparator() || !item.value().isSong())
 		return filter;
-	return Regex::search(SEItemToString(item.value()), rx);
+	return Regex::search(SEItemToString(item.value()), rx, Config.ignore_diacritics);
 }
 
 }

src/screens/sel_items_adder.cpp

@@ -45,7 +45,7 @@ void DisplayComponent(SelectedItemsAdder::Component &menu)
 bool EntryMatcher(const Regex::Regex &rx, const NC::Menu<SelectedItemsAdder::Entry>::Item &item)
 {
 	if (!item.isSeparator())
-		return Regex::search(item.value().item(), rx);
+		return Regex::search(item.value().item(), rx, Config.ignore_diacritics);
 	else
 		return false;
 }

src/screens/tag_editor.cpp

@@ -1191,12 +1191,12 @@ bool DirEntryMatcher(const Regex::Regex &rx, const std::pair<std::string, std::s
 {
 	if (dir.first == "." || dir.first == "..")
 		return filter;
-	return Regex::search(dir.first, rx);
+	return Regex::search(dir.first, rx, Config.ignore_diacritics);
 }
 
 bool SongEntryMatcher(const Regex::Regex &rx, const MPD::MutableSong &s)
 {
-	return Regex::search(SongToString(s), rx);
+	return Regex::search(SongToString(s), rx, Config.ignore_diacritics);
 }
 
 }

src/settings.cpp

@@ -466,6 +466,7 @@ bool Configuration::read(const std::vector<std::string> &config_paths, bool igno
 				invalid_value(v);
 	});
 	p.add("ignore_leading_the", &ignore_leading_the, "no", yes_no);
+	p.add("ignore_diacritics", &ignore_diacritics, "no", yes_no);
 	p.add("block_search_constraints_change_if_items_found",
 	      &block_search_constraints_change, "yes", yes_no);
 	p.add("mouse_support", &mouse_support, "yes", yes_no);

src/settings.h

@@ -154,6 +154,7 @@ struct Configuration
 	bool display_bitrate;
 	bool display_remaining_time;
 	bool ignore_leading_the;
+	bool ignore_diacritics;
 	bool block_search_constraints_change;
 	bool use_console_editor;
 	bool use_cyclic_scrolling;