From f751aff2fb3c94953375854b4e683ac407a3cc58 Mon Sep 17 00:00:00 2001
From: Veloman Yunkan <veloman.yunkan@gmail.com>
Date: Tue, 13 Apr 2021 23:23:40 +0400
Subject: [PATCH] Full case/diacritics insensitivity in catalog filtering

Catalog filtering should now be case/diacritics insensitive for all
fields. However it is not validated for language, name and category
fields, and is validated for tags, creator & publisher only for text
supplied in the filter (but not for values read from the book).
---
 src/library.cpp  | 16 ++++++++--------
 test/library.cpp |  8 +++++---
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/library.cpp b/src/library.cpp
index 4f3b58754..20a3e00a9 100644
--- a/src/library.cpp
+++ b/src/library.cpp
@@ -278,11 +278,11 @@ void Library::updateBookDB(const Book& book)
 
   const std::string title = normalizeText(book.getTitle());
   const std::string desc = normalizeText(book.getDescription());
-  const std::string name = book.getName(); // this is supposed to be normalized
-  const std::string category = book.getCategory(); // this is supposed to be normalized
+  const std::string name = normalizeText(book.getName());
+  const std::string category = normalizeText(book.getCategory());
   const std::string publisher = normalizeText(book.getPublisher());
   const std::string creator = normalizeText(book.getCreator());
-  const std::string tags = book.getTags(); // normalization not needed
+  const std::string tags = normalizeText(book.getTags());
   doc.add_value(0, title);
   doc.add_value(1, desc);
   doc.add_value(2, name);
@@ -359,17 +359,17 @@ Xapian::Query buildXapianQueryFromFilterQuery(const Filter& filter)
 
 Xapian::Query nameQuery(const std::string& name)
 {
-  return Xapian::Query("XN" + name);
+  return Xapian::Query("XN" + normalizeText(name));
 }
 
 Xapian::Query categoryQuery(const std::string& category)
 {
-  return Xapian::Query("XC" + category);
+  return Xapian::Query("XC" + normalizeText(category));
 }
 
 Xapian::Query langQuery(const std::string& lang)
 {
-  return Xapian::Query("L" + lang);
+  return Xapian::Query("L" + normalizeText(lang));
 }
 
 Xapian::Query publisherQuery(const std::string& publisher)
@@ -397,12 +397,12 @@ Xapian::Query tagsQuery(const Filter::Tags& acceptTags, const Filter::Tags& reje
   Xapian::Query q = Xapian::Query(std::string());
   if (!acceptTags.empty()) {
     for ( const auto& tag : acceptTags )
-      q &= Xapian::Query("XT" + tag);
+      q &= Xapian::Query("XT" + normalizeText(tag));
   }
 
   if (!rejectTags.empty()) {
     for ( const auto& tag : rejectTags )
-      q = Xapian::Query(Xapian::Query::OP_AND_NOT, q, "XT" + tag);
+      q = Xapian::Query(Xapian::Query::OP_AND_NOT, q, "XT" + normalizeText(tag));
   }
   return q;
 }
diff --git a/test/library.cpp b/test/library.cpp
index 209be4a87..5c831e367 100644
--- a/test/library.cpp
+++ b/test/library.cpp
@@ -376,9 +376,11 @@ TEST_F(LibraryTest, filterByTags)
     "Mythology & Folklore Stack Exchange"
   );
 
-  // filtering by tags is case sensitive
-  EXPECT_FILTER_RESULTS(kiwix::Filter().acceptTags({"stackEXChange"}),
-    /* no results */
+  // filtering by tags is case and diacritics insensitive
+  EXPECT_FILTER_RESULTS(kiwix::Filter().acceptTags({"ståckEXÇhange"}),
+    "Islam Stack Exchange",
+    "Movies & TV Stack Exchange",
+    "Mythology & Folklore Stack Exchange"
   );
 
   // filtering by tags requires full match of the search term