Merge pull request #729 from kiwix/multizimsearch

include/library.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <mutex>
 #include <zim/archive.h>
+#include <zim/search.h>
 
 #include "book.h"
 #include "bookmark.h"
@@ -140,6 +141,22 @@ private: // functions
     bool accept(const Book& book) const;
 };
 
+
+class ZimSearcher : public zim::Searcher
+{
+  public:
+    explicit ZimSearcher(zim::Searcher&& searcher)
+      : zim::Searcher(searcher)
+    {}
+
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~ZimSearcher() = default;
+  private:
+    std::mutex m_mutex;
+};
+
 /**
  * A Library store several books.
  */
@@ -152,6 +169,7 @@ class Library
   typedef uint64_t Revision;
   typedef std::vector<std::string> BookIdCollection;
   typedef std::map<std::string, int> AttributeCounts;
+  typedef std::set<std::string> BookIdSet;
 
  public:
   Library();
@@ -207,6 +225,10 @@ class Library
 
   DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
   std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
+  std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
+    return getSearcherByIds(BookIdSet{id});
+  }
+  std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
 
   /**
    * Remove a book from the library.
@@ -338,7 +360,7 @@ private: // functions
   std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
   void updateBookDB(const Book& book);
-  void dropReader(const std::string& bookId);
+  void dropCache(const std::string& bookId);
 
 private: //data
   std::unique_ptr<Impl> mp_impl;

include/search_renderer.h

@@ -81,9 +81,9 @@ class SearchRenderer
   void setSearchPattern(const std::string& pattern);
 
   /**
-   * Set the search content id.
+   * Set the querystring used to select books
    */
-  void setSearchContent(const std::string& name);
+  void setSearchBookQuery(const std::string& bookQuery);
 
   /**
    * Set protocol prefix.
@@ -112,7 +112,7 @@ class SearchRenderer
   zim::SearchResultSet m_srs;
   NameMapper* mp_nameMapper;
   Library* mp_library;
-  std::string searchContent;
+  std::string searchBookQuery;
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;

include/server.h

@@ -54,6 +54,7 @@ namespace kiwix
        void setAddress(const std::string& addr) { m_addr = addr; }
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
        void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
@@ -72,6 +73,7 @@ namespace kiwix
        std::string m_indexTemplateString = "";
        int m_port = 80;
        int m_nbThreads = 1;
+       unsigned int m_multizimSearchLimit = 0;
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;

src/library.cpp

@@ -27,10 +27,13 @@
 #include "tools/regexTools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/concurrent_cache.h"
 
 #include <pugixml.hpp>
 #include <algorithm>
 #include <set>
+#include <cmath>
 #include <unicode/locid.h>
 #include <xapian.h>
 
@@ -56,6 +59,27 @@ bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
       && book1.getPath() == book2.getPath();
 }
 
+template<typename Key, typename Value>
+class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
+{
+  public:
+    explicit MultiKeyCache(size_t maxEntries)
+      : ConcurrentCache<std::set<Key>, Value>(maxEntries)
+    {}
+
+    bool drop(const Key& key)
+    {
+      std::unique_lock<std::mutex> l(this->lock_);
+      bool removed = false;
+      for(auto& cache_key: this->impl_.keys()) {
+        if(cache_key.find(key)!=cache_key.end()) {
+          removed |= this->impl_.drop(cache_key);
+        }
+      }
+      return removed;
+    }
+};
+
 } // unnamed namespace
 
 struct Library::Impl
@@ -63,15 +87,14 @@ struct Library::Impl
   struct Entry : Book
   {
     Library::Revision lastUpdatedRevision = 0;
-
-    // May also keep the Archive and Reader pointers here and get
-    // rid of the m_readers and m_archives data members in Library
   };
 
   Library::Revision m_revision;
   std::map<std::string, Entry> m_books;
-  std::map<std::string, std::shared_ptr<Reader>> m_readers;
+  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
-  std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
+  std::unique_ptr<ArchiveCache> mp_archiveCache;
+  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
+  std::unique_ptr<SearcherCache> mp_searcherCache;
   std::vector<kiwix::Bookmark> m_bookmarks;
   Xapian::WritableDatabase m_bookDB;
 
@@ -85,7 +108,9 @@ struct Library::Impl
 };
 
 Library::Impl::Impl()
-  : m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
+  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+    m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
 {
 }
 
@@ -139,7 +164,7 @@ bool Library::addBook(const Book& book)
   try {
     auto& oldbook = mp_impl->m_books.at(book.getId());
     if ( ! booksReferToTheSameArchive(oldbook, book) ) {
-      dropReader(book.getId());
+      dropCache(book.getId());
     }
     oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
                           // XXX: Then m_bookDB will become out-of-sync with
@@ -150,6 +175,13 @@ bool Library::addBook(const Book& book)
     auto& newEntry = mp_impl->m_books[book.getId()];
     static_cast<Book&>(newEntry) = book;
     newEntry.lastUpdatedRevision = mp_impl->m_revision;
+    size_t new_cache_size = std::ceil(mp_impl->getBookCount(true, true)*0.1);
+    if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+    }
+    if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+    }
     return true;
   }
 }
@@ -173,17 +205,23 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 }
 
 
-void Library::dropReader(const std::string& id)
+void Library::dropCache(const std::string& id)
 {
-  mp_impl->m_readers.erase(id);
+  mp_impl->mp_archiveCache->drop(id);
-  mp_impl->m_archives.erase(id);
+  mp_impl->mp_searcherCache->drop(id);
 }
 
 bool Library::removeBookById(const std::string& id)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
   mp_impl->m_bookDB.delete_document("Q" + id);
-  dropReader(id);
+  dropCache(id);
+  // We do not change the cache size here
+  // Most of the time, the book is remove in case of library refresh, it is
+  // often associated with addBook calls (which will properly set the cache size)
+  // Having a too big cache is not a problem here (or it would have been before)
+  // (And setMaxSize doesn't actually reduce the cache size, extra cached items
+  //  will be removed in put or getOrPut).
   return mp_impl->m_books.erase(id) == 1;
 }
 
@@ -242,35 +280,49 @@ const Book& Library::getBookByPath(const std::string& path) const
 
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
-  try {
+  auto archive = getArchiveById(id);
-    std::lock_guard<std::mutex> lock(m_mutex);
+  if(archive) {
-    return mp_impl->m_readers.at(id);
+    return std::make_shared<Reader>(archive);
-  } catch (std::out_of_range& e) {}
+  } else {
-
-  const auto archive = getArchiveById(id);
-  if ( !archive )
     return nullptr;
-
+  }
-  const shared_ptr<Reader> reader(new Reader(archive, true));
-  std::lock_guard<std::mutex> lock(m_mutex);
-  mp_impl->m_readers[id] = reader;
-  return reader;
 }
 
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
   try {
-    return mp_impl->m_archives.at(id);
+    return mp_impl->mp_archiveCache->getOrPut(id,
-  } catch (std::out_of_range& e) {}
+    [&](){
-
       auto book = getBookById(id);
-  if (!book.isPathValid())
+      if (!book.isPathValid()) {
+        throw std::invalid_argument("");
+      }
+      return std::make_shared<zim::Archive>(book.getPath());
+    });
+  } catch (std::invalid_argument&) {
     return nullptr;
+  }
+}
 
-  auto sptr = make_shared<zim::Archive>(book.getPath());
+std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
-  mp_impl->m_archives[id] = sptr;
+{
-  return sptr;
+  assert(!ids.empty());
+  try {
+    return mp_impl->mp_searcherCache->getOrPut(ids,
+    [&](){
+      std::vector<zim::Archive> archives;
+      for(auto& id:ids) {
+        auto archive = getArchiveById(id);
+        if(!archive) {
+          throw std::invalid_argument("");
+        }
+        archives.push_back(*archive);
+      }
+      return std::make_shared<ZimSearcher>(zim::Searcher(archives));
+    });
+  } catch (std::invalid_argument&) {
+    return nullptr;
+  }
 }
 
 unsigned int Library::getBookCount(const bool localBooks,

src/search_renderer.cpp

@@ -71,9 +71,9 @@ void SearchRenderer::setSearchPattern(const std::string& pattern)
   searchPattern = pattern;
 }
 
-void SearchRenderer::setSearchContent(const std::string& content)
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
 {
-  searchContent = content;
+  searchBookQuery = bookQuery;
 }
 
 void SearchRenderer::setProtocolPrefix(const std::string& prefix)
@@ -90,13 +90,13 @@ kainjow::mustache::data buildQueryData
 (
   const std::string& searchProtocolPrefix,
   const std::string& pattern,
-  const std::string& searchContent
+  const std::string& bookQuery
 ) {
   kainjow::mustache::data query;
   query.set("pattern", kiwix::encodeDiples(pattern));
   std::ostringstream ss;
   ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
-  ss << "&content=" << urlEncode(searchContent, true);
+  ss << "&" << bookQuery;
   query.set("unpaginatedQuery", ss.str());
   return query;
 }
@@ -197,7 +197,7 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data query = buildQueryData(
     searchProtocolPrefix,
     searchPattern,
-    searchContent
+    searchBookQuery
   );
 
   std::string template_str = RESOURCE::templates::search_result_html;

src/server.cpp

@@ -45,6 +45,7 @@ bool Server::start() {
     m_port,
     m_root,
     m_nbThreads,
+    m_multizimSearchLimit,
     m_verbose,
     m_withTaskbar,
     m_withLibraryButton,

src/server/internalServer.cpp

@@ -95,35 +95,197 @@ inline std::string normalizeRootUrl(std::string rootUrl)
   return rootUrl.empty() ? rootUrl : "/" + rootUrl;
 }
 
-// Returns the value of env var `name` if found, otherwise returns defaultVal
+Filter get_search_filter(const RequestContext& request, const std::string& prefix="")
-unsigned int getCacheLength(const char* name, unsigned int defaultVal) {
+{
+    auto filter = kiwix::Filter().valid(true).local(true);
     try {
-    const char* envString = std::getenv(name);
+      filter.query(request.get_argument(prefix+"q"));
-    if (envString == nullptr) {
+    } catch (const std::out_of_range&) {}
-      throw std::runtime_error("Environment variable not set");
+    try {
-    }
+      filter.maxSize(request.get_argument<unsigned long>(prefix+"maxsize"));
-    return extractFromString<unsigned int>(envString);
     } catch (...) {}
-
+    try {
-  return defaultVal;
+      filter.name(request.get_argument(prefix+"name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.category(request.get_argument(prefix+"category"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument(prefix+"lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument(prefix+"tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument(prefix+"notag"), ";"));
+    } catch (...) {}
+    return filter;
 }
+
+template<class T>
+std::vector<T> subrange(const std::vector<T>& v, size_t s, size_t n)
+{
+  const size_t e = std::min(v.size(), s+n);
+  return std::vector<T>(v.begin()+std::min(v.size(), s), v.begin()+e);
+}
+
+std::string renderUrl(const std::string& root, const std::string& urlTemplate)
+{
+  MustacheData data;
+  data.set("root", root);
+  auto url = kainjow::mustache::mustache(urlTemplate).render(data);
+  if ( url.back() == '\n' )
+    url.pop_back();
+  return url;
+}
+
+std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
+{
+  return i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
+}
+
+ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
+{
+  return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
+}
+
+ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
+{
+  return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
+}
+
+ParameterizedMessage noValueForArgMsg(const std::string& argument)
+{
+  return ParameterizedMessage("no-value-for-arg", { {"ARGUMENT", argument} });
+}
+
+ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
+{
+  return ParameterizedMessage("raw-entry-not-found",
+                              {
+                                {"DATATYPE", dt},
+                                {"ENTRY", entry},
+                              }
+  );
+}
+
+ParameterizedMessage tooManyBooksMsg(size_t nbBooks, size_t limit)
+{
+  return ParameterizedMessage("too-many-books",
+                              {
+                                {"NB_BOOKS", beautifyInteger(nbBooks)},
+                                {"LIMIT", beautifyInteger(limit)},
+                              }
+  );
+}
+
+ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
+{
+  const ParameterizedMessage::Parameters noParams;
+  return ParameterizedMessage(msgId, noParams);
+}
+
+struct Error : public std::runtime_error {
+  explicit Error(const ParameterizedMessage& message)
+    : std::runtime_error("Error while handling request"),
+      _message(message)
+  {}
+
+  const ParameterizedMessage& message() const
+  {
+    return _message;
+  }
+
+  const ParameterizedMessage _message;
+};
+
+void checkBookNumber(const Library::BookIdSet& bookIds, size_t limit) {
+  if (bookIds.empty()) {
+    throw Error(nonParameterizedMessage("no-book-found"));
+  }
+  if (limit > 0 && bookIds.size() > limit) {
+    throw Error(tooManyBooksMsg(bookIds.size(), limit));
+  }
+}
+
 } // unnamed namespace
 
-SearchInfo::SearchInfo(const std::string& pattern)
+std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const RequestContext& request) const
-  : pattern(pattern),
-    geoQuery()
-{}
-
-SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery)
-  : pattern(pattern),
-    geoQuery(geoQuery)
-{}
-
-SearchInfo::SearchInfo(const RequestContext& request)
-  : pattern(request.get_optional_param<std::string>("pattern", "")),
-    geoQuery(),
-    bookName(request.get_optional_param<std::string>("content", ""))
 {
+  // Try old API
+  try {
+    auto bookName = request.get_argument("content");
+    try {
+      const auto bookIds = Library::BookIdSet{mp_nameMapper->getIdForName(bookName)};
+      const auto queryString = request.get_query([&](const std::string& key){return key == "content";}, true);
+      return {queryString, bookIds};
+    } catch (const std::out_of_range&) {
+      throw Error(noSuchBookErrorMsg(bookName));
+    }
+  } catch(const std::out_of_range&) {
+    // We've catch the out_of_range of get_argument
+    // continue
+  }
+
+  // Does user directly gives us ids ?
+  try {
+    auto id_vec = request.get_arguments("books.id");
+    if (id_vec.empty()) {
+      throw Error(noValueForArgMsg("books.id"));
+    }
+    for(const auto& bookId: id_vec) {
+      try {
+        // This is a silly way to check that bookId exists
+        mp_nameMapper->getNameForId(bookId);
+      } catch (const std::out_of_range&) {
+        throw Error(noSuchBookErrorMsg(bookId));
+      }
+    }
+    const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
+    const auto queryString = request.get_query([&](const std::string& key){return key == "books.id";}, true);
+    return {queryString, bookIds};
+  } catch(const std::out_of_range&) {}
+
+  // Use the names
+  try {
+    auto name_vec = request.get_arguments("books.name");
+    if (name_vec.empty()) {
+      throw Error(noValueForArgMsg("books.name"));
+    }
+    Library::BookIdSet bookIds;
+    for(const auto& bookName: name_vec) {
+      try {
+        bookIds.insert(mp_nameMapper->getIdForName(bookName));
+      } catch(const std::out_of_range&) {
+        throw Error(noSuchBookErrorMsg(bookName));
+      }
+    }
+    const auto queryString = request.get_query([&](const std::string& key){return key == "books.name";}, true);
+    return {queryString, bookIds};
+  } catch(const std::out_of_range&) {}
+
+  // Check for filtering
+  Filter filter = get_search_filter(request, "books.filter.");
+  auto id_vec = mp_library->filter(filter);
+  if (id_vec.empty()) {
+    throw Error(nonParameterizedMessage("no-book-found"));
+  }
+  const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
+  const auto queryString = request.get_query([&](const std::string& key){return startsWith(key, "books.filter.");}, true);
+  return {queryString, bookIds};
+}
+
+SearchInfo InternalServer::getSearchInfo(const RequestContext& request) const
+{
+  auto bookIds = selectBooks(request);
+  checkBookNumber(bookIds.second, m_multizimSearchLimit);
+  auto pattern = request.get_optional_param<std::string>("pattern", "");
+  GeoQuery geoQuery;
+
   /* Retrive geo search */
   try {
     auto latitude = request.get_argument<float>("latitude");
@@ -134,10 +296,19 @@ SearchInfo::SearchInfo(const RequestContext& request)
     catch(const std::invalid_argument&) {}
 
   if (!geoQuery && pattern.empty()) {
-    throw std::invalid_argument("No query provided.");
+    throw Error(nonParameterizedMessage("no-query"));
   }
+
+  return SearchInfo(pattern, geoQuery, bookIds.second, bookIds.first);
 }
 
+SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterQuery)
+  : pattern(pattern),
+    geoQuery(geoQuery),
+    bookIds(bookIds),
+    bookFilterQuery(bookFilterQuery)
+{}
+
 zim::Query SearchInfo::getZimQuery(bool verbose) const {
   zim::Query query;
   if (verbose) {
@@ -175,6 +346,7 @@ InternalServer::InternalServer(Library* library,
                                int port,
                                std::string root,
                                int nbThreads,
+                               unsigned int multizimSearchLimit,
                                bool verbose,
                                bool withTaskbar,
                                bool withLibraryButton,
@@ -185,6 +357,7 @@ InternalServer::InternalServer(Library* library,
   m_port(port),
   m_root(normalizeRootUrl(root)),
   m_nbThreads(nbThreads),
+  m_multizimSearchLimit(multizimSearchLimit),
   m_verbose(verbose),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
@@ -194,9 +367,8 @@ InternalServer::InternalServer(Library* library,
   mp_daemon(nullptr),
   mp_library(library),
   mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
-  searcherCache(getCacheLength("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
+  searchCache(getEnvVar<int>("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
-  searchCache(getCacheLength("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
+  suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
-  suggestionSearcherCache(getCacheLength("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
 {}
 
 bool InternalServer::start() {
@@ -439,56 +611,6 @@ SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Arch
   return suggestions;
 }
 
-namespace
-{
-
-std::string renderUrl(const std::string& root, const std::string& urlTemplate)
-{
-  MustacheData data;
-  data.set("root", root);
-  auto url = kainjow::mustache::mustache(urlTemplate).render(data);
-  if ( url.back() == '\n' )
-    url.pop_back();
-  return url;
-}
-
-std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
-{
-  return i18n::expandParameterizedString(lang, "suggest-full-text-search",
-               {
-                  {"SEARCH_TERMS", queryString}
-               }
-         );
-}
-
-ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
-{
-  return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
-}
-
-ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
-{
-  return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
-}
-
-ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
-{
-  return ParameterizedMessage("raw-entry-not-found",
-                              {
-                                {"DATATYPE", dt},
-                                {"ENTRY", entry},
-                              }
-  );
-}
-
-ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
-{
-  const ParameterizedMessage::Parameters noParams;
-  return ParameterizedMessage(msgId, noParams);
-}
-
-} // unnamed namespace
-
 std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -591,41 +713,18 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
   }
 
   try {
-    auto searchInfo = SearchInfo(request);
+    auto searchInfo = getSearchInfo(request);
-
+    auto bookIds = searchInfo.getBookIds();
-    std::string bookId;
-    std::shared_ptr<zim::Archive> archive;
-    if (!searchInfo.bookName.empty()) {
-      try {
-        bookId = mp_nameMapper->getIdForName(searchInfo.bookName);
-        archive = mp_library->getArchiveById(bookId);
-      } catch (const std::out_of_range&) {
-        throw std::invalid_argument("The requested book doesn't exist.");
-      }
-    }
-
 
     /* Make the search */
     // Try to get a search from the searchInfo, else build it
+    auto searcher = mp_library->getSearcherByIds(bookIds);
+    auto lock(searcher->getLock());
+
     std::shared_ptr<zim::Search> search;
     try {
       search = searchCache.getOrPut(searchInfo,
         [=](){
-          std::shared_ptr<zim::Searcher> searcher;
-          if (archive) {
-            searcher = searcherCache.getOrPut(bookId, [=](){ return std::make_shared<zim::Searcher>(*archive);});
-          } else {
-            for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-              auto currentArchive = mp_library->getArchiveById(bookId);
-              if (currentArchive) {
-                if (! searcher) {
-                  searcher = std::make_shared<zim::Searcher>(*currentArchive);
-                } else {
-                  searcher->addArchive(*currentArchive);
-                }
-              }
-           }
-          }
           return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
         }
       );
@@ -638,11 +737,14 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
                                      "404-page-heading",
                                      cssUrl);
       response += nonParameterizedMessage("no-search-results");
-      response += TaskbarInfo(searchInfo.bookName, archive.get());
+      if(bookIds.size() == 1) {
+        auto bookId = *bookIds.begin();
+        auto bookName = mp_nameMapper->getNameForId(bookId);
+        response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
+      }
       return response;
     }
 
-
     auto start = 0;
     try {
       start = request.get_argument<unsigned int>("start");
@@ -663,17 +765,21 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     SearchRenderer renderer(search->getResults(start, pageLength), mp_nameMapper, mp_library, start,
                             search->getEstimatedMatches());
     renderer.setSearchPattern(searchInfo.pattern);
-    renderer.setSearchContent(searchInfo.bookName);
+    renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
     renderer.setProtocolPrefix(m_root + "/");
     renderer.setSearchProtocolPrefix(m_root + "/search");
     renderer.setPageLength(pageLength);
     auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
-    response->set_taskbar(searchInfo.bookName, archive.get());
+    if(bookIds.size() == 1) {
+      auto bookId = *bookIds.begin();
+      auto bookName = mp_nameMapper->getNameForId(bookId);
+      response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
+    }
     return std::move(response);
-  } catch (const std::invalid_argument& e) {
+  } catch (const Error& e) {
     return HTTP400HtmlResponse(*this, request)
       + invalidUrlMsg
-      + std::string(e.what());
+      + e.message();
   }
 }
 
@@ -776,45 +882,6 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   return std::move(response);
 }
 
-namespace
-{
-
-Filter get_search_filter(const RequestContext& request)
-{
-    auto filter = kiwix::Filter().valid(true).local(true);
-    try {
-      filter.query(request.get_argument("q"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.maxSize(request.get_argument<unsigned long>("maxsize"));
-    } catch (...) {}
-    try {
-      filter.name(request.get_argument("name"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.category(request.get_argument("category"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.lang(request.get_argument("lang"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
-    } catch (...) {}
-    try {
-      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
-    } catch (...) {}
-    return filter;
-}
-
-template<class T>
-std::vector<T> subrange(const std::vector<T>& v, size_t s, size_t n)
-{
-  const size_t e = std::min(v.size(), s+n);
-  return std::vector<T>(v.begin()+std::min(v.size(), s), v.begin()+e);
-}
-
-} // unnamed namespace
-
 std::vector<std::string>
 InternalServer::search_catalog(const RequestContext& request,
                                kiwix::OPDSDumper& opdsDumper)

src/server/internalServer.h

@@ -68,27 +68,26 @@ struct GeoQuery {
 
 class SearchInfo {
   public:
-    SearchInfo(const std::string& pattern);
+    SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterString);
-    SearchInfo(const std::string& pattern, GeoQuery geoQuery);
-    SearchInfo(const RequestContext& request);
 
     zim::Query getZimQuery(bool verbose) const;
+    const Library::BookIdSet& getBookIds() const { return bookIds; }
 
     friend bool operator<(const SearchInfo& l, const SearchInfo& r)
     {
-        return std::tie(l.bookName, l.pattern, l.geoQuery)
+        return std::tie(l.bookIds, l.pattern, l.geoQuery)
-             < std::tie(r.bookName, r.pattern, r.geoQuery); // keep the same order
+             < std::tie(r.bookIds, r.pattern, r.geoQuery); // keep the same order
     }
 
   public: //data
     std::string pattern;
     GeoQuery geoQuery;
-    std::string bookName;
+    Library::BookIdSet bookIds;
+    std::string bookFilterQuery;
 };
 
 
 typedef kainjow::mustache::data MustacheData;
-typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache;
 typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
 typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
 
@@ -103,6 +102,7 @@ class InternalServer {
                    int port,
                    std::string root,
                    int nbThreads,
+                   unsigned int multizimSearchLimit,
                    bool verbose,
                    bool withTaskbar,
                    bool withLibraryButton,
@@ -150,12 +150,15 @@ class InternalServer {
 
     bool etag_not_needed(const RequestContext& r) const;
     ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+    std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
+    SearchInfo getSearchInfo(const RequestContext& r) const;
 
   private: // data
     std::string m_addr;
     int m_port;
     std::string m_root;
     int m_nbThreads;
+    unsigned int m_multizimSearchLimit;
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
@@ -167,7 +170,6 @@ class InternalServer {
     Library* mp_library;
     NameMapper* mp_nameMapper;
 
-    SearcherCache searcherCache;
     SearchCache searchCache;
     SuggestionSearcherCache suggestionSearcherCache;
 

src/server/request_context.cpp

@@ -106,7 +106,7 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
                                          const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->arguments[key] = value == nullptr ? "" : value;
+  _this->arguments[key].push_back(value == nullptr ? "" : value);
   return MHD_YES;
 }
 
@@ -121,8 +121,14 @@ void RequestContext::print_debug_info() const {
     printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
   }
   printf("arguments :\n");
-  for (auto it=arguments.begin(); it!=arguments.end(); it++) {
+  for (auto& pair:arguments) {
-    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+    printf(" - %s :", pair.first.c_str());
+    bool first = true;
+    for (auto& v: pair.second) {
+      printf("%s %s", first?"":",", v.c_str());
+      first = false;
+    }
+    printf("\n");
   }
   printf("Parsed : \n");
   printf("full_url: %s\n", full_url.c_str());
@@ -176,23 +182,13 @@ ByteRange RequestContext::get_range() const {
 
 template<>
 std::string RequestContext::get_argument(const std::string& name) const {
-  return arguments.at(name);
+  return arguments.at(name)[0];
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
   return headers.at(lcAll(name));
 }
 
-std::string RequestContext::get_query() const {
-  std::string q;
-  const char* sep = "";
-  for ( const auto& a : arguments ) {
-    q += sep + a.first + '=' + a.second;
-    sep = "&";
-  }
-  return q;
-}
-
 std::string RequestContext::get_user_language() const
 {
   try {

src/server/request_context.h

@@ -25,6 +25,7 @@
 #include <string>
 #include <sstream>
 #include <map>
+#include <vector>
 #include <stdexcept>
 
 #include "byte_range.h"
@@ -69,7 +70,11 @@ class RequestContext {
     std::string get_header(const std::string& name) const;
     template<typename T=std::string>
     T get_argument(const std::string& name) const {
-        return extractFromString<T>(arguments.at(name));
+        return extractFromString<T>(get_argument(name));
+    }
+
+    std::vector<std::string> get_arguments(const std::string& name) const {
+      return arguments.at(name);
     }
 
     template<class T>
@@ -86,7 +91,27 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
-    std::string get_query() const;
+
+    std::string get_query(bool mustEncode = false) const {
+      return get_query([](const std::string& key) {return true;}, mustEncode);
+    }
+
+    template<class F>
+    std::string get_query(F filter, bool mustEncode) const {
+      std::string q;
+      const char* sep = "";
+      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value, true):value; };
+      for ( const auto& a : arguments ) {
+        if (!filter(a.first)) {
+          continue;
+        }
+        for (const auto& v: a.second) {
+          q += sep + encode(a.first) + '=' + encode(v);
+          sep = "&";
+        }
+      }
+      return q;
+    }
 
     ByteRange get_range() const;
 
@@ -105,7 +130,7 @@ class RequestContext {
 
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
-    std::map<std::string, std::string> arguments;
+    std::map<std::string, std::vector<std::string>> arguments;
 
   private: // functions
     static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);

src/tools/concurrent_cache.h

@@ -1,3 +1,4 @@
+
 /*
  * Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
  * Copyright (C) 2020 Veloman Yunkan
@@ -84,11 +85,123 @@ public: // types
     return impl_.drop(key);
   }
 
-private: // data
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
   Impl impl_;
   std::mutex lock_;
 };
 
+
+/**
+  WeakStore represent a thread safe store (map) of weak ptr.
+  It allows to store weak_ptr from shared_ptr and retrieve shared_ptr from
+  potential non expired weak_ptr.
+  It is not limited in size.
+*/
+template<typename Key, typename Value>
+class WeakStore {
+  private: // types
+    typedef std::weak_ptr<Value> WeakValue;
+
+  public:
+    explicit WeakStore() = default;
+
+    std::shared_ptr<Value> get(const Key& key)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      auto it = m_weakMap.find(key);
+      if (it != m_weakMap.end()) {
+        auto shared = it->second.lock();
+        if (shared) {
+          return shared;
+        } else {
+          m_weakMap.erase(it);
+        }
+      }
+      throw std::runtime_error("No weak ptr");
+    }
+
+    void add(const Key& key, std::shared_ptr<Value> shared)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      m_weakMap[key] = WeakValue(shared);
+    }
+
+
+  private: //data
+    std::map<Key, WeakValue> m_weakMap;
+    std::mutex m_lock;
+};
+
+template <typename Key, typename RawValue>
+class ConcurrentCache<Key, std::shared_ptr<RawValue>>
+{
+private: // types
+  typedef std::shared_ptr<RawValue> Value;
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      // Try to get back the shared_ptr from the weak_ptr first.
+      try {
+        valuePromise.set_value(m_weakStore.get(key));
+      } catch(const std::runtime_error& e) {
+        try {
+          const auto value = f();
+          valuePromise.set_value(value);
+          m_weakStore.add(key, value);
+        } catch (std::exception& e) {
+          drop(key);
+          throw;
+        }
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
+  std::mutex lock_;
+  Impl impl_;
+  WeakStore<Key, RawValue> m_weakStore;
+};
+
 } // namespace kiwix
 
 #endif // ZIM_CONCURRENT_CACHE_H

src/tools/lrucache.h

@@ -40,6 +40,7 @@
 
 #include <map>
 #include <list>
+#include <set>
 #include <cstddef>
 #include <stdexcept>
 #include <cassert>
@@ -138,12 +139,26 @@ public: // functions
     return _cache_items_map.size();
   }
 
+  size_t setMaxSize(size_t new_size) {
+    size_t previous = _max_size;
+    _max_size = new_size;
+    return previous;
+  }
+
+  std::set<key_t> keys() const  {
+    std::set<key_t> keys;
+    for(auto& item:_cache_items_map) {
+      keys.insert(item.first);
+    }
+    return keys;
+  }
+
 private: // functions
   void putMissing(const key_t& key, const value_t& value) {
     assert(_cache_items_map.find(key) == _cache_items_map.end());
     _cache_items_list.push_front(key_value_pair_t(key, value));
     _cache_items_map[key] = _cache_items_list.begin();
-    if (_cache_items_map.size() > _max_size) {
+    while (_cache_items_map.size() > _max_size) {
       _cache_items_map.erase(_cache_items_list.back().first);
       _cache_items_list.pop_back();
     }

src/tools/otherTools.h

@@ -23,9 +23,12 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <cstdlib>
 #include <zim/zim.h>
 #include <mustache.hpp>
 
+#include "stringTools.h"
+
 namespace pugi {
   class xml_node;
 }
@@ -53,6 +56,20 @@ namespace kiwix
   kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
 
   std::string render_template(const std::string& template_str, kainjow::mustache::data data);
+
+  template<typename T>
+  T getEnvVar(const char* name, const T& defaultValue)
+  {
+    try {
+      const char* envString = std::getenv(name);
+      if (envString == nullptr) {
+        throw std::runtime_error("Environment variable not set");
+      }
+      return extractFromString<T>(envString);
+    } catch (...) {}
+
+    return defaultValue;
+  }
 }
 
 #endif

static/i18n/en.json

@@ -4,12 +4,16 @@
 		]
 	},
 	"name":"English",
-	"suggest-full-text-search": "containing '{{{SEARCH_TERMS}}}'..."
+	"suggest-full-text-search" : "containing '{{{SEARCH_TERMS}}}'..."
-	, "no-such-book": "No such book: {{BOOK_NAME}}"
+	, "no-such-book" : "No such book: {{BOOK_NAME}}"
+	, "too-many-books" : "Too many books requested ({{NB_BOOKS}}) where limit is {{LIMIT}}"
+	, "no-book-found" : "No book matches selection criteria"
 	, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
 	, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
 	, "random-article-failure" :  "Oops! Failed to pick a random article :("
 	, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
+	, "no-value-for-arg": "No value provided for argument {{ARGUMENT}}"
+	, "no-query" : "No query provided."
 	, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
 	, "400-page-title" : "Invalid request"
 	, "400-page-heading" : "Invalid request"

static/i18n/qqq.json

@@ -2,16 +2,21 @@
 	"@metadata": {
 		"authors": [
 			"Veloman Yunkan",
-			"Verdy p"
+			"Verdy p",
+			"Matthieu Gautier"
 		]
 	},
 	"name": "{{Doc-important|Don't write \"English\" in your language!}}\n\n'''Write the name of ''your'' language in its native script.'''\n\nCurrent language to which the string is being translated to.\n\nFor example, write \"français\" when translating to French, or \"Deutsch\" when translating to German.\n\n'''Important:''' Do not use your language’s word for “English”. Use the word that your language uses to refer to itself. If you translate this message to mean “English” in your language, your change will be reverted.",
 	"suggest-full-text-search": "Text appearing in the suggestion list that, when selected, runs a full text search instead of the title search",
 	"no-such-book": "Error text when the requested book is not found in the library",
+	"too-many-books":"Error text when user request more books than the limit set by the administrator",
 	"url-not-found": "Error text about wrong URL for an HTTP 404 error",
+	"no-book-found": "Error text when no book matches the selection criteria",
 	"suggest-search": "Suggest a search when the URL points to a non existing article",
 	"random-article-failure": "Failure of the random article selection procedure",
 	"invalid-raw-data-type": "Invalid DATATYPE was used with the /raw endpoint (/raw/<book>/DATATYPE/...); allowed values are 'meta' and 'content'",
+	"no-value-for-arg" : "Error text when no value has been provided for ARGUMENT in the request's query string",
+	"no-query" : "Error text when no query has been provided for fulltext search",
 	"raw-entry-not-found": "Entry requested via the /raw endpoint was not found",
 	"400-page-title": "Title of the 400 error page",
 	"400-page-heading": "Heading of the 400 error page",

test/lrucache.cpp

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2014, lamerman
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of lamerman nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "../src/tools/lrucache.h"
+#include "../src/tools/concurrent_cache.h"
+#include "gtest/gtest.h"
+
+const unsigned int NUM_OF_TEST2_RECORDS = 100;
+const unsigned int TEST2_CACHE_CAPACITY = 50;
+
+TEST(CacheTest, SimplePut) {
+    kiwix::lru_cache<int, int> cache_lru(1);
+    cache_lru.put(7, 777);
+    EXPECT_TRUE(cache_lru.exists(7));
+    EXPECT_EQ(777, cache_lru.get(7));
+    EXPECT_EQ(1U, cache_lru.size());
+}
+
+TEST(CacheTest, OverwritingPut) {
+    kiwix::lru_cache<int, int> cache_lru(1);
+    cache_lru.put(7, 777);
+    cache_lru.put(7, 222);
+    EXPECT_TRUE(cache_lru.exists(7));
+    EXPECT_EQ(222, cache_lru.get(7));
+    EXPECT_EQ(1U, cache_lru.size());
+}
+
+TEST(CacheTest, MissingValue) {
+    kiwix::lru_cache<int, int> cache_lru(1);
+    EXPECT_TRUE(cache_lru.get(7).miss());
+    EXPECT_FALSE(cache_lru.get(7).hit());
+    EXPECT_THROW(cache_lru.get(7).value(), std::range_error);
+}
+
+TEST(CacheTest, DropValue) {
+    kiwix::lru_cache<int, int> cache_lru(3);
+    cache_lru.put(7, 777);
+    cache_lru.put(8, 888);
+    cache_lru.put(9, 999);
+    EXPECT_EQ(3U, cache_lru.size());
+    EXPECT_TRUE(cache_lru.exists(7));
+    EXPECT_EQ(777, cache_lru.get(7));
+
+    EXPECT_TRUE(cache_lru.drop(7));
+
+    EXPECT_EQ(2U, cache_lru.size());
+    EXPECT_FALSE(cache_lru.exists(7));
+    EXPECT_THROW(cache_lru.get(7).value(), std::range_error);
+
+    EXPECT_FALSE(cache_lru.drop(7));
+}
+
+TEST(CacheTest1, KeepsAllValuesWithinCapacity) {
+    kiwix::lru_cache<int, int> cache_lru(TEST2_CACHE_CAPACITY);
+
+    for (uint i = 0; i < NUM_OF_TEST2_RECORDS; ++i) {
+        cache_lru.put(i, i);
+    }
+
+    for (uint i = 0; i < NUM_OF_TEST2_RECORDS - TEST2_CACHE_CAPACITY; ++i) {
+        EXPECT_FALSE(cache_lru.exists(i));
+    }
+
+    for (uint i = NUM_OF_TEST2_RECORDS - TEST2_CACHE_CAPACITY; i < NUM_OF_TEST2_RECORDS; ++i) {
+        EXPECT_TRUE(cache_lru.exists(i));
+        EXPECT_EQ((int)i, cache_lru.get(i));
+    }
+
+    size_t size = cache_lru.size();
+    EXPECT_EQ(TEST2_CACHE_CAPACITY, size);
+}
+
+TEST(ConcurrentCacheTest, handleException) {
+    kiwix::ConcurrentCache<int, int> cache(1);
+    auto val = cache.getOrPut(7, []() { return 777; });
+    EXPECT_EQ(val, 777);
+    EXPECT_THROW(cache.getOrPut(8, []() { throw std::runtime_error("oups"); return 0; }), std::runtime_error);
+    val = cache.getOrPut(8, []() { return 888; });
+    EXPECT_EQ(val, 888);
+}
+
+TEST(ConcurrentCacheTest, weakPtr) {
+    kiwix::ConcurrentCache<int, std::shared_ptr<int>> cache(1);
+    auto refValue = cache.getOrPut(7, []() { return std::make_shared<int>(777); });
+    EXPECT_EQ(*refValue, 777);
+    EXPECT_EQ(refValue.use_count(), 2);
+
+    // This will drop shared(777) from the cache
+    cache.getOrPut(8, []() { return std::make_shared<int>(888); });
+    EXPECT_EQ(refValue.use_count(), 1);
+
+    // We must get the shared value from the weakPtr we have
+    EXPECT_NO_THROW(cache.getOrPut(7, []() { throw std::runtime_error("oups"); return nullptr; }));
+    EXPECT_EQ(refValue.use_count(), 2);
+
+    // Drop all ref
+    cache.getOrPut(8, []() { return std::make_shared<int>(888); });
+    refValue.reset();
+
+    // Be sure we call the construction function
+    EXPECT_THROW(cache.getOrPut(7, []() { throw std::runtime_error("oups"); return nullptr; }), std::runtime_error);
+}

test/meson.build

@@ -10,7 +10,8 @@ tests = [
     'manager',
     'name_mapper',
     'opds_catalog',
-    'server_helper'
+    'server_helper',
+    'lrucache'
 ]
 
 if build_machine.system() != 'windows'

test/server.cpp

@@ -133,6 +133,7 @@ void ZimFileServer::run(int serverPort, std::string indexTemplateString)
   server->setNbThreads(2);
   server->setVerbose(false);
   server->setTaskbar(withTaskbar, withTaskbar);
+  server->setMultiZimSearchLimit(3);
   if (!indexTemplateString.empty()) {
     server->setIndexTemplateString(indexTemplateString);
   }
@@ -156,6 +157,7 @@ protected:
   const int PORT = 8001;
   const ZimFileServer::FilePathCollection ZIMFILES {
     "./test/zimfile.zim",
+    "./test/example.zim",
     "./test/poor.zim",
     "./test/corner_cases.zim"
   };
@@ -394,6 +396,10 @@ const char* urls400[] = {
   "/ROOT/search?content=zimfile",
   "/ROOT/search?content=non-existing-book&pattern=asdfqwerty",
   "/ROOT/search?content=non-existing-book&pattern=asd<qwerty",
+  "/ROOT/search?books.name=non-exsitent-book&pattern=asd<qwerty",
+  "/ROOT/search?books.id=non-exsitent-id&pattern=asd<qwerty",
+  "/ROOT/search?books.filter.lang=unk&pattern=asd<qwerty",
+  "/ROOT/search?pattern=foo",
   "/ROOT/search?pattern"
 };
 
@@ -899,7 +905,7 @@ TEST_F(ServerTest, 400WithBodyTesting)
       The requested URL "/ROOT/search" is not a valid request.
     </p>
     <p>
-      No query provided.
+      Too many books requested (4) where limit is 3
     </p>
 )"  },
     { /* url */ "/ROOT/search?content=zimfile",
@@ -919,7 +925,7 @@ TEST_F(ServerTest, 400WithBodyTesting)
       The requested URL "/ROOT/search?content=non-existing-book&pattern=asdfqwerty" is not a valid request.
     </p>
     <p>
-      The requested book doesn't exist.
+      No such book: non-existing-book
     </p>
 )"  },
     { /* url */ "/ROOT/search?content=non-existing-book&pattern=a\"<script foo>",
@@ -929,20 +935,30 @@ TEST_F(ServerTest, 400WithBodyTesting)
       The requested URL "/ROOT/search?content=non-existing-book&pattern=a"&lt;script foo&gt;" is not a valid request.
     </p>
     <p>
-      The requested book doesn't exist.
+      No such book: non-existing-book
     </p>
 )"  },
     // There is a flaw in our way to handle query string, we cannot differenciate
     // between `pattern` and `pattern=`
-    { /* url */ "/ROOT/search?pattern",
+    { /* url */ "/ROOT/search?books.filter.lang=eng&pattern",
       expected_body==R"(
     <h1>Invalid request</h1>
     <p>
-      The requested URL "/ROOT/search?pattern=" is not a valid request.
+      The requested URL "/ROOT/search?books.filter.lang=eng&pattern=" is not a valid request.
     </p>
     <p>
       No query provided.
     </p>
+)"  },
+    { /* url */ "/ROOT/search?pattern=foo",
+      expected_body==R"(
+    <h1>Invalid request</h1>
+    <p>
+      The requested URL "/ROOT/search?pattern=foo" is not a valid request.
+    </p>
+    <p>
+      Too many books requested (4) where limit is 3
+    </p>
 )"  },
   };
 
@@ -1590,6 +1606,9 @@ bool isSubSnippet(std::string subSnippet, const std::string& superSnippet)
   return true;
 }
 
+#define  RAYCHARLESZIMID "6f1d19d0-633f-087b-fb55-7ac324ff9baf"
+#define  EXAMPLEZIMID    "5dc0b3af-5df2-0925-f0ca-d2bf75e78af6"
+
 TEST_F(TaskbarlessServerTest, searchResults)
 {
   struct TestData
@@ -1601,7 +1620,7 @@ TEST_F(TaskbarlessServerTest, searchResults)
       bool selected;
     };
 
-    std::string pattern;
+    std::string query;
     int start;
     size_t resultsPerPage;
     size_t totalResultCount;
@@ -1609,9 +1628,9 @@ TEST_F(TaskbarlessServerTest, searchResults)
     std::vector<std::string> results;
     std::vector<PaginationEntry> pagination;
 
-    static std::string makeUrl(const std::string pattern, int start, size_t resultsPerPage)
+    static std::string makeUrl(const std::string& query, int start, size_t resultsPerPage)
     {
-      std::string url = "/ROOT/search?pattern=" + pattern + "&content=zimfile";
+      std::string url = "/ROOT/search?" + query;
 
       if ( start >= 0 ) {
         url += "&start=" + to_string(start);
@@ -1624,15 +1643,23 @@ TEST_F(TaskbarlessServerTest, searchResults)
       return url;
     }
 
+    std::string getPattern() const
+    {
+      const std::string p = "pattern=";
+      const size_t i = query.find(p);
+      std::string r = query.substr(i + p.size());
+      return r.substr(0, r.find("&"));
+    }
+
     std::string url() const
     {
-      return makeUrl(pattern, start, resultsPerPage);
+      return makeUrl(query, start, resultsPerPage);
     }
 
     std::string expectedHeader() const
     {
       if ( totalResultCount == 0 ) {
-        return "\n        No results were found for <b>\"" + pattern + "\"</b>";
+        return "\n        No results were found for <b>\"" + getPattern() + "\"</b>";
       }
 
       std::string header = R"(  Results
@@ -1649,7 +1676,7 @@ TEST_F(TaskbarlessServerTest, searchResults)
       header = replace(header, "FIRSTRESULT", to_string(firstResultIndex));
       header = replace(header, "LASTRESULT",  to_string(lastResultIndex));
       header = replace(header, "RESULTCOUNT", to_string(totalResultCount));
-      header = replace(header, "PATTERN",     pattern);
+      header = replace(header, "PATTERN",     getPattern());
       return header;
     }
 
@@ -1677,7 +1704,7 @@ TEST_F(TaskbarlessServerTest, searchResults)
       std::ostringstream oss;
       oss << "\n        <ul>\n";
       for ( const auto& p : pagination ) {
-        const auto url = makeUrl(pattern, p.start, resultsPerPage);
+        const auto url = makeUrl(query, p.start, resultsPerPage);
         oss << "            <li>\n";
         oss << "              <a ";
         if ( p.selected ) {
@@ -1695,7 +1722,7 @@ TEST_F(TaskbarlessServerTest, searchResults)
     std::string expectedHtml() const
     {
       return makeSearchResultsHtml(
-               pattern,
+               getPattern(),
                expectedHeader(),
                expectedResultsString(),
                expectedFooter()
@@ -1768,7 +1795,7 @@ TEST_F(TaskbarlessServerTest, searchResults)
 
   const TestData testData[] = {
     {
-      /* pattern */          "velomanyunkan",
+      /* query */          "pattern=velomanyunkan&books.id=" RAYCHARLESZIMID,
       /* start */            -1,
       /* resultsPerPage */   0,
       /* totalResultCount */ 0,
@@ -1778,7 +1805,7 @@ TEST_F(TaskbarlessServerTest, searchResults)
     },
 
     {
-      /* pattern */          "razaf",
+      /* query */          "pattern=razaf&books.id=" RAYCHARLESZIMID,
       /* start */            -1,
       /* resultsPerPage */   0,
       /* totalResultCount */ 1,
@@ -1797,7 +1824,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "yellow",
+      /* query */          "pattern=yellow&books.id=" RAYCHARLESZIMID,
       /* start */            -1,
       /* resultsPerPage */   0,
       /* totalResultCount */ 2,
@@ -1825,7 +1852,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            -1,
       /* resultsPerPage */   100,
       /* totalResultCount */ 44,
@@ -1835,7 +1862,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            -1,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -1859,7 +1886,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            5,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -1884,7 +1911,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            10,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -1910,7 +1937,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            15,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -1937,7 +1964,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */            "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            20,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -1964,7 +1991,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            25,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -1991,7 +2018,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            30,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -2017,7 +2044,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            35,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -2042,7 +2069,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            40,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -2065,7 +2092,7 @@ R"SEARCHRESULT(
     },
 
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            21,
       /* resultsPerPage */   3,
       /* totalResultCount */ 44,
@@ -2094,7 +2121,7 @@ R"SEARCHRESULT(
     // This test-point only documents how the current implementation
     // works, not how it should work!
     {
-      /* pattern */          "jazz",
+      /* query */          "pattern=jazz&books.id=" RAYCHARLESZIMID,
       /* start */            45,
       /* resultsPerPage */   5,
       /* totalResultCount */ 44,
@@ -2109,6 +2136,148 @@ R"SEARCHRESULT(
         { "9", 40, false  },
       }
     },
+
+    // We must return results from the two books
+    {
+      /* query */          "pattern=travel"
+                           "&books.id=" RAYCHARLESZIMID
+                           "&books.id=" EXAMPLEZIMID,
+      /* start */            0,
+      /* resultsPerPage */   10,
+      /* totalResultCount */ 2,
+      /* firstResultIndex */ 1,
+      /* results */          {
+R"SEARCHRESULT(
+            <a href="/ROOT/zimfile/A/If_You_Go_Away">
+              If You Go Away
+            </a>
+              <cite>...<b>Travel</b> On" (1965) "If You Go Away" (1966) "Walk Away" (1967) Damita Jo reached #10 on the Adult Contemporary chart and #68 on the Billboard Hot 100 in 1966 for her version of the song. Terry Jacks recorded a version of the song which was released as a single in 1974 and reached #29 on the Adult Contemporary chart, #68 on the Billboard Hot 100, and went to #8 in the UK. The complex melody is partly derivative of classical music - the poignant "But if you stay..." passage comes from Franz Liszt's......</cite>
+              <div class="book-title">from Ray Charles</div>
+              <div class="informations">204 words</div>
+)SEARCHRESULT",
+
+R"SEARCHRESULT(
+            <a href="/ROOT/example/Wikibooks.html">
+              Wikibooks
+            </a>
+              <cite>...<b>Travel</b> guide Wikidata Knowledge database Commons Media repository Meta Coordination MediaWiki MediaWiki software Phabricator MediaWiki bug tracker Wikimedia Labs MediaWiki development The Wikimedia Foundation is a non-profit organization that depends on your voluntarism and donations to operate. If you find Wikibooks or other projects hosted by the Wikimedia Foundation useful, please volunteer or make a donation. Your donations primarily helps to purchase server equipment, launch new projects......</cite>
+              <div class="book-title">from Wikibooks</div>
+              <div class="informations">538 words</div>
+)SEARCHRESULT"
+      },
+      /* pagination */       {}
+    },
+
+    // Only RayCharles is in English.
+    // [TODO] We should extend our test data to have another zim file in english returning results.
+    {
+          /* query */          "pattern=travel"
+                               "&books.filter.lang=eng",
+          /* start */            0,
+          /* resultsPerPage */   10,
+          /* totalResultCount */ 1,
+          /* firstResultIndex */ 1,
+          /* results */          {
+R"SEARCHRESULT(
+            <a href="/ROOT/zimfile/A/If_You_Go_Away">
+              If You Go Away
+            </a>
+              <cite>...<b>Travel</b> On" (1965) "If You Go Away" (1966) "Walk Away" (1967) Damita Jo reached #10 on the Adult Contemporary chart and #68 on the Billboard Hot 100 in 1966 for her version of the song. Terry Jacks recorded a version of the song which was released as a single in 1974 and reached #29 on the Adult Contemporary chart, #68 on the Billboard Hot 100, and went to #8 in the UK. The complex melody is partly derivative of classical music - the poignant "But if you stay..." passage comes from Franz Liszt's......</cite>
+              <div class="book-title">from Ray Charles</div>
+              <div class="informations">204 words</div>
+)SEARCHRESULT",
+          },
+          /* pagination */       {}
+        },
+
+    // Adding a book (without match) doesn't change the results
+    {
+      /* query */          "pattern=jazz"
+                           "&books.id=" RAYCHARLESZIMID
+                           "&books.id=" EXAMPLEZIMID,
+      /* start */            -1,
+      /* resultsPerPage */   100,
+      /* totalResultCount */ 44,
+      /* firstResultIndex */ 1,
+      /* results */ LARGE_SEARCH_RESULTS,
+      /* pagination */ {}
+    },
+
+    {
+      /* query */          "pattern=jazz"
+                           "&books.filter.lang=eng",
+      /* start */            -1,
+      /* resultsPerPage */   5,
+      /* totalResultCount */ 44,
+      /* firstResultIndex */ 1,
+      /* results */ {
+        LARGE_SEARCH_RESULTS[0],
+        LARGE_SEARCH_RESULTS[1],
+        LARGE_SEARCH_RESULTS[2],
+        LARGE_SEARCH_RESULTS[3],
+        LARGE_SEARCH_RESULTS[4],
+      },
+
+      /* pagination */ {
+        { "1", 0,  true  },
+        { "2", 5,  false },
+        { "3", 10, false },
+        { "4", 15, false },
+        { "5", 20, false },
+        { "▶", 40, false },
+      }
+    },
+
+    {
+      /* query */          "pattern=jazz"
+                           "&books.filter.tag=wikipedia",
+      /* start */            -1,
+      /* resultsPerPage */   5,
+      /* totalResultCount */ 44,
+      /* firstResultIndex */ 1,
+      /* results */ {
+        LARGE_SEARCH_RESULTS[0],
+        LARGE_SEARCH_RESULTS[1],
+        LARGE_SEARCH_RESULTS[2],
+        LARGE_SEARCH_RESULTS[3],
+        LARGE_SEARCH_RESULTS[4],
+      },
+
+      /* pagination */ {
+        { "1", 0,  true  },
+        { "2", 5,  false },
+        { "3", 10, false },
+        { "4", 15, false },
+        { "5", 20, false },
+        { "▶", 40, false },
+      }
+    },
+
+    {
+      /* query */          "pattern=jazz"
+                           "&books.filter.lang=eng"
+                           "&books.filter.title=Ray%20Charles",
+      /* start */            -1,
+      /* resultsPerPage */   5,
+      /* totalResultCount */ 44,
+      /* firstResultIndex */ 1,
+      /* results */ {
+        LARGE_SEARCH_RESULTS[0],
+        LARGE_SEARCH_RESULTS[1],
+        LARGE_SEARCH_RESULTS[2],
+        LARGE_SEARCH_RESULTS[3],
+        LARGE_SEARCH_RESULTS[4],
+      },
+
+      /* pagination */ {
+        { "1", 0,  true  },
+        { "2", 5,  false },
+        { "3", 10, false },
+        { "4", 15, false },
+        { "5", 20, false },
+        { "▶", 40, false },
+      }
+    },
   };
 
   for ( const auto& t : testData ) {