Move the searcher cache into the Library

This commit is contained in:
Matthieu Gautier 2022-04-27 15:18:09 +02:00
parent 740581c55c
commit f5af0633ec
4 changed files with 30 additions and 12 deletions

View File

@ -26,6 +26,7 @@
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <zim/archive.h> #include <zim/archive.h>
#include <zim/search.h>
#include "book.h" #include "book.h"
#include "bookmark.h" #include "bookmark.h"
@ -207,6 +208,7 @@ class Library
DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id); DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
std::shared_ptr<zim::Archive> getArchiveById(const std::string& id); std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
std::shared_ptr<zim::Searcher> getSearcherById(const std::string& id);
/** /**
* Remove a book from the library. * Remove a book from the library.

View File

@ -66,13 +66,14 @@ struct Library::Impl
struct Entry : Book struct Entry : Book
{ {
Library::Revision lastUpdatedRevision = 0; Library::Revision lastUpdatedRevision = 0;
}; };
Library::Revision m_revision; Library::Revision m_revision;
std::map<std::string, Entry> m_books; std::map<std::string, Entry> m_books;
using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>; using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
std::unique_ptr<ArchiveCache> mp_archiveCache; std::unique_ptr<ArchiveCache> mp_archiveCache;
using SearcherCache = ConcurrentCache<std::string, std::shared_ptr<zim::Searcher>>;
std::unique_ptr<SearcherCache> mp_searcherCache;
std::vector<kiwix::Bookmark> m_bookmarks; std::vector<kiwix::Bookmark> m_bookmarks;
Xapian::WritableDatabase m_bookDB; Xapian::WritableDatabase m_bookDB;
@ -87,6 +88,7 @@ struct Library::Impl
Library::Impl::Impl() Library::Impl::Impl()
: mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("ARCHIVE_CACHE_SIZE", 1), 1))), : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("ARCHIVE_CACHE_SIZE", 1), 1))),
mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("SEARCHER_CACHE_SIZE", 1), 1))),
m_bookDB("", Xapian::DB_BACKEND_INMEMORY) m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
{ {
} }
@ -156,6 +158,9 @@ bool Library::addBook(const Book& book)
if (getEnvVar<int>("ARCHIVE_CACHE_SIZE", -1) <= 0) { if (getEnvVar<int>("ARCHIVE_CACHE_SIZE", -1) <= 0) {
mp_impl->mp_archiveCache->setMaxSize(new_cache_size); mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
} }
if (getEnvVar<int>("SEARCHER_CACHE_SIZE", -1) <= 0) {
mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
}
return true; return true;
} }
} }
@ -182,6 +187,7 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
void Library::dropCache(const std::string& id) void Library::dropCache(const std::string& id)
{ {
mp_impl->mp_archiveCache->drop(id); mp_impl->mp_archiveCache->drop(id);
mp_impl->mp_searcherCache->drop(id);
} }
bool Library::removeBookById(const std::string& id) bool Library::removeBookById(const std::string& id)
@ -275,7 +281,22 @@ std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
} catch (std::invalid_argument&) { } catch (std::invalid_argument&) {
return nullptr; return nullptr;
} }
}
std::shared_ptr<zim::Searcher> Library::getSearcherById(const std::string& id)
{
try {
return mp_impl->mp_searcherCache->getOrPut(id,
[&](){
auto archive = getArchiveById(id);
if(!archive) {
throw std::invalid_argument("");
}
return std::make_shared<zim::Searcher>(*archive);
});
} catch (std::invalid_argument&) {
return nullptr;
}
} }
unsigned int Library::getBookCount(const bool localBooks, unsigned int Library::getBookCount(const bool localBooks,

View File

@ -182,7 +182,6 @@ InternalServer::InternalServer(Library* library,
mp_daemon(nullptr), mp_daemon(nullptr),
mp_library(library), mp_library(library),
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper), mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
searcherCache(getEnvVar<int>("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
searchCache(getEnvVar<int>("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)), searchCache(getEnvVar<int>("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
suggestionSearcherCache(getEnvVar<int>("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))) suggestionSearcherCache(getEnvVar<int>("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
{} {}
@ -582,11 +581,9 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
auto searchInfo = SearchInfo(request); auto searchInfo = SearchInfo(request);
std::string bookId; std::string bookId;
std::shared_ptr<zim::Archive> archive;
if (!searchInfo.bookName.empty()) { if (!searchInfo.bookName.empty()) {
try { try {
bookId = mp_nameMapper->getIdForName(searchInfo.bookName); bookId = mp_nameMapper->getIdForName(searchInfo.bookName);
archive = mp_library->getArchiveById(bookId);
} catch (const std::out_of_range&) { } catch (const std::out_of_range&) {
throw std::invalid_argument("The requested book doesn't exist."); throw std::invalid_argument("The requested book doesn't exist.");
} }
@ -600,8 +597,8 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
search = searchCache.getOrPut(searchInfo, search = searchCache.getOrPut(searchInfo,
[=](){ [=](){
std::shared_ptr<zim::Searcher> searcher; std::shared_ptr<zim::Searcher> searcher;
if (archive) { if(!bookId.empty()) {
searcher = searcherCache.getOrPut(bookId, [=](){ return std::make_shared<zim::Searcher>(*archive);}); searcher = mp_library->getSearcherById(bookId);
} else { } else {
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto currentArchive = mp_library->getArchiveById(bookId); auto currentArchive = mp_library->getArchiveById(bookId);
@ -626,7 +623,7 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
"404-page-heading", "404-page-heading",
cssUrl); cssUrl);
response += nonParameterizedMessage("no-search-results"); response += nonParameterizedMessage("no-search-results");
response += TaskbarInfo(searchInfo.bookName, archive.get()); response += TaskbarInfo(searchInfo.bookName, bookId.empty()?nullptr:mp_library->getArchiveById(bookId).get());
return response; return response;
} }
@ -656,7 +653,7 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
renderer.setSearchProtocolPrefix(m_root + "/search"); renderer.setSearchProtocolPrefix(m_root + "/search");
renderer.setPageLength(pageLength); renderer.setPageLength(pageLength);
auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8"); auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
response->set_taskbar(searchInfo.bookName, archive.get()); response->set_taskbar(searchInfo.bookName, bookId.empty()?nullptr:mp_library->getArchiveById(bookId).get());
return std::move(response); return std::move(response);
} catch (const std::invalid_argument& e) { } catch (const std::invalid_argument& e) {
return HTTP400HtmlResponse(*this, request) return HTTP400HtmlResponse(*this, request)

View File

@ -88,7 +88,6 @@ class SearchInfo {
typedef kainjow::mustache::data MustacheData; typedef kainjow::mustache::data MustacheData;
typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache;
typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache; typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache; typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
@ -167,7 +166,6 @@ class InternalServer {
Library* mp_library; Library* mp_library;
NameMapper* mp_nameMapper; NameMapper* mp_nameMapper;
SearcherCache searcherCache;
SearchCache searchCache; SearchCache searchCache;
SuggestionSearcherCache suggestionSearcherCache; SuggestionSearcherCache suggestionSearcherCache;