smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Retrieve SuggestionSearcher from LRU Cache 

We create a cache for SuggestionSearcher very similar to that of FT
searcher. User can specify a custom cache size using the environment
variable SUGGESTION_SEARCHER_CACHE_SIZE. It has a default value of 10%
of the number of books in the library.
This commit is contained in:
Maneesh P M 2021-10-13 20:52:02 +05:30 committed by Matthieu Gautier
parent 7cb4c1361f
commit 6523d9f563
2 changed files with 15 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/server/internalServer.cpp
View File

@ -147,7 +147,8 @@ InternalServer::InternalServer(Library* library,
mp_library(library), mp_library(library),
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper), mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
searcherCache(getCacheLength("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))), searcherCache(getCacheLength("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
searchCache(getCacheLength("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)) searchCache(getCacheLength("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
suggestionSearcherCache(getCacheLength("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
{} {}
bool InternalServer::start() { bool InternalServer::start() {
@ -352,14 +353,15 @@ std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& r
* Archive and Zim handlers begin * Archive and Zim handlers begin
**/ **/
// TODO: retrieve searcher from caching mechanism SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Archive* const archive,
SuggestionsList_t getSuggestions(const zim::Archive* const archive, const std::string& bookId, const std::string& queryString, int start, int suggestionCount)
const std::string& queryString, int start, int suggestionCount)
{ {
SuggestionsList_t suggestions; SuggestionsList_t suggestions;
auto searcher = zim::SuggestionSearcher(*archive); std::shared_ptr<zim::SuggestionSearcher> searcher;
searcher = cache.getOrPut(bookId, [=](){ return make_shared<zim::SuggestionSearcher>(*archive); });
if (archive->hasTitleIndex()) { if (archive->hasTitleIndex()) {
auto search = searcher.suggest(queryString); auto search = searcher->suggest(queryString);
auto srs = search.getResults(start, suggestionCount); auto srs = search.getResults(start, suggestionCount);
for (auto it : srs) { for (auto it : srs) {
@ -372,7 +374,7 @@ SuggestionsList_t getSuggestions(const zim::Archive* const archive,
std::vector<std::string> variants = getTitleVariants(queryString); std::vector<std::string> variants = getTitleVariants(queryString);
int currCount = 0; int currCount = 0;
for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) { for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) {
auto search = searcher.suggest(queryString); auto search = searcher->suggest(queryString);
auto srs = search.getResults(0, suggestionCount); auto srs = search.getResults(0, suggestionCount);
for (auto it : srs) { for (auto it : srs) {
SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()), SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
@ -392,11 +394,11 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
printf("** running handle_suggest\n"); printf("** running handle_suggest\n");
} }
std::string bookName; std::string bookName, bookId;
std::shared_ptr<zim::Archive> archive; std::shared_ptr<zim::Archive> archive;
try { try {
bookName = request.get_argument("content"); bookName = request.get_argument("content");
const std::string bookId = mp_nameMapper->getIdForName(bookName); bookId = mp_nameMapper->getIdForName(bookName);
archive = mp_library->getArchiveById(bookId); archive = mp_library->getArchiveById(bookId);
} catch (const std::out_of_range&) { } catch (const std::out_of_range&) {
// error handled by the archive == nullptr check below // error handled by the archive == nullptr check below
@ -423,7 +425,8 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
bool first = true; bool first = true;
/* Get the suggestions */ /* Get the suggestions */
SuggestionsList_t suggestions = getSuggestions(archive.get(), queryString, start, count); SuggestionsList_t suggestions = getSuggestions(suggestionSearcherCache, archive.get(),
bookId, queryString, start, count);
for(auto& suggestion:suggestions) { for(auto& suggestion:suggestions) {
MustacheData result; MustacheData result;
result.set("label", suggestion.getTitle()); result.set("label", suggestion.getTitle());

2
src/server/internalServer.h
View File

@ -46,6 +46,7 @@ namespace kiwix {
typedef kainjow::mustache::data MustacheData; typedef kainjow::mustache::data MustacheData;
typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache; typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache;
typedef ConcurrentCache<string, std::shared_ptr<zim::Search>> SearchCache; typedef ConcurrentCache<string, std::shared_ptr<zim::Search>> SearchCache;
typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
class Entry; class Entry;
class OPDSDumper; class OPDSDumper;
@ -124,6 +125,7 @@ class InternalServer {
SearcherCache searcherCache; SearcherCache searcherCache;
SearchCache searchCache; SearchCache searchCache;
SuggestionSearcherCache suggestionSearcherCache;
std::string m_server_id; std::string m_server_id;
std::string m_library_id; std::string m_library_id;