Merge pull request #679 from kiwix/kiwix-serve-i18n

debian/libkiwix-dev.manpages

@@ -1 +1,2 @@
 usr/share/man/man1/kiwix-compile-resources.1*
+usr/share/man/man1/kiwix-compile-i18n.1*

scripts/kiwix-compile-i18n

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import os.path
+import re
+import json
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+def lang_code(filename):
+    filename = os.path.basename(filename)
+    lang = to_identifier(os.path.splitext(filename)[0])
+    print(filename, '->', lang)
+    return lang
+
+from string import Template
+
+def expand_cxx_template(t, **kwargs):
+    return Template(t).substitute(**kwargs)
+
+def cxx_string_literal(s):
+    # Taking advantage of the fact the JSON string escape rules match
+    # those of C++
+    return 'u8' + json.dumps(s)
+
+string_table_cxx_template = '''
+const I18nString $TABLE_NAME[] = {
+  $TABLE_ENTRIES
+};
+'''
+
+lang_table_entry_cxx_template = '''
+    {
+        $LANG_STRING_LITERAL,
+        ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
+        $STRING_TABLE_NAME
+    }'''
+
+cxxfile_template = '''// This file is automatically generated. Do not modify it.
+
+#include "server/i18n.h"
+
+namespace kiwix {
+namespace i18n {
+
+namespace
+{
+
+$STRING_DATA
+
+} // unnamed namespace
+
+#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
+
+extern const I18nStringTable stringTables[] = {
+  $LANG_TABLE
+};
+
+extern const size_t langCount = $LANG_COUNT;
+
+} // namespace i18n
+} // namespace kiwix
+'''
+
+class Resource:
+    def __init__(self, base_dirs, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.lang_code = lang_code(filename)
+        found = False
+        for base_dir in base_dirs:
+            try:
+                with open(os.path.join(base_dir, filename), 'r') as f:
+                    self.data = f.read()
+                found = True
+                break
+            except FileNotFoundError:
+                continue
+        if not found:
+            raise Exception("Impossible to find {}".format(filename))
+
+
+    def get_string_table_name(self):
+        return "string_table_for_" + self.lang_code
+
+    def get_string_table(self):
+        table_entries = ",\n  ".join(self.get_string_table_entries())
+        return expand_cxx_template(string_table_cxx_template,
+                    TABLE_NAME=self.get_string_table_name(),
+                    TABLE_ENTRIES=table_entries)
+
+    def get_string_table_entries(self):
+        d = json.loads(self.data)
+        for k in sorted(d.keys()):
+            if k != "@metadata":
+                key_string = cxx_string_literal(k)
+                value_string = cxx_string_literal(d[k])
+                yield '{ ' + key_string + ', ' + value_string + ' }'
+
+    def get_lang_table_entry(self):
+        return expand_cxx_template(lang_table_entry_cxx_template,
+                LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
+                STRING_TABLE_NAME=self.get_string_table_name())
+
+
+
+def gen_c_file(resources):
+    string_data = []
+    lang_table = []
+    for r in resources:
+        string_data.append(r.get_string_table())
+        lang_table.append(r.get_lang_table_entry())
+
+    return expand_cxx_template(cxxfile_template,
+                STRING_DATA="\n".join(string_data),
+                LANG_TABLE=",\n  ".join(lang_table),
+                LANG_COUNT=len(resources)
+           )
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        required=True,
+                        help='The Cpp file name to generate')
+    parser.add_argument('i18n_resource_file',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    base_dir = os.path.dirname(os.path.realpath(args.i18n_resource_file))
+    with open(args.i18n_resource_file, 'r') as f:
+        resources = [Resource([base_dir], filename)
+                        for filename in f.readlines()]
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources))
+

scripts/kiwix-compile-i18n.1

@@ -0,0 +1,18 @@
+.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
+.SH SYNOPSIS
+\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_file\fR
+.SH DESCRIPTION
+.TP
+i18n_resource_file
+The list of i18n resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+.SH AUTHOR
+Veloman Yunkan <veloman.yunkan@gmail.com>

scripts/kiwix-compile-resources

@@ -102,7 +102,7 @@ class Resource:
 
 
 
-master_c_template = """//This file is automaically generated. Do not modify it.
+master_c_template = """//This file is automatically generated. Do not modify it.
 
 #include <stdlib.h>
 #include <fstream>

scripts/meson.build

@@ -4,3 +4,9 @@ res_compiler = find_program('kiwix-compile-resources')
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
 
 install_man('kiwix-compile-resources.1')
+
+i18n_compiler = find_program('kiwix-compile-i18n')
+
+install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-i18n.1')

src/meson.build

@@ -28,10 +28,12 @@ kiwix_sources = [
   'server/response.cpp',
   'server/internalServer.cpp',
   'server/internalServer_catalog_v2.cpp',
+  'server/i18n.cpp',
   'opds_catalog.cpp',
   'version.cpp'
 ]
 kiwix_sources += lib_resources
+kiwix_sources += i18n_resources
 
 if host_machine.system() == 'windows'
   kiwix_sources += 'subprocess_windows.cpp'

src/server/i18n.cpp

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "i18n.h"
+
+#include "tools/otherTools.h"
+
+#include <algorithm>
+#include <map>
+
+namespace kiwix
+{
+
+const char* I18nStringTable::get(const std::string& key) const
+{
+  const I18nString* const begin = entries;
+  const I18nString* const end = begin + entryCount;
+  const I18nString* found = std::lower_bound(begin, end, key,
+      [](const I18nString& a, const std::string& k) {
+        return a.key < k;
+  });
+  return (found == end || found->key != key) ? nullptr : found->value;
+}
+
+namespace i18n
+{
+// this data is generated by the i18n resource compiler
+extern const I18nStringTable stringTables[];
+extern const size_t langCount;
+}
+
+namespace
+{
+
+class I18nStringDB
+{
+public: // functions
+  I18nStringDB() {
+    for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
+      const auto& t = kiwix::i18n::stringTables[i];
+      lang2TableMap[t.lang] = &t;
+    }
+    enStrings = lang2TableMap.at("en");
+  };
+
+  std::string get(const std::string& lang, const std::string& key) const {
+    const char* s = getStringsFor(lang)->get(key);
+    if ( s == nullptr ) {
+      s = enStrings->get(key);
+      if ( s == nullptr ) {
+        throw std::runtime_error("Invalid message id");
+      }
+    }
+    return s;
+  }
+
+private: // functions
+  const I18nStringTable* getStringsFor(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang);
+    } catch(const std::out_of_range&) {
+      return enStrings;
+    }
+  }
+
+private: // data
+  std::map<std::string, const I18nStringTable*> lang2TableMap;
+  const I18nStringTable* enStrings;
+};
+
+} // unnamed namespace
+
+std::string getTranslatedString(const std::string& lang, const std::string& key)
+{
+  static const I18nStringDB stringDb;
+
+  return stringDb.get(lang, key);
+}
+
+namespace i18n
+{
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params)
+{
+  const std::string tmpl = getTranslatedString(lang, key);
+  return render_template(tmpl, params);
+}
+
+} // namespace i18n
+
+std::string ParameterizedMessage::getText(const std::string& lang) const
+{
+  return i18n::expandParameterizedString(lang, msgId, params);
+}
+
+} // namespace kiwix

src/server/i18n.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_I18N
+#define KIWIX_SERVER_I18N
+
+#include <string>
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+struct I18nString {
+  const char* const key;
+  const char* const value;
+};
+
+struct I18nStringTable {
+  const char* const lang;
+  const size_t entryCount;
+  const I18nString* const entries;
+
+  const char* get(const std::string& key) const;
+};
+
+std::string getTranslatedString(const std::string& lang, const std::string& key);
+
+namespace i18n
+{
+
+typedef kainjow::mustache::object Parameters;
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params);
+
+class GetTranslatedString
+{
+public:
+  explicit GetTranslatedString(const std::string& lang) : m_lang(lang) {}
+
+  std::string operator()(const std::string& key) const
+  {
+    return getTranslatedString(m_lang, key);
+  }
+
+  std::string operator()(const std::string& key, const Parameters& params) const
+  {
+    return expandParameterizedString(m_lang, key, params);
+  }
+
+private:
+  const std::string m_lang;
+};
+
+} // namespace i18n
+
+struct ParameterizedMessage
+{
+public: // types
+  typedef kainjow::mustache::object Parameters;
+
+public: // functions
+  ParameterizedMessage(const std::string& msgId, const Parameters& params)
+    : msgId(msgId)
+    , params(params)
+  {}
+
+  std::string getText(const std::string& lang) const;
+
+private: // data
+  const std::string msgId;
+  const Parameters  params;
+};
+
+} // namespace kiwix
+
+#endif // KIWIX_SERVER_I18N

src/server/internalServer.cpp

@@ -55,6 +55,7 @@ extern "C" {
 #include "searcher.h"
 #include "search_renderer.h"
 #include "opds_dumper.h"
+#include "i18n.h"
 
 #include <zim/uuid.h>
 #include <zim/error.h>
@@ -442,14 +443,39 @@ SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Arch
 namespace
 {
 
-std::string noSuchBookErrorMsg(const std::string& bookName)
+std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
 {
-  return "No such book: " + bookName;
+  return i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
 }
 
-std::string noSearchResultsMsg()
+ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
 {
-  return "The fulltext search engine is not available for this content.";
+  return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
+}
+
+ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
+{
+  return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
+}
+
+ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
+{
+  return ParameterizedMessage("raw-entry-not-found",
+                              {
+                                {"DATATYPE", dt},
+                                {"ENTRY", entry},
+                              }
+  );
+}
+
+ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
+{
+  const ParameterizedMessage::Parameters noParams;
+  return ParameterizedMessage(msgId, noParams);
 }
 
 } // unnamed namespace
@@ -514,7 +540,8 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
   /* Propose the fulltext search if possible */
   if (archive->hasFulltextIndex()) {
     MustacheData result;
-    result.set("label", "containing '" + queryString + "'...");
+    const auto lang = request.get_user_language();
+    result.set("label", makeFulltextSearchSuggestion(lang, queryString));
     result.set("value", queryString + " ");
     result.set("kind", "pattern");
     result.set("first", first);
@@ -597,10 +624,10 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
       // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
       // (in case of zim file not containing a index)
       return HTTPErrorHtmlResponse(*this, request, MHD_HTTP_NOT_FOUND,
-                                   "Fulltext search unavailable",
+                                   "fulltext-search-unavailable",
-                                   "Not Found",
+                                   "404-page-heading",
                                    m_root + "/skin/search_results.css")
-           + noSearchResultsMsg()
+           + nonParameterizedMessage("no-search-results")
            + TaskbarInfo(searchInfo.bookName, archive.get());
     }
 
@@ -669,9 +696,8 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
     auto entry = archive->getRandomEntry();
     return build_redirect(bookName, getFinalItem(*archive, entry));
   } catch(zim::EntryNotFound& e) {
-    const std::string error_details = "Oops! Failed to pick a random article :(";
     return HTTP404HtmlResponse(*this, request)
-           + error_details
+           + nonParameterizedMessage("random-article-failure")
            + TaskbarInfo(bookName, archive.get());
   }
 }
@@ -812,13 +838,13 @@ std::string get_book_name(const RequestContext& request)
   }
 }
 
-std::string searchSuggestionHTML(const std::string& searchURL, const std::string& pattern)
+ParameterizedMessage suggestSearchMsg(const std::string& searchURL, const std::string& pattern)
 {
-  kainjow::mustache::mustache tmpl("Make a full text search for <a href=\"{{{searchURL}}}\">{{pattern}}</a>");
+  return ParameterizedMessage("suggest-search",
-  MustacheData data;
+                              {
-  data.set("pattern", pattern);
+                                { "PATTERN",    pattern   },
-  data.set("searchURL", searchURL);
+                                { "SEARCH_URL", searchURL }
-  return (tmpl.render(data));
+                              });
 }
 
 } // unnamed namespace
@@ -852,7 +878,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
     return HTTP404HtmlResponse(*this, request)
            + urlNotFoundMsg
-           + searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
            + TaskbarInfo(bookName);
   }
 
@@ -886,7 +912,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
     return HTTP404HtmlResponse(*this, request)
            + urlNotFoundMsg
-           + searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
            + TaskbarInfo(bookName, archive.get());
   }
 }
@@ -909,10 +935,9 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
   }
 
   if (kind != "meta" && kind!= "content") {
-    const std::string error_details = kind + " is not a valid request for raw content.";
     return HTTP404HtmlResponse(*this, request)
            + urlNotFoundMsg
-           + error_details;
+           + invalidRawAccessMsg(kind);
   }
 
   std::shared_ptr<zim::Archive> archive;
@@ -948,10 +973,9 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
     if (m_verbose.load()) {
       printf("Failed to find %s\n", itemPath.c_str());
     }
-    const std::string error_details = "Cannot find " + kind + " entry " + itemPath;
     return HTTP404HtmlResponse(*this, request)
            + urlNotFoundMsg
-           + error_details;
+           + rawEntryNotFoundMsg(kind, itemPath);
   }
 }
 

src/server/request_context.cpp

@@ -193,4 +193,17 @@ std::string RequestContext::get_query() const {
   return q;
 }
 
+std::string RequestContext::get_user_language() const
+{
+  try {
+    return get_argument("userlang");
+  } catch(const std::out_of_range&) {}
+
+  try {
+    return get_header("Accept-Language");
+  } catch(const std::out_of_range&) {}
+
+  return "en";
+}
+
 }

src/server/request_context.h

@@ -94,6 +94,8 @@ class RequestContext {
 
     bool can_compress() const { return acceptEncodingDeflate; }
 
+    std::string get_user_language() const;
+
   private: // data
     std::string full_url;
     std::string url;

src/server/response.cpp

@@ -87,6 +87,11 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
 const UrlNotFoundMsg urlNotFoundMsg;
 const InvalidUrlMsg invalidUrlMsg;
 
+std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
+{
+  return getTranslatedString(m_request.get_user_language(), msgId);
+}
+
 std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
 {
   auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
@@ -100,8 +105,8 @@ std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObjec
 HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
                                              const RequestContext& request,
                                              int httpStatusCode,
-                                             const std::string& pageTitleMsg,
+                                             const std::string& pageTitleMsgId,
-                                             const std::string& headingMsg,
+                                             const std::string& headingMsgId,
                                              const std::string& cssUrl)
   : ContentResponseBlueprint(&server,
                              &request,
@@ -112,8 +117,8 @@ HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
   kainjow::mustache::list emptyList;
   this->m_data = kainjow::mustache::object{
                     {"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
-                    {"PAGE_TITLE",   pageTitleMsg},
+                    {"PAGE_TITLE",   getMessage(pageTitleMsgId)},
-                    {"PAGE_HEADING", headingMsg},
+                    {"PAGE_HEADING", getMessage(headingMsgId)},
                     {"details", emptyList}
   };
 }
@@ -123,16 +128,15 @@ HTTP404HtmlResponse::HTTP404HtmlResponse(const InternalServer& server,
   : HTTPErrorHtmlResponse(server,
                           request,
                           MHD_HTTP_NOT_FOUND,
-                          "Content not found",
+                          "404-page-title",
-                          "Not Found")
+                          "404-page-heading")
 {
 }
 
 HTTPErrorHtmlResponse& HTTP404HtmlResponse::operator+(UrlNotFoundMsg /*unused*/)
 {
   const std::string requestUrl = m_request.get_full_url();
-  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{url}}" was not found on this server.)");
+  return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
-  return *this + msgTmpl.render({"url", requestUrl});
 }
 
 HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
@@ -141,13 +145,19 @@ HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
   return *this;
 }
 
+HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const ParameterizedMessage& details)
+{
+  return *this + details.getText(m_request.get_user_language());
+}
+
+
 HTTP400HtmlResponse::HTTP400HtmlResponse(const InternalServer& server,
                                          const RequestContext& request)
   : HTTPErrorHtmlResponse(server,
                           request,
                           MHD_HTTP_BAD_REQUEST,
-                          "Invalid request",
+                          "400-page-title",
-                          "Invalid request")
+                          "400-page-heading")
 {
 }
 
@@ -167,8 +177,8 @@ HTTP500HtmlResponse::HTTP500HtmlResponse(const InternalServer& server,
   : HTTPErrorHtmlResponse(server,
                           request,
                           MHD_HTTP_INTERNAL_SERVER_ERROR,
-                          "Internal Server Error",
+                          "500-page-title",
-                          "Internal Server Error")
+                          "500-page-heading")
 {
   // operator+() is a state-modifying operator (akin to operator+=)
   *this + "An internal server error occured. We are sorry about that :/";
@@ -270,14 +280,20 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar()
+void ContentResponse::introduce_taskbar(const std::string& lang)
 {
-  kainjow::mustache::data data;
+  i18n::GetTranslatedString t(lang);
-  data.set("root", m_root);
+  kainjow::mustache::object data{
-  data.set("content", m_bookName);
+    {"root", m_root},
-  data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
+    {"content", m_bookName},
-  data.set("title", m_bookTitle);
+    {"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
-  data.set("withlibrarybutton", m_withLibraryButton);
+    {"title", m_bookTitle},
+    {"withlibrarybutton", m_withLibraryButton},
+    {"LIBRARY_BUTTON_TEXT", t("library-button-text")},
+    {"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
+    {"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
+    {"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
+  };
   auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
   m_content = prependToFirstOccurence(
     m_content,
@@ -342,7 +358,7 @@ ContentResponse::create_mhd_response(const RequestContext& request)
     inject_root_link();
 
     if (m_withTaskbar) {
-      introduce_taskbar();
+      introduce_taskbar(request.get_user_language());
     }
     if (m_blockExternalLinks) {
       inject_externallinks_blocker();

src/server/response.h

@@ -28,6 +28,7 @@
 #include "byte_range.h"
 #include "entry.h"
 #include "etag.h"
+#include "i18n.h"
 
 extern "C" {
 #include "microhttpd_wrapper.h"
@@ -105,7 +106,7 @@ class ContentResponse : public Response {
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar();
+    void introduce_taskbar(const std::string& lang);
     void inject_externallinks_blocker();
     void inject_root_link();
     bool can_compress(const RequestContext& request) const;
@@ -166,6 +167,7 @@ public: // functions
   ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
 
 protected: // functions
+  std::string getMessage(const std::string& msgId) const;
   virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
 
 public: //data
@@ -183,12 +185,13 @@ struct HTTPErrorHtmlResponse : ContentResponseBlueprint
   HTTPErrorHtmlResponse(const InternalServer& server,
                       const RequestContext& request,
                       int httpStatusCode,
-                      const std::string& pageTitleMsg,
+                      const std::string& pageTitleMsgId,
-                      const std::string& headingMsg,
+                      const std::string& headingMsgId,
                       const std::string& cssUrl = "");
 
   using ContentResponseBlueprint::operator+;
   HTTPErrorHtmlResponse& operator+(const std::string& msg);
+  HTTPErrorHtmlResponse& operator+(const ParameterizedMessage& errorDetails);
 };
 
 class UrlNotFoundMsg {};

static/i18n/en.json

@@ -0,0 +1,26 @@
+{
+	"@metadata": {
+		"authors": [
+		]
+	},
+	"name":"English",
+	"suggest-full-text-search": "containing '{{{SEARCH_TERMS}}}'..."
+	, "no-such-book": "No such book: {{BOOK_NAME}}"
+	, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
+	, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
+	, "random-article-failure" :  "Oops! Failed to pick a random article :("
+	, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
+	, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
+	, "400-page-title" : "Invalid request"
+	, "400-page-heading" : "Invalid request"
+	, "404-page-title" : "Content not found"
+	, "404-page-heading" : "Not Found"
+	, "500-page-title" : "Internal Server Error"
+	, "500-page-heading" : "Internal Server Error"
+	, "fulltext-search-unavailable" : "Fulltext search unavailable"
+	, "no-search-results": "The fulltext search engine is not available for this content."
+	, "library-button-text": "Go to welcome page"
+	, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
+	, "random-page-button-text": "Go to a randomly selected page"
+	, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
+}

static/i18n/hy.json

@@ -0,0 +1,17 @@
+{
+	"@metadata": {
+		"authors": [
+		]
+	},
+	"name":"Հայերեն",
+	"suggest-full-text-search": "որոնել '{{{SEARCH_TERMS}}}'..."
+	, "no-such-book": "Գիրքը բացակայում է՝ {{BOOK_NAME}}"
+	, "url-not-found" : "Սխալ հասցե՝ {{url}}"
+	, "suggest-search" : "Որոնել <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
+	, "404-page-title" : "Սխալ հասցե"
+	, "404-page-heading" : "Սխալ հասցե"
+	, "library-button-text": "Գրադարանի էջ"
+	, "home-button-text": "Դեպի '{{BOOK_TITLE}}'֊ի գլխավոր էջը"
+	, "random-page-button-text": "Բացել պատահական էջ"
+	, "searchbox-tooltip": "Որոնել '{{BOOK_TITLE}}'֊ում"
+}

static/i18n/qqq.json

@@ -0,0 +1,27 @@
+{
+	"@metadata": {
+		"authors": [
+			"Veloman Yunkan"
+		]
+	},
+	"name": "Current language to which the string is being translated to.",
+	"suggest-full-text-search": "Text appearing in the suggestion list that, when selected, runs a full text search instead of the title search"
+	, "no-such-book": "Error text when the requested book is not found in the library"
+	, "url-not-found" : "Error text about wrong URL for an HTTP 404 error"
+	, "suggest-search" : "Suggest a search when the URL points to a non existing article"
+	, "random-article-failure" :  "Failure of the random article selection procedure"
+	, "invalid-raw-data-type" : "Invalid DATATYPE was used with the /raw endpoint (/raw/<book>/DATATYPE/...); allowed values are 'meta' and 'content'"
+	, "raw-entry-not-found" : "Entry requested via the /raw endpoint was not found"
+	, "400-page-title" : "Title of the 400 error page"
+	, "400-page-heading" : "Heading of the 400 error page"
+	, "404-page-title" : "Title of the 404 error page"
+	, "404-page-heading" : "Heading of the 404 error page"
+	, "500-page-title" : "Title of the 500 error page"
+	, "500-page-heading" : "Heading of the 500 error page"
+	, "fulltext-search-unavailable" : "Title of the error page returned when search is attempted in a book without fulltext search database"
+	, "no-search-results": "Text of the error page returned when search is attempted in a book without fulltext search database"
+	, "library-button-text": "Tooltip of the button leading to the welcome page"
+	, "home-button-text": "Tooltip of the button leading to the main page of a book"
+	, "random-page-button-text": "Tooltip of the button opening a randomly selected page"
+	, "searchbox-tooltip": "Tooltip displayed for the search box"
+}

static/i18n_resources_list.txt

@@ -0,0 +1,2 @@
+i18n/en.json
+i18n/hy.json

static/meson.build

@@ -14,3 +14,18 @@ lib_resources = custom_target('resources',
              '@INPUT@'],
     depend_files: resource_files
 )
+
+i18n_resource_files = run_command(find_program('python3'),
+                    '-c',
+                    'import sys; f=open(sys.argv[1]); print(f.read())',
+                    files('i18n_resources_list.txt')
+                 ).stdout().strip().split('\n')
+
+i18n_resources = custom_target('i18n_resources',
+    input: 'i18n_resources_list.txt',
+    output: ['libkiwix-i18n-resources.cpp'],
+    command:[i18n_compiler,
+             '--cxxfile', '@OUTPUT0@',
+             '@INPUT@'],
+    depend_files: i18n_resource_files
+)

static/skin/taskbar.js

@@ -12,9 +12,10 @@ jq(document).ready(() => {
                ? (new URLSearchParams(window.location.search)).get('content') 
                : window.location.pathname.split(`${root}/`)[1].split('/')[0];
     
+        const userlang = (new URLSearchParams(window.location.search)).get('userlang') || "en";
         $( "#kiwixsearchbox" ).autocomplete({
     
-            source: `${root}/suggest?content=${bookName}`,
+            source: `${root}/suggest?content=${bookName}&userlang=${userlang}`,
             dataType: "json",
             cache: false,
     

static/templates/taskbar_part.html

@@ -5,18 +5,18 @@
         <form class="kiwixsearch" method="GET" action="{{root}}/search" id="kiwixsearchform">
           {{#hascontent}}<input type="hidden" name="content" value="{{content}}" />{{/hascontent}}
           <label for="kiwixsearchbox">&#x1f50d;</label>
-          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search '{{title}}'" aria-label="Search '{{title}}'">
+          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="{{{SEARCHBOX_TOOLTIP}}}" aria-label="{{{SEARCHBOX_TOOLTIP}}}">
         </form>
       </div>
         <input type="checkbox" id="kiwix_button_show_toggle">
         <label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
         <div class="kiwix_button_cont">
           {{#withlibrarybutton}}
-            <a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="{{root}}/"><button>&#x1f3e0;</button></a>
+            <a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>&#x1f3e0;</button></a>
           {{/withlibrarybutton}}
           {{#hascontent}}
-          <a id="kiwix_serve_taskbar_home_button" title="Go to the main page of '{{title}}'" aria-label="Go to the main page of '{{title}}'" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
+          <a id="kiwix_serve_taskbar_home_button" title="{{{HOME_BUTTON_TEXT}}}" aria-label="{{{HOME_BUTTON_TEXT}}}" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
-          <a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
+          <a id="kiwix_serve_taskbar_random_button" title="{{{RANDOM_PAGE_BUTTON_TEXT}}}" aria-label="{{{RANDOM_PAGE_BUTTON_TEXT}}}"
             href="{{root}}/random?content={{#urlencoded}}{{{content}}}{{/urlencoded}}"><button>&#x1F3B2;</button></a>
           {{/hascontent}}
         </div>

test/server.cpp

@@ -57,7 +57,6 @@ std::string removeEOLWhitespaceMarkers(const std::string& s)
   return std::regex_replace(s, pattern, "");
 }
 
-
 class ZimFileServer
 {
 public: // types
@@ -411,11 +410,13 @@ public:
   std::string expectedResponse() const;
 
 private:
+  bool isTranslatedVersion() const;
   virtual std::string pageTitle() const;
   std::string pageCssLink() const;
   std::string hiddenBookNameInput() const;
   std::string searchPatternInput() const;
   std::string taskbarLinks() const;
+  std::string goToWelcomePageText() const;
 };
 
 std::string TestContentIn404HtmlResponse::expectedResponse() const
@@ -454,7 +455,11 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
         <input type="checkbox" id="kiwix_button_show_toggle">
         <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
         <div class="kiwix_button_cont">
-            <a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="/ROOT/"><button>&#x1f3e0;</button></a>
+            <a id="kiwix_serve_taskbar_library_button" title=")FRAG",
+
+  R"FRAG(" aria-label=")FRAG",
+
+  R"FRAG(" href="/ROOT/"><button>&#x1f3e0;</button></a>
           )FRAG",
 
   R"FRAG(
@@ -478,10 +483,14 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
        + frag[3]
        + searchPatternInput()
        + frag[4]
-       + taskbarLinks()
+       + goToWelcomePageText()
        + frag[5]
-       + removeEOLWhitespaceMarkers(expectedBody)
+       + goToWelcomePageText()
-       + frag[6];
+       + frag[6]
+       + taskbarLinks()
+       + frag[7]
+       + expectedBody
+       + frag[8];
 }
 
 std::string TestContentIn404HtmlResponse::pageTitle() const
@@ -510,11 +519,15 @@ std::string TestContentIn404HtmlResponse::hiddenBookNameInput() const
 
 std::string TestContentIn404HtmlResponse::searchPatternInput() const
 {
-  return R"(          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search ')"
+  const std::string searchboxTooltip = isTranslatedVersion()
-       + bookTitle
+                                     ? "Որոնել '" + bookTitle + "'֊ում"
-       + R"('" aria-label="Search ')"
+                                     : "Search '" + bookTitle + "'";
-       + bookTitle
+
-       + R"('">
+  return R"(          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title=")"
+       + searchboxTooltip
+       + R"(" aria-label=")"
+       + searchboxTooltip
+       + R"(">
 )";
 }
 
@@ -523,21 +536,46 @@ std::string TestContentIn404HtmlResponse::taskbarLinks() const
   if ( bookName.empty() )
     return "";
 
-  return R"(<a id="kiwix_serve_taskbar_home_button" title="Go to the main page of ')"
+  const auto goToMainPageOfBook = isTranslatedVersion()
-       + bookTitle
+                                ? "Դեպի '" + bookTitle + "'֊ի գլխավոր էջը"
-       + R"('" aria-label="Go to the main page of ')"
+                                : "Go to the main page of '" + bookTitle + "'";
-       + bookTitle
+
-       + R"('" href="/ROOT/)"
+  const std::string goToRandomPage = isTranslatedVersion()
+                                   ? "Բացել պատահական էջ"
+                                   : "Go to a randomly selected page";
+
+  return R"(<a id="kiwix_serve_taskbar_home_button" title=")"
+       + goToMainPageOfBook
+       + R"(" aria-label=")"
+       + goToMainPageOfBook
+       + R"(" href="/ROOT/)"
        + bookName
        + R"(/"><button>)"
        + bookTitle
        + R"(</button></a>
-          <a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
+          <a id="kiwix_serve_taskbar_random_button" title=")"
+       + goToRandomPage
+       + R"(" aria-label=")"
+       + goToRandomPage
+       + R"("
             href="/ROOT/random?content=)"
        + bookName
        + R"("><button>&#x1F3B2;</button></a>)";
 }
 
+bool TestContentIn404HtmlResponse::isTranslatedVersion() const
+{
+  return url.find("userlang=hy") != std::string::npos;
+}
+
+std::string TestContentIn404HtmlResponse::goToWelcomePageText() const
+{
+  return isTranslatedVersion()
+       ? "Գրադարանի էջ"
+       : "Go to welcome page";
+}
+
+
 class TestContentIn400HtmlResponse : public TestContentIn404HtmlResponse
 {
 public:
@@ -556,7 +594,6 @@ std::string TestContentIn400HtmlResponse::pageTitle() const {
      : expectedPageTitle;
 }
 
-
 } // namespace TestingOfHtmlResponses
 
 TEST_F(ServerTest, 404WithBodyTesting)
@@ -571,6 +608,15 @@ TEST_F(ServerTest, 404WithBodyTesting)
     </p>
 )"  },
 
+    { /* url */ "/ROOT/random?content=non-existent-book&userlang=hy",
+      expected_page_title=="Սխալ հասցե" &&
+      expected_body==R"(
+    <h1>Սխալ հասցե</h1>
+    <p>
+      Գիրքը բացակայում է՝ non-existent-book
+    </p>
+)"  },
+
     { /* url */ "/ROOT/suggest?content=no-such-book&term=whatever",
       expected_body==R"(
     <h1>Not Found</h1>
@@ -587,6 +633,15 @@ TEST_F(ServerTest, 404WithBodyTesting)
     </p>
 )"  },
 
+    { /* url */ "/ROOT/catalog/?userlang=hy",
+      expected_page_title=="Սխալ հասցե" &&
+      expected_body==R"(
+    <h1>Սխալ հասցե</h1>
+    <p>
+      Սխալ հասցե՝ /ROOT/catalog/
+    </p>
+)"  },
+
     { /* url */ "/ROOT/catalog/invalid_endpoint",
       expected_body==R"(
     <h1>Not Found</h1>
@@ -595,6 +650,15 @@ TEST_F(ServerTest, 404WithBodyTesting)
     </p>
 )"  },
 
+    { /* url */ "/ROOT/catalog/invalid_endpoint?userlang=hy",
+      expected_page_title=="Սխալ հասցե" &&
+      expected_body==R"(
+    <h1>Սխալ հասցե</h1>
+    <p>
+      Սխալ հասցե՝ /ROOT/catalog/invalid_endpoint
+    </p>
+)"  },
+
     { /* url */ "/ROOT/invalid-book/whatever",
       expected_body==R"(
     <h1>Not Found</h1>
@@ -643,6 +707,20 @@ TEST_F(ServerTest, 404WithBodyTesting)
     </p>
 )"  },
 
+    { /* url */ "/ROOT/zimfile/invalid-article?userlang=hy",
+      expected_page_title=="Սխալ հասցե" &&
+      book_name=="zimfile" &&
+      book_title=="Ray Charles" &&
+      expected_body==R"(
+    <h1>Սխալ հասցե</h1>
+    <p>
+      Սխալ հասցե՝ /ROOT/zimfile/invalid-article
+    </p>
+    <p>
+      Որոնել <a href="/ROOT/search?content=zimfile&pattern=invalid-article">invalid-article</a>
+    </p>
+)"  },
+
     { /* url */ "/ROOT/raw/no-such-book/meta/Title",
       expected_body==R"(
     <h1>Not Found</h1>
@@ -800,6 +878,79 @@ TEST_F(ServerTest, 500)
   EXPECT_EQ(r->body, expectedBody);
 }
 
+TEST_F(ServerTest, UserLanguageControl)
+{
+  struct TestData
+  {
+    const std::string url;
+    const std::string acceptLanguageHeader;
+    const std::string expectedH1;
+
+    operator TestContext() const
+    {
+      return TestContext{
+          {"url", url},
+          {"acceptLanguageHeader", acceptLanguageHeader},
+      };
+    }
+  };
+
+  const TestData testData[] = {
+    {
+      /*url*/ "/ROOT/zimfile/invalid-article",
+      /*Accept-Language:*/ "",
+      /* expected <h1> */ "Not Found"
+    },
+    {
+      /*url*/ "/ROOT/zimfile/invalid-article?userlang=en",
+      /*Accept-Language:*/ "",
+      /* expected <h1> */ "Not Found"
+    },
+    {
+      /*url*/ "/ROOT/zimfile/invalid-article?userlang=hy",
+      /*Accept-Language:*/ "",
+      /* expected <h1> */ "Սխալ հասցե"
+    },
+    {
+      /*url*/ "/ROOT/zimfile/invalid-article",
+      /*Accept-Language:*/ "*",
+      /* expected <h1> */ "Not Found"
+    },
+    {
+      /*url*/ "/ROOT/zimfile/invalid-article",
+      /*Accept-Language:*/ "hy",
+      /* expected <h1> */ "Սխալ հասցե"
+    },
+    {
+      // userlang query parameter takes precedence over Accept-Language
+      /*url*/ "/ROOT/zimfile/invalid-article?userlang=en",
+      /*Accept-Language:*/ "hy",
+      /* expected <h1> */ "Not Found"
+    },
+    {
+      // The value of the Accept-Language header is not currently parsed.
+      // In case of a comma separated list of languages (optionally weighted
+      // with quality values) the default (en) language is used instead.
+      /*url*/ "/ROOT/zimfile/invalid-article",
+      /*Accept-Language:*/ "hy;q=0.9, en;q=0.2",
+      /* expected <h1> */ "Not Found"
+    },
+  };
+
+  const std::regex h1Regex("<h1>(.+)</h1>");
+  for ( const auto& t : testData ) {
+    std::smatch h1Match;
+    Headers headers;
+    if ( !t.acceptLanguageHeader.empty() ) {
+      headers.insert({"Accept-Language", t.acceptLanguageHeader});
+    }
+    const auto r = zfs1_->GET(t.url.c_str(), headers);
+    std::regex_search(r->body, h1Match, h1Regex);
+    const std::string h1(h1Match[1]);
+    EXPECT_EQ(h1, t.expectedH1) << t;
+  }
+}
+
 TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
 {
   auto g = zfs1_->GET("/ROOT/random?content=zimfile");
@@ -1182,6 +1333,17 @@ R"EXPECTEDRESPONSE([
     //EOLWHITESPACEMARKER
   }
 ]
+)EXPECTEDRESPONSE"
+    },
+    { /* url: */ "/ROOT/suggest?content=zimfile&term=abracadabra&userlang=hy",
+R"EXPECTEDRESPONSE([
+  {
+    "value" : "abracadabra ",
+    "label" : "որոնել &apos;abracadabra&apos;...",
+    "kind" : "pattern"
+    //EOLWHITESPACEMARKER
+  }
+]
 )EXPECTEDRESPONSE"
     },
   };