Merge pull request #679 from kiwix/kiwix-serve-i18n

This commit is contained in:
Matthieu Gautier 2022-04-14 15:21:47 +02:00 committed by GitHub
commit c43c637bea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 772 additions and 68 deletions

View File

@ -1 +1,2 @@
usr/share/man/man1/kiwix-compile-resources.1*
usr/share/man/man1/kiwix-compile-i18n.1*

161
scripts/kiwix-compile-i18n Executable file
View File

@ -0,0 +1,161 @@
#!/usr/bin/env python3
'''
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
'''
import argparse
import os.path
import re
import json
def to_identifier(name):
ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
if ident[0].isnumeric():
return "_"+ident
return ident
def lang_code(filename):
filename = os.path.basename(filename)
lang = to_identifier(os.path.splitext(filename)[0])
print(filename, '->', lang)
return lang
from string import Template
def expand_cxx_template(t, **kwargs):
return Template(t).substitute(**kwargs)
def cxx_string_literal(s):
# Taking advantage of the fact the JSON string escape rules match
# those of C++
return 'u8' + json.dumps(s)
string_table_cxx_template = '''
const I18nString $TABLE_NAME[] = {
$TABLE_ENTRIES
};
'''
lang_table_entry_cxx_template = '''
{
$LANG_STRING_LITERAL,
ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
$STRING_TABLE_NAME
}'''
cxxfile_template = '''// This file is automatically generated. Do not modify it.
#include "server/i18n.h"
namespace kiwix {
namespace i18n {
namespace
{
$STRING_DATA
} // unnamed namespace
#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
extern const I18nStringTable stringTables[] = {
$LANG_TABLE
};
extern const size_t langCount = $LANG_COUNT;
} // namespace i18n
} // namespace kiwix
'''
class Resource:
def __init__(self, base_dirs, filename):
filename = filename.strip()
self.filename = filename
self.lang_code = lang_code(filename)
found = False
for base_dir in base_dirs:
try:
with open(os.path.join(base_dir, filename), 'r') as f:
self.data = f.read()
found = True
break
except FileNotFoundError:
continue
if not found:
raise Exception("Impossible to find {}".format(filename))
def get_string_table_name(self):
return "string_table_for_" + self.lang_code
def get_string_table(self):
table_entries = ",\n ".join(self.get_string_table_entries())
return expand_cxx_template(string_table_cxx_template,
TABLE_NAME=self.get_string_table_name(),
TABLE_ENTRIES=table_entries)
def get_string_table_entries(self):
d = json.loads(self.data)
for k in sorted(d.keys()):
if k != "@metadata":
key_string = cxx_string_literal(k)
value_string = cxx_string_literal(d[k])
yield '{ ' + key_string + ', ' + value_string + ' }'
def get_lang_table_entry(self):
return expand_cxx_template(lang_table_entry_cxx_template,
LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
STRING_TABLE_NAME=self.get_string_table_name())
def gen_c_file(resources):
string_data = []
lang_table = []
for r in resources:
string_data.append(r.get_string_table())
lang_table.append(r.get_lang_table_entry())
return expand_cxx_template(cxxfile_template,
STRING_DATA="\n".join(string_data),
LANG_TABLE=",\n ".join(lang_table),
LANG_COUNT=len(resources)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--cxxfile',
required=True,
help='The Cpp file name to generate')
parser.add_argument('i18n_resource_file',
help='The list of resources to compile.')
args = parser.parse_args()
base_dir = os.path.dirname(os.path.realpath(args.i18n_resource_file))
with open(args.i18n_resource_file, 'r') as f:
resources = [Resource([base_dir], filename)
for filename in f.readlines()]
with open(args.cxxfile, 'w') as f:
f.write(gen_c_file(resources))

View File

@ -0,0 +1,18 @@
.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
.SH NAME
kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
.SH SYNOPSIS
\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_file\fR
.SH DESCRIPTION
.TP
i18n_resource_file
The list of i18n resources to compile.
.TP
\fB\-h\fR, \fB\-\-help\fR
show a help message and exit
.TP
\fB\-\-cxxfile\fR CXXFILE
The Cpp file name to generate
.TP
.SH AUTHOR
Veloman Yunkan <veloman.yunkan@gmail.com>

View File

@ -102,7 +102,7 @@ class Resource:
master_c_template = """//This file is automaically generated. Do not modify it.
master_c_template = """//This file is automatically generated. Do not modify it.
#include <stdlib.h>
#include <fstream>

View File

@ -4,3 +4,9 @@ res_compiler = find_program('kiwix-compile-resources')
install_data(res_compiler.path(), install_dir:get_option('bindir'))
install_man('kiwix-compile-resources.1')
i18n_compiler = find_program('kiwix-compile-i18n')
install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
install_man('kiwix-compile-i18n.1')

View File

@ -28,10 +28,12 @@ kiwix_sources = [
'server/response.cpp',
'server/internalServer.cpp',
'server/internalServer_catalog_v2.cpp',
'server/i18n.cpp',
'opds_catalog.cpp',
'version.cpp'
]
kiwix_sources += lib_resources
kiwix_sources += i18n_resources
if host_machine.system() == 'windows'
kiwix_sources += 'subprocess_windows.cpp'

114
src/server/i18n.cpp Normal file
View File

@ -0,0 +1,114 @@
/*
* Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "i18n.h"
#include "tools/otherTools.h"
#include <algorithm>
#include <map>
namespace kiwix
{
const char* I18nStringTable::get(const std::string& key) const
{
const I18nString* const begin = entries;
const I18nString* const end = begin + entryCount;
const I18nString* found = std::lower_bound(begin, end, key,
[](const I18nString& a, const std::string& k) {
return a.key < k;
});
return (found == end || found->key != key) ? nullptr : found->value;
}
namespace i18n
{
// this data is generated by the i18n resource compiler
extern const I18nStringTable stringTables[];
extern const size_t langCount;
}
namespace
{
class I18nStringDB
{
public: // functions
I18nStringDB() {
for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
const auto& t = kiwix::i18n::stringTables[i];
lang2TableMap[t.lang] = &t;
}
enStrings = lang2TableMap.at("en");
};
std::string get(const std::string& lang, const std::string& key) const {
const char* s = getStringsFor(lang)->get(key);
if ( s == nullptr ) {
s = enStrings->get(key);
if ( s == nullptr ) {
throw std::runtime_error("Invalid message id");
}
}
return s;
}
private: // functions
const I18nStringTable* getStringsFor(const std::string& lang) const {
try {
return lang2TableMap.at(lang);
} catch(const std::out_of_range&) {
return enStrings;
}
}
private: // data
std::map<std::string, const I18nStringTable*> lang2TableMap;
const I18nStringTable* enStrings;
};
} // unnamed namespace
std::string getTranslatedString(const std::string& lang, const std::string& key)
{
static const I18nStringDB stringDb;
return stringDb.get(lang, key);
}
namespace i18n
{
std::string expandParameterizedString(const std::string& lang,
const std::string& key,
const Parameters& params)
{
const std::string tmpl = getTranslatedString(lang, key);
return render_template(tmpl, params);
}
} // namespace i18n
std::string ParameterizedMessage::getText(const std::string& lang) const
{
return i18n::expandParameterizedString(lang, msgId, params);
}
} // namespace kiwix

94
src/server/i18n.h Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_SERVER_I18N
#define KIWIX_SERVER_I18N
#include <string>
#include <mustache.hpp>
namespace kiwix
{
struct I18nString {
const char* const key;
const char* const value;
};
struct I18nStringTable {
const char* const lang;
const size_t entryCount;
const I18nString* const entries;
const char* get(const std::string& key) const;
};
std::string getTranslatedString(const std::string& lang, const std::string& key);
namespace i18n
{
typedef kainjow::mustache::object Parameters;
std::string expandParameterizedString(const std::string& lang,
const std::string& key,
const Parameters& params);
class GetTranslatedString
{
public:
explicit GetTranslatedString(const std::string& lang) : m_lang(lang) {}
std::string operator()(const std::string& key) const
{
return getTranslatedString(m_lang, key);
}
std::string operator()(const std::string& key, const Parameters& params) const
{
return expandParameterizedString(m_lang, key, params);
}
private:
const std::string m_lang;
};
} // namespace i18n
struct ParameterizedMessage
{
public: // types
typedef kainjow::mustache::object Parameters;
public: // functions
ParameterizedMessage(const std::string& msgId, const Parameters& params)
: msgId(msgId)
, params(params)
{}
std::string getText(const std::string& lang) const;
private: // data
const std::string msgId;
const Parameters params;
};
} // namespace kiwix
#endif // KIWIX_SERVER_I18N

View File

@ -55,6 +55,7 @@ extern "C" {
#include "searcher.h"
#include "search_renderer.h"
#include "opds_dumper.h"
#include "i18n.h"
#include <zim/uuid.h>
#include <zim/error.h>
@ -442,14 +443,39 @@ SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Arch
namespace
{
std::string noSuchBookErrorMsg(const std::string& bookName)
std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
{
return "No such book: " + bookName;
return i18n::expandParameterizedString(lang, "suggest-full-text-search",
{
{"SEARCH_TERMS", queryString}
}
);
}
std::string noSearchResultsMsg()
ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
{
return "The fulltext search engine is not available for this content.";
return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
}
ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
{
return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
}
ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
{
return ParameterizedMessage("raw-entry-not-found",
{
{"DATATYPE", dt},
{"ENTRY", entry},
}
);
}
ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
{
const ParameterizedMessage::Parameters noParams;
return ParameterizedMessage(msgId, noParams);
}
} // unnamed namespace
@ -514,7 +540,8 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
/* Propose the fulltext search if possible */
if (archive->hasFulltextIndex()) {
MustacheData result;
result.set("label", "containing '" + queryString + "'...");
const auto lang = request.get_user_language();
result.set("label", makeFulltextSearchSuggestion(lang, queryString));
result.set("value", queryString + " ");
result.set("kind", "pattern");
result.set("first", first);
@ -597,10 +624,10 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
// Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
// (in case of zim file not containing a index)
return HTTPErrorHtmlResponse(*this, request, MHD_HTTP_NOT_FOUND,
"Fulltext search unavailable",
"Not Found",
"fulltext-search-unavailable",
"404-page-heading",
m_root + "/skin/search_results.css")
+ noSearchResultsMsg()
+ nonParameterizedMessage("no-search-results")
+ TaskbarInfo(searchInfo.bookName, archive.get());
}
@ -669,9 +696,8 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
auto entry = archive->getRandomEntry();
return build_redirect(bookName, getFinalItem(*archive, entry));
} catch(zim::EntryNotFound& e) {
const std::string error_details = "Oops! Failed to pick a random article :(";
return HTTP404HtmlResponse(*this, request)
+ error_details
+ nonParameterizedMessage("random-article-failure")
+ TaskbarInfo(bookName, archive.get());
}
}
@ -812,13 +838,13 @@ std::string get_book_name(const RequestContext& request)
}
}
std::string searchSuggestionHTML(const std::string& searchURL, const std::string& pattern)
ParameterizedMessage suggestSearchMsg(const std::string& searchURL, const std::string& pattern)
{
kainjow::mustache::mustache tmpl("Make a full text search for <a href=\"{{{searchURL}}}\">{{pattern}}</a>");
MustacheData data;
data.set("pattern", pattern);
data.set("searchURL", searchURL);
return (tmpl.render(data));
return ParameterizedMessage("suggest-search",
{
{ "PATTERN", pattern },
{ "SEARCH_URL", searchURL }
});
}
} // unnamed namespace
@ -852,7 +878,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+ suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
+ TaskbarInfo(bookName);
}
@ -886,7 +912,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+ suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
+ TaskbarInfo(bookName, archive.get());
}
}
@ -909,10 +935,9 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
}
if (kind != "meta" && kind!= "content") {
const std::string error_details = kind + " is not a valid request for raw content.";
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ error_details;
+ invalidRawAccessMsg(kind);
}
std::shared_ptr<zim::Archive> archive;
@ -948,10 +973,9 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
if (m_verbose.load()) {
printf("Failed to find %s\n", itemPath.c_str());
}
const std::string error_details = "Cannot find " + kind + " entry " + itemPath;
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ error_details;
+ rawEntryNotFoundMsg(kind, itemPath);
}
}

View File

@ -193,4 +193,17 @@ std::string RequestContext::get_query() const {
return q;
}
std::string RequestContext::get_user_language() const
{
try {
return get_argument("userlang");
} catch(const std::out_of_range&) {}
try {
return get_header("Accept-Language");
} catch(const std::out_of_range&) {}
return "en";
}
}

View File

@ -94,6 +94,8 @@ class RequestContext {
bool can_compress() const { return acceptEncodingDeflate; }
std::string get_user_language() const;
private: // data
std::string full_url;
std::string url;

View File

@ -87,6 +87,11 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
const UrlNotFoundMsg urlNotFoundMsg;
const InvalidUrlMsg invalidUrlMsg;
std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
{
return getTranslatedString(m_request.get_user_language(), msgId);
}
std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
{
auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
@ -100,8 +105,8 @@ std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObjec
HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
const RequestContext& request,
int httpStatusCode,
const std::string& pageTitleMsg,
const std::string& headingMsg,
const std::string& pageTitleMsgId,
const std::string& headingMsgId,
const std::string& cssUrl)
: ContentResponseBlueprint(&server,
&request,
@ -112,8 +117,8 @@ HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
kainjow::mustache::list emptyList;
this->m_data = kainjow::mustache::object{
{"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
{"PAGE_TITLE", pageTitleMsg},
{"PAGE_HEADING", headingMsg},
{"PAGE_TITLE", getMessage(pageTitleMsgId)},
{"PAGE_HEADING", getMessage(headingMsgId)},
{"details", emptyList}
};
}
@ -123,16 +128,15 @@ HTTP404HtmlResponse::HTTP404HtmlResponse(const InternalServer& server,
: HTTPErrorHtmlResponse(server,
request,
MHD_HTTP_NOT_FOUND,
"Content not found",
"Not Found")
"404-page-title",
"404-page-heading")
{
}
HTTPErrorHtmlResponse& HTTP404HtmlResponse::operator+(UrlNotFoundMsg /*unused*/)
{
const std::string requestUrl = m_request.get_full_url();
kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{url}}" was not found on this server.)");
return *this + msgTmpl.render({"url", requestUrl});
return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
}
HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
@ -141,13 +145,19 @@ HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
return *this;
}
HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const ParameterizedMessage& details)
{
return *this + details.getText(m_request.get_user_language());
}
HTTP400HtmlResponse::HTTP400HtmlResponse(const InternalServer& server,
const RequestContext& request)
: HTTPErrorHtmlResponse(server,
request,
MHD_HTTP_BAD_REQUEST,
"Invalid request",
"Invalid request")
"400-page-title",
"400-page-heading")
{
}
@ -167,8 +177,8 @@ HTTP500HtmlResponse::HTTP500HtmlResponse(const InternalServer& server,
: HTTPErrorHtmlResponse(server,
request,
MHD_HTTP_INTERNAL_SERVER_ERROR,
"Internal Server Error",
"Internal Server Error")
"500-page-title",
"500-page-heading")
{
// operator+() is a state-modifying operator (akin to operator+=)
*this + "An internal server error occured. We are sorry about that :/";
@ -270,14 +280,20 @@ void print_response_info(int retCode, MHD_Response* response)
}
void ContentResponse::introduce_taskbar()
void ContentResponse::introduce_taskbar(const std::string& lang)
{
kainjow::mustache::data data;
data.set("root", m_root);
data.set("content", m_bookName);
data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
data.set("title", m_bookTitle);
data.set("withlibrarybutton", m_withLibraryButton);
i18n::GetTranslatedString t(lang);
kainjow::mustache::object data{
{"root", m_root},
{"content", m_bookName},
{"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
{"title", m_bookTitle},
{"withlibrarybutton", m_withLibraryButton},
{"LIBRARY_BUTTON_TEXT", t("library-button-text")},
{"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
{"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
{"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
};
auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
m_content = prependToFirstOccurence(
m_content,
@ -342,7 +358,7 @@ ContentResponse::create_mhd_response(const RequestContext& request)
inject_root_link();
if (m_withTaskbar) {
introduce_taskbar();
introduce_taskbar(request.get_user_language());
}
if (m_blockExternalLinks) {
inject_externallinks_blocker();

View File

@ -28,6 +28,7 @@
#include "byte_range.h"
#include "entry.h"
#include "etag.h"
#include "i18n.h"
extern "C" {
#include "microhttpd_wrapper.h"
@ -105,7 +106,7 @@ class ContentResponse : public Response {
private:
MHD_Response* create_mhd_response(const RequestContext& request);
void introduce_taskbar();
void introduce_taskbar(const std::string& lang);
void inject_externallinks_blocker();
void inject_root_link();
bool can_compress(const RequestContext& request) const;
@ -166,6 +167,7 @@ public: // functions
ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
protected: // functions
std::string getMessage(const std::string& msgId) const;
virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
public: //data
@ -183,12 +185,13 @@ struct HTTPErrorHtmlResponse : ContentResponseBlueprint
HTTPErrorHtmlResponse(const InternalServer& server,
const RequestContext& request,
int httpStatusCode,
const std::string& pageTitleMsg,
const std::string& headingMsg,
const std::string& pageTitleMsgId,
const std::string& headingMsgId,
const std::string& cssUrl = "");
using ContentResponseBlueprint::operator+;
HTTPErrorHtmlResponse& operator+(const std::string& msg);
HTTPErrorHtmlResponse& operator+(const ParameterizedMessage& errorDetails);
};
class UrlNotFoundMsg {};

26
static/i18n/en.json Normal file
View File

@ -0,0 +1,26 @@
{
"@metadata": {
"authors": [
]
},
"name":"English",
"suggest-full-text-search": "containing '{{{SEARCH_TERMS}}}'..."
, "no-such-book": "No such book: {{BOOK_NAME}}"
, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
, "random-article-failure" : "Oops! Failed to pick a random article :("
, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
, "400-page-title" : "Invalid request"
, "400-page-heading" : "Invalid request"
, "404-page-title" : "Content not found"
, "404-page-heading" : "Not Found"
, "500-page-title" : "Internal Server Error"
, "500-page-heading" : "Internal Server Error"
, "fulltext-search-unavailable" : "Fulltext search unavailable"
, "no-search-results": "The fulltext search engine is not available for this content."
, "library-button-text": "Go to welcome page"
, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
, "random-page-button-text": "Go to a randomly selected page"
, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
}

17
static/i18n/hy.json Normal file
View File

@ -0,0 +1,17 @@
{
"@metadata": {
"authors": [
]
},
"name":"Հայերեն",
"suggest-full-text-search": "որոնել '{{{SEARCH_TERMS}}}'..."
, "no-such-book": "Գիրքը բացակայում է՝ {{BOOK_NAME}}"
, "url-not-found" : "Սխալ հասցե՝ {{url}}"
, "suggest-search" : "Որոնել <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
, "404-page-title" : "Սխալ հասցե"
, "404-page-heading" : "Սխալ հասցե"
, "library-button-text": "Գրադարանի էջ"
, "home-button-text": "Դեպի '{{BOOK_TITLE}}'֊ի գլխավոր էջը"
, "random-page-button-text": "Բացել պատահական էջ"
, "searchbox-tooltip": "Որոնել '{{BOOK_TITLE}}'֊ում"
}

27
static/i18n/qqq.json Normal file
View File

@ -0,0 +1,27 @@
{
"@metadata": {
"authors": [
"Veloman Yunkan"
]
},
"name": "Current language to which the string is being translated to.",
"suggest-full-text-search": "Text appearing in the suggestion list that, when selected, runs a full text search instead of the title search"
, "no-such-book": "Error text when the requested book is not found in the library"
, "url-not-found" : "Error text about wrong URL for an HTTP 404 error"
, "suggest-search" : "Suggest a search when the URL points to a non existing article"
, "random-article-failure" : "Failure of the random article selection procedure"
, "invalid-raw-data-type" : "Invalid DATATYPE was used with the /raw endpoint (/raw/<book>/DATATYPE/...); allowed values are 'meta' and 'content'"
, "raw-entry-not-found" : "Entry requested via the /raw endpoint was not found"
, "400-page-title" : "Title of the 400 error page"
, "400-page-heading" : "Heading of the 400 error page"
, "404-page-title" : "Title of the 404 error page"
, "404-page-heading" : "Heading of the 404 error page"
, "500-page-title" : "Title of the 500 error page"
, "500-page-heading" : "Heading of the 500 error page"
, "fulltext-search-unavailable" : "Title of the error page returned when search is attempted in a book without fulltext search database"
, "no-search-results": "Text of the error page returned when search is attempted in a book without fulltext search database"
, "library-button-text": "Tooltip of the button leading to the welcome page"
, "home-button-text": "Tooltip of the button leading to the main page of a book"
, "random-page-button-text": "Tooltip of the button opening a randomly selected page"
, "searchbox-tooltip": "Tooltip displayed for the search box"
}

View File

@ -0,0 +1,2 @@
i18n/en.json
i18n/hy.json

View File

@ -14,3 +14,18 @@ lib_resources = custom_target('resources',
'@INPUT@'],
depend_files: resource_files
)
i18n_resource_files = run_command(find_program('python3'),
'-c',
'import sys; f=open(sys.argv[1]); print(f.read())',
files('i18n_resources_list.txt')
).stdout().strip().split('\n')
i18n_resources = custom_target('i18n_resources',
input: 'i18n_resources_list.txt',
output: ['libkiwix-i18n-resources.cpp'],
command:[i18n_compiler,
'--cxxfile', '@OUTPUT0@',
'@INPUT@'],
depend_files: i18n_resource_files
)

View File

@ -12,9 +12,10 @@ jq(document).ready(() => {
? (new URLSearchParams(window.location.search)).get('content')
: window.location.pathname.split(`${root}/`)[1].split('/')[0];
const userlang = (new URLSearchParams(window.location.search)).get('userlang') || "en";
$( "#kiwixsearchbox" ).autocomplete({
source: `${root}/suggest?content=${bookName}`,
source: `${root}/suggest?content=${bookName}&userlang=${userlang}`,
dataType: "json",
cache: false,

View File

@ -5,18 +5,18 @@
<form class="kiwixsearch" method="GET" action="{{root}}/search" id="kiwixsearchform">
{{#hascontent}}<input type="hidden" name="content" value="{{content}}" />{{/hascontent}}
<label for="kiwixsearchbox">&#x1f50d;</label>
<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search '{{title}}'" aria-label="Search '{{title}}'">
<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="{{{SEARCHBOX_TOOLTIP}}}" aria-label="{{{SEARCHBOX_TOOLTIP}}}">
</form>
</div>
<input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
<div class="kiwix_button_cont">
{{#withlibrarybutton}}
<a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="{{root}}/"><button>&#x1f3e0;</button></a>
<a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>&#x1f3e0;</button></a>
{{/withlibrarybutton}}
{{#hascontent}}
<a id="kiwix_serve_taskbar_home_button" title="Go to the main page of '{{title}}'" aria-label="Go to the main page of '{{title}}'" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
<a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
<a id="kiwix_serve_taskbar_home_button" title="{{{HOME_BUTTON_TEXT}}}" aria-label="{{{HOME_BUTTON_TEXT}}}" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
<a id="kiwix_serve_taskbar_random_button" title="{{{RANDOM_PAGE_BUTTON_TEXT}}}" aria-label="{{{RANDOM_PAGE_BUTTON_TEXT}}}"
href="{{root}}/random?content={{#urlencoded}}{{{content}}}{{/urlencoded}}"><button>&#x1F3B2;</button></a>
{{/hascontent}}
</div>

View File

@ -57,7 +57,6 @@ std::string removeEOLWhitespaceMarkers(const std::string& s)
return std::regex_replace(s, pattern, "");
}
class ZimFileServer
{
public: // types
@ -411,11 +410,13 @@ public:
std::string expectedResponse() const;
private:
bool isTranslatedVersion() const;
virtual std::string pageTitle() const;
std::string pageCssLink() const;
std::string hiddenBookNameInput() const;
std::string searchPatternInput() const;
std::string taskbarLinks() const;
std::string goToWelcomePageText() const;
};
std::string TestContentIn404HtmlResponse::expectedResponse() const
@ -454,7 +455,11 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
<input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
<div class="kiwix_button_cont">
<a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="/ROOT/"><button>&#x1f3e0;</button></a>
<a id="kiwix_serve_taskbar_library_button" title=")FRAG",
R"FRAG(" aria-label=")FRAG",
R"FRAG(" href="/ROOT/"><button>&#x1f3e0;</button></a>
)FRAG",
R"FRAG(
@ -478,10 +483,14 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
+ frag[3]
+ searchPatternInput()
+ frag[4]
+ taskbarLinks()
+ goToWelcomePageText()
+ frag[5]
+ removeEOLWhitespaceMarkers(expectedBody)
+ frag[6];
+ goToWelcomePageText()
+ frag[6]
+ taskbarLinks()
+ frag[7]
+ expectedBody
+ frag[8];
}
std::string TestContentIn404HtmlResponse::pageTitle() const
@ -510,11 +519,15 @@ std::string TestContentIn404HtmlResponse::hiddenBookNameInput() const
std::string TestContentIn404HtmlResponse::searchPatternInput() const
{
return R"( <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search ')"
+ bookTitle
+ R"('" aria-label="Search ')"
+ bookTitle
+ R"('">
const std::string searchboxTooltip = isTranslatedVersion()
? "Որոնել '" + bookTitle + "'֊ում"
: "Search '" + bookTitle + "'";
return R"( <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title=")"
+ searchboxTooltip
+ R"(" aria-label=")"
+ searchboxTooltip
+ R"(">
)";
}
@ -523,21 +536,46 @@ std::string TestContentIn404HtmlResponse::taskbarLinks() const
if ( bookName.empty() )
return "";
return R"(<a id="kiwix_serve_taskbar_home_button" title="Go to the main page of ')"
+ bookTitle
+ R"('" aria-label="Go to the main page of ')"
+ bookTitle
+ R"('" href="/ROOT/)"
const auto goToMainPageOfBook = isTranslatedVersion()
? "Դեպի '" + bookTitle + "'֊ի գլխավոր էջը"
: "Go to the main page of '" + bookTitle + "'";
const std::string goToRandomPage = isTranslatedVersion()
? "Բացել պատահական էջ"
: "Go to a randomly selected page";
return R"(<a id="kiwix_serve_taskbar_home_button" title=")"
+ goToMainPageOfBook
+ R"(" aria-label=")"
+ goToMainPageOfBook
+ R"(" href="/ROOT/)"
+ bookName
+ R"(/"><button>)"
+ bookTitle
+ R"(</button></a>
<a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
<a id="kiwix_serve_taskbar_random_button" title=")"
+ goToRandomPage
+ R"(" aria-label=")"
+ goToRandomPage
+ R"("
href="/ROOT/random?content=)"
+ bookName
+ R"("><button>&#x1F3B2;</button></a>)";
}
bool TestContentIn404HtmlResponse::isTranslatedVersion() const
{
return url.find("userlang=hy") != std::string::npos;
}
std::string TestContentIn404HtmlResponse::goToWelcomePageText() const
{
return isTranslatedVersion()
? "Գրադարանի էջ"
: "Go to welcome page";
}
class TestContentIn400HtmlResponse : public TestContentIn404HtmlResponse
{
public:
@ -556,7 +594,6 @@ std::string TestContentIn400HtmlResponse::pageTitle() const {
: expectedPageTitle;
}
} // namespace TestingOfHtmlResponses
TEST_F(ServerTest, 404WithBodyTesting)
@ -571,6 +608,15 @@ TEST_F(ServerTest, 404WithBodyTesting)
</p>
)" },
{ /* url */ "/ROOT/random?content=non-existent-book&userlang=hy",
expected_page_title=="Սխալ հասցե" &&
expected_body==R"(
<h1>Սխալ հասցե</h1>
<p>
Գիրքը բացակայում է՝ non-existent-book
</p>
)" },
{ /* url */ "/ROOT/suggest?content=no-such-book&term=whatever",
expected_body==R"(
<h1>Not Found</h1>
@ -587,6 +633,15 @@ TEST_F(ServerTest, 404WithBodyTesting)
</p>
)" },
{ /* url */ "/ROOT/catalog/?userlang=hy",
expected_page_title=="Սխալ հասցե" &&
expected_body==R"(
<h1>Սխալ հասցե</h1>
<p>
Սխալ հասցե՝ /ROOT/catalog/
</p>
)" },
{ /* url */ "/ROOT/catalog/invalid_endpoint",
expected_body==R"(
<h1>Not Found</h1>
@ -595,6 +650,15 @@ TEST_F(ServerTest, 404WithBodyTesting)
</p>
)" },
{ /* url */ "/ROOT/catalog/invalid_endpoint?userlang=hy",
expected_page_title=="Սխալ հասցե" &&
expected_body==R"(
<h1>Սխալ հասցե</h1>
<p>
Սխալ հասցե՝ /ROOT/catalog/invalid_endpoint
</p>
)" },
{ /* url */ "/ROOT/invalid-book/whatever",
expected_body==R"(
<h1>Not Found</h1>
@ -643,6 +707,20 @@ TEST_F(ServerTest, 404WithBodyTesting)
</p>
)" },
{ /* url */ "/ROOT/zimfile/invalid-article?userlang=hy",
expected_page_title=="Սխալ հասցե" &&
book_name=="zimfile" &&
book_title=="Ray Charles" &&
expected_body==R"(
<h1>Սխալ հասցե</h1>
<p>
Սխալ հասցե՝ /ROOT/zimfile/invalid-article
</p>
<p>
Որոնել <a href="/ROOT/search?content=zimfile&pattern=invalid-article">invalid-article</a>
</p>
)" },
{ /* url */ "/ROOT/raw/no-such-book/meta/Title",
expected_body==R"(
<h1>Not Found</h1>
@ -800,6 +878,79 @@ TEST_F(ServerTest, 500)
EXPECT_EQ(r->body, expectedBody);
}
TEST_F(ServerTest, UserLanguageControl)
{
struct TestData
{
const std::string url;
const std::string acceptLanguageHeader;
const std::string expectedH1;
operator TestContext() const
{
return TestContext{
{"url", url},
{"acceptLanguageHeader", acceptLanguageHeader},
};
}
};
const TestData testData[] = {
{
/*url*/ "/ROOT/zimfile/invalid-article",
/*Accept-Language:*/ "",
/* expected <h1> */ "Not Found"
},
{
/*url*/ "/ROOT/zimfile/invalid-article?userlang=en",
/*Accept-Language:*/ "",
/* expected <h1> */ "Not Found"
},
{
/*url*/ "/ROOT/zimfile/invalid-article?userlang=hy",
/*Accept-Language:*/ "",
/* expected <h1> */ "Սխալ հասցե"
},
{
/*url*/ "/ROOT/zimfile/invalid-article",
/*Accept-Language:*/ "*",
/* expected <h1> */ "Not Found"
},
{
/*url*/ "/ROOT/zimfile/invalid-article",
/*Accept-Language:*/ "hy",
/* expected <h1> */ "Սխալ հասցե"
},
{
// userlang query parameter takes precedence over Accept-Language
/*url*/ "/ROOT/zimfile/invalid-article?userlang=en",
/*Accept-Language:*/ "hy",
/* expected <h1> */ "Not Found"
},
{
// The value of the Accept-Language header is not currently parsed.
// In case of a comma separated list of languages (optionally weighted
// with quality values) the default (en) language is used instead.
/*url*/ "/ROOT/zimfile/invalid-article",
/*Accept-Language:*/ "hy;q=0.9, en;q=0.2",
/* expected <h1> */ "Not Found"
},
};
const std::regex h1Regex("<h1>(.+)</h1>");
for ( const auto& t : testData ) {
std::smatch h1Match;
Headers headers;
if ( !t.acceptLanguageHeader.empty() ) {
headers.insert({"Accept-Language", t.acceptLanguageHeader});
}
const auto r = zfs1_->GET(t.url.c_str(), headers);
std::regex_search(r->body, h1Match, h1Regex);
const std::string h1(h1Match[1]);
EXPECT_EQ(h1, t.expectedH1) << t;
}
}
TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
{
auto g = zfs1_->GET("/ROOT/random?content=zimfile");
@ -1182,6 +1333,17 @@ R"EXPECTEDRESPONSE([
//EOLWHITESPACEMARKER
}
]
)EXPECTEDRESPONSE"
},
{ /* url: */ "/ROOT/suggest?content=zimfile&term=abracadabra&userlang=hy",
R"EXPECTEDRESPONSE([
{
"value" : "abracadabra ",
"label" : "որոնել &apos;abracadabra&apos;...",
"kind" : "pattern"
//EOLWHITESPACEMARKER
}
]
)EXPECTEDRESPONSE"
},
};