Merge pull request #1044 from kiwix/default_ui_language_is_resolved_in_the_frontend

Default UI language is resolved in the frontend
This commit is contained in:
Matthieu Gautier 2024-01-31 17:54:56 +01:00 committed by GitHub
commit 795fcb9de4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 248 additions and 161 deletions

View File

@ -767,8 +767,7 @@ std::unique_ptr<Response> InternalServer::handle_viewer_settings(const RequestCo
const kainjow::mustache::object data{
{"enable_toolbar", m_withTaskbar ? "true" : "false" },
{"enable_link_blocking", m_blockExternalLinks ? "true" : "false" },
{"enable_library_button", m_withLibraryButton ? "true" : "false" },
{"default_user_language", request.get_user_language() }
{"enable_library_button", m_withLibraryButton ? "true" : "false" }
};
return ContentResponse::build(RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8");
}

View File

@ -30,7 +30,10 @@ def get_translation_info(filepath):
with open(filepath, 'r', encoding="utf-8") as f:
content = json.load(f)
lang_name = content.get("name")
return lang_code, lang_name
translation_count = len(content)
return dict(iso_code=lang_code,
self_name=lang_name,
translation_count=translation_count)
language_list = []
json_files = translation_dir.glob("*.json")
@ -40,14 +43,14 @@ with open(resource_file, 'w', encoding="utf-8") as f:
continue
print("Processing", i18n_file.name)
if i18n_file.name != "test.json":
lang_code, lang_name = get_translation_info(i18n_file)
translation_info = get_translation_info(i18n_file)
lang_name = translation_info["self_name"]
if lang_name:
language_list.append((lang_code, lang_name))
language_list.append(translation_info)
else:
print(f"Warning: missing 'name' in {i18n_file.name}")
f.write(str(i18n_file.relative_to(script_path.parent)) + '\n')
language_list = [{name: code} for code, name in sorted(language_list)]
language_list_jsobj_str = json.dumps(language_list,
indent=2,
ensure_ascii=False)

View File

@ -104,11 +104,31 @@ const DEFAULT_UI_LANGUAGE = 'en';
Translations.load(DEFAULT_UI_LANGUAGE, /*asDefault=*/true);
// Below function selects the most suitable UI language from the list
// of preferred languages in browser preferences and available translations.
// Since, unlike Accept-Language header, navigator.languages doesn't contain
// qvalues, they are computed using the same algorithm as in Firefox 121
function getDefaultUserLanguage() {
const mostSuitableLang = { code: DEFAULT_UI_LANGUAGE, score: 0 }
const n = navigator.languages.length;
for (const lang of uiLanguages ) {
const rank = navigator.languages.indexOf(lang.iso_code);
if ( rank >= 0 ) {
const qvalue = Math.round(10*(1 - rank/n))/10;
const score = qvalue * lang.translation_count;
if ( score > mostSuitableLang.score ) {
mostSuitableLang.code = lang.iso_code;
mostSuitableLang.score = score;
}
}
}
return mostSuitableLang.code;
}
function getUserLanguage() {
return new URLSearchParams(window.location.search).get('userlang')
|| window.localStorage.getItem('userlang')
|| viewerSettings.defaultUserLanguage
|| DEFAULT_UI_LANGUAGE;
|| getDefaultUserLanguage();
}
function setUserLanguage(lang, callback) {
@ -164,10 +184,8 @@ function initUILanguageSelector(activeLanguage, languageChangeCallback) {
}
const languageSelector = document.getElementById("ui_language");
for (const lang of uiLanguages ) {
const lang_name = Object.getOwnPropertyNames(lang)[0];
const lang_code = lang[lang_name];
const is_selected = lang_code == activeLanguage;
languageSelector.appendChild(new Option(lang_name, lang_code, is_selected, is_selected));
const is_selected = lang.iso_code == activeLanguage;
languageSelector.appendChild(new Option(lang.self_name, lang.iso_code, is_selected, is_selected));
}
languageSelector.onchange = languageChangeCallback;
}

View File

@ -1,104 +1,172 @@
const uiLanguages = [
{
"الإنجليزية": "ar"
"iso_code": "ar",
"self_name": "الإنجليزية",
"translation_count": 25
},
{
"বাংলা": "bn"
"iso_code": "bn",
"self_name": "বাংলা",
"translation_count": 12
},
{
"Čeština": "cs"
"iso_code": "cs",
"self_name": "Čeština",
"translation_count": 25
},
{
"Deutsch": "de"
"iso_code": "de",
"self_name": "Deutsch",
"translation_count": 49
},
{
"English": "en"
"iso_code": "en",
"self_name": "English",
"translation_count": 53
},
{
"español": "es"
"iso_code": "es",
"self_name": "español",
"translation_count": 48
},
{
"suomi": "fi"
"iso_code": "fi",
"self_name": "suomi",
"translation_count": 22
},
{
"Français": "fr"
"iso_code": "fr",
"self_name": "Français",
"translation_count": 52
},
{
"עברית": "he"
"iso_code": "he",
"self_name": "עברית",
"translation_count": 52
},
{
"हिन्दी": "hi"
"iso_code": "hi",
"self_name": "हिन्दी",
"translation_count": 49
},
{
"Հայերեն": "hy"
"iso_code": "hy",
"self_name": "Հայերեն",
"translation_count": 15
},
{
"interlingua": "ia"
"iso_code": "ia",
"self_name": "interlingua",
"translation_count": 49
},
{
"italiano": "it"
"iso_code": "it",
"self_name": "italiano",
"translation_count": 29
},
{
"日本語": "ja"
"iso_code": "ja",
"self_name": "日本語",
"translation_count": 26
},
{
"한국어": "ko"
"iso_code": "ko",
"self_name": "한국어",
"translation_count": 13
},
{
"kurdî": "ku-latn"
"iso_code": "ku-latn",
"self_name": "kurdî",
"translation_count": 26
},
{
"Lëtzebuergesch": "lb"
"iso_code": "lb",
"self_name": "Lëtzebuergesch",
"translation_count": 22
},
{
"македонски": "mk"
"iso_code": "mk",
"self_name": "македонски",
"translation_count": 52
},
{
"Bahasa Melayu": "ms"
"iso_code": "ms",
"self_name": "Bahasa Melayu",
"translation_count": 14
},
{
"Nederlands": "nl"
"iso_code": "nl",
"self_name": "Nederlands",
"translation_count": 49
},
{
"ߒߞߏ": "nqo"
"iso_code": "nqo",
"self_name": "ߒߞߏ",
"translation_count": 43
},
{
"ଓଡ଼ିଆ": "or"
"iso_code": "or",
"self_name": "ଓଡ଼ିଆ",
"translation_count": 49
},
{
"Polski": "pl"
"iso_code": "pl",
"self_name": "Polski",
"translation_count": 24
},
{
"русский": "ru"
"iso_code": "ru",
"self_name": "русский",
"translation_count": 45
},
{
"Sardu": "sc"
"iso_code": "sc",
"self_name": "Sardu",
"translation_count": 49
},
{
"slovenčina": "sk"
"iso_code": "sk",
"self_name": "slovenčina",
"translation_count": 25
},
{
"سرائیکی": "skr-arab"
"iso_code": "skr-arab",
"self_name": "سرائیکی",
"translation_count": 20
},
{
"slovenščina": "sl"
"iso_code": "sl",
"self_name": "slovenščina",
"translation_count": 52
},
{
"Shqip": "sq"
"iso_code": "sq",
"self_name": "Shqip",
"translation_count": 49
},
{
"Svenska": "sv"
"iso_code": "sv",
"self_name": "Svenska",
"translation_count": 52
},
{
"ఇంగ్లీషు": "te"
"iso_code": "te",
"self_name": "ఇంగ్లీషు",
"translation_count": 49
},
{
"Türkçe": "tr"
"iso_code": "tr",
"self_name": "Türkçe",
"translation_count": 25
},
{
"英语": "zh-hans"
"iso_code": "zh-hans",
"self_name": "英语",
"translation_count": 16
},
{
"繁體中文": "zh-hant"
"iso_code": "zh-hant",
"self_name": "繁體中文",
"translation_count": 52
}
]

View File

@ -1,6 +1,5 @@
const viewerSettings = {
toolbarEnabled: {{enable_toolbar}},
linkBlockingEnabled: {{enable_link_blocking}},
libraryButtonEnabled: {{enable_library_button}},
defaultUserLanguage: "{{default_user_language}}"
libraryButtonEnabled: {{enable_library_button}}
}

View File

@ -59,7 +59,7 @@ const ResourceCollection resources200Compressible{
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/autoComplete/css/autoComplete.css" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/autoComplete/css/autoComplete.css?cacheid=ef30cd42" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/i18n.js" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/i18n.js?cacheid=4ab55b42" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/i18n.js?cacheid=071abc9a" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/index.css" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/index.css?cacheid=1e78e7cf" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/index.js" },
@ -83,6 +83,8 @@ const ResourceCollection resources200Compressible{
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/i18n/test.json" },
// TODO: implement cache management of i18n resources
//{ STATIC_CONTENT, "/ROOT%23%3F/skin/i18n/test.json?cacheid=unknown" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/languages.js" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/languages.js?cacheid=c41aae47" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/catalog/search" },
@ -148,8 +150,6 @@ const ResourceCollection resources200Uncompressible{
{ STATIC_CONTENT, "/ROOT%23%3F/skin/search-icon.svg?cacheid=b10ae7ed" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/search_results.css" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/search_results.css?cacheid=76d39c84" },
{ DYNAMIC_CONTENT, "/ROOT%23%3F/skin/languages.js" },
{ STATIC_CONTENT, "/ROOT%23%3F/skin/languages.js?cacheid=96f2cf73" },
{ ZIM_CONTENT, "/ROOT%23%3F/raw/zimfile/meta/Title" },
{ ZIM_CONTENT, "/ROOT%23%3F/raw/zimfile/meta/Description" },
@ -285,8 +285,8 @@ R"EXPECTEDRESULT( href="/ROOT%23%3F/skin/kiwix.css?cacheid=2158fad9"
<link rel="mask-icon" href="/ROOT%23%3F/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" color="#5bbad5">
<link rel="shortcut icon" href="/ROOT%23%3F/skin/favicon/favicon.ico?cacheid=92663314">
<meta name="msapplication-config" content="/ROOT%23%3F/skin/favicon/browserconfig.xml?cacheid=f29a7c4a">
<script type="module" src="/ROOT%23%3F/skin/i18n.js?cacheid=4ab55b42" defer></script>
<script type="text/javascript" src="/ROOT%23%3F/skin/languages.js?cacheid=96f2cf73" defer></script>
<script type="module" src="/ROOT%23%3F/skin/i18n.js?cacheid=071abc9a" defer></script>
<script type="text/javascript" src="/ROOT%23%3F/skin/languages.js?cacheid=c41aae47" defer></script>
<script src="/ROOT%23%3F/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
<script src="/ROOT%23%3F/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
<script type="text/javascript" src="/ROOT%23%3F/skin/index.js?cacheid=ce19da2a" defer></script>
@ -318,8 +318,8 @@ R"EXPECTEDRESULT( <img src="${root}/skin/download
R"EXPECTEDRESULT( <link type="text/css" href="./skin/kiwix.css?cacheid=2158fad9" rel="Stylesheet" />
<link type="text/css" href="./skin/taskbar.css?cacheid=e014a885" rel="Stylesheet" />
<link type="text/css" href="./skin/autoComplete/css/autoComplete.css?cacheid=ef30cd42" rel="Stylesheet" />
<script type="module" src="./skin/i18n.js?cacheid=4ab55b42" defer></script>
<script type="text/javascript" src="./skin/languages.js?cacheid=96f2cf73" defer></script>
<script type="module" src="./skin/i18n.js?cacheid=071abc9a" defer></script>
<script type="text/javascript" src="./skin/languages.js?cacheid=c41aae47" defer></script>
<script type="text/javascript" src="./skin/viewer.js?cacheid=e9c025f2" defer></script>
<script type="text/javascript" src="./skin/autoComplete/autoComplete.min.js?cacheid=1191aaaf"></script>
const blankPageUrl = root + "/skin/blank.html?cacheid=6b1fa032";
@ -1126,106 +1126,174 @@ TEST_F(ServerTest, UserLanguageList)
EXPECT_EQ(r->body,
R"EXPECTEDRESPONSE(const uiLanguages = [
{
"الإنجليزية": "ar"
"iso_code": "ar",
"self_name": "الإنجليزية",
"translation_count": 25
},
{
"বাংলা": "bn"
"iso_code": "bn",
"self_name": "বাংলা",
"translation_count": 12
},
{
"Čeština": "cs"
"iso_code": "cs",
"self_name": "Čeština",
"translation_count": 25
},
{
"Deutsch": "de"
"iso_code": "de",
"self_name": "Deutsch",
"translation_count": 49
},
{
"English": "en"
"iso_code": "en",
"self_name": "English",
"translation_count": 53
},
{
"español": "es"
"iso_code": "es",
"self_name": "español",
"translation_count": 48
},
{
"suomi": "fi"
"iso_code": "fi",
"self_name": "suomi",
"translation_count": 22
},
{
"Français": "fr"
"iso_code": "fr",
"self_name": "Français",
"translation_count": 52
},
{
"עברית": "he"
"iso_code": "he",
"self_name": "עברית",
"translation_count": 52
},
{
"हिन्दी": "hi"
"iso_code": "hi",
"self_name": "हिन्दी",
"translation_count": 49
},
{
"Հայերեն": "hy"
"iso_code": "hy",
"self_name": "Հայերեն",
"translation_count": 15
},
{
"interlingua": "ia"
"iso_code": "ia",
"self_name": "interlingua",
"translation_count": 49
},
{
"italiano": "it"
"iso_code": "it",
"self_name": "italiano",
"translation_count": 29
},
{
"日本語": "ja"
"iso_code": "ja",
"self_name": "日本語",
"translation_count": 26
},
{
"한국어": "ko"
"iso_code": "ko",
"self_name": "한국어",
"translation_count": 13
},
{
"kurdî": "ku-latn"
"iso_code": "ku-latn",
"self_name": "kurdî",
"translation_count": 26
},
{
"Lëtzebuergesch": "lb"
"iso_code": "lb",
"self_name": "Lëtzebuergesch",
"translation_count": 22
},
{
"македонски": "mk"
"iso_code": "mk",
"self_name": "македонски",
"translation_count": 52
},
{
"Bahasa Melayu": "ms"
"iso_code": "ms",
"self_name": "Bahasa Melayu",
"translation_count": 14
},
{
"Nederlands": "nl"
"iso_code": "nl",
"self_name": "Nederlands",
"translation_count": 49
},
{
"ߒߞߏ": "nqo"
"iso_code": "nqo",
"self_name": "ߒߞߏ",
"translation_count": 43
},
{
"ଓଡ଼ିଆ": "or"
"iso_code": "or",
"self_name": "ଓଡ଼ିଆ",
"translation_count": 49
},
{
"Polski": "pl"
"iso_code": "pl",
"self_name": "Polski",
"translation_count": 24
},
{
"русский": "ru"
"iso_code": "ru",
"self_name": "русский",
"translation_count": 45
},
{
"Sardu": "sc"
"iso_code": "sc",
"self_name": "Sardu",
"translation_count": 49
},
{
"slovenčina": "sk"
"iso_code": "sk",
"self_name": "slovenčina",
"translation_count": 25
},
{
"سرائیکی": "skr-arab"
"iso_code": "skr-arab",
"self_name": "سرائیکی",
"translation_count": 20
},
{
"slovenščina": "sl"
"iso_code": "sl",
"self_name": "slovenščina",
"translation_count": 52
},
{
"Shqip": "sq"
"iso_code": "sq",
"self_name": "Shqip",
"translation_count": 49
},
{
"Svenska": "sv"
"iso_code": "sv",
"self_name": "Svenska",
"translation_count": 52
},
{
"ఇంగ్లీషు": "te"
"iso_code": "te",
"self_name": "ఇంగ్లీషు",
"translation_count": 49
},
{
"Türkçe": "tr"
"iso_code": "tr",
"self_name": "Türkçe",
"translation_count": 25
},
{
"英语": "zh-hans"
"iso_code": "zh-hans",
"self_name": "英语",
"translation_count": 16
},
{
"繁體中文": "zh-hant"
"iso_code": "zh-hant",
"self_name": "繁體中文",
"translation_count": 52
}
])EXPECTEDRESPONSE");
}
@ -1237,7 +1305,6 @@ TEST_F(ServerTest, UserLanguageControl)
const std::string description;
const std::string url;
const std::string acceptLanguageHeader;
const char* const requestCookie; // Cookie: header of the request
const std::string expectedH1;
operator TestContext() const
@ -1248,64 +1315,45 @@ TEST_F(ServerTest, UserLanguageControl)
{"acceptLanguageHeader", acceptLanguageHeader},
};
if ( requestCookie ) {
ctx.push_back({"requestCookie", requestCookie});
}
return ctx;
}
};
const char* const NO_COOKIE = nullptr;
const TestData testData[] = {
{
"Default user language is English",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article",
/*Accept-Language:*/ "",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "Not Found"
},
{
"userlang URL query parameter is respected",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article?userlang=en",
/*Accept-Language:*/ "",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "Not Found"
},
{
"userlang URL query parameter is respected",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article?userlang=test",
/*Accept-Language:*/ "",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "[I18N TESTING] Content not found, but at least the server is alive"
},
{
"'Accept-Language: *' is handled",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article",
/*Accept-Language:*/ "*",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "Not Found"
},
{
"Accept-Language: header is respected",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article",
/*Accept-Language:*/ "test",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "[I18N TESTING] Content not found, but at least the server is alive"
},
{
"userlang cookie is ignored",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article",
/*Accept-Language:*/ "",
/*Request Cookie:*/ "userlang=test",
/* expected <h1> */ "Not Found"
},
{
"userlang query parameter takes precedence over Accept-Language",
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article?userlang=en",
/*Accept-Language:*/ "test",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "Not Found"
},
{
@ -1314,7 +1362,6 @@ TEST_F(ServerTest, UserLanguageControl)
// with quality values) the most suitable language is selected.
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article",
/*Accept-Language:*/ "test;q=0.9, en;q=0.2",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "[I18N TESTING] Content not found, but at least the server is alive"
},
{
@ -1323,7 +1370,6 @@ TEST_F(ServerTest, UserLanguageControl)
// with quality values) the most suitable language is selected.
/*url*/ "/ROOT%23%3F/content/zimfile/invalid-article",
/*Accept-Language:*/ "test;q=0.2, en;q=0.9",
/*Request Cookie:*/ NO_COOKIE,
/* expected <h1> */ "Not Found"
},
};
@ -1335,9 +1381,6 @@ TEST_F(ServerTest, UserLanguageControl)
if ( !t.acceptLanguageHeader.empty() ) {
headers.insert({"Accept-Language", t.acceptLanguageHeader});
}
if ( t.requestCookie ) {
headers.insert({"Cookie", t.requestCookie});
}
const auto r = zfs1_->GET(t.url.c_str(), headers);
EXPECT_FALSE(r->has_header("Set-Cookie"));
std::regex_search(r->body, h1Match, h1Regex);
@ -2034,8 +2077,7 @@ TEST_F(ServerTest, viewerSettings)
R"(const viewerSettings = {
toolbarEnabled: false,
linkBlockingEnabled: false,
libraryButtonEnabled: false,
defaultUserLanguage: "en"
libraryButtonEnabled: false
}
)");
}
@ -2046,8 +2088,7 @@ R"(const viewerSettings = {
R"(const viewerSettings = {
toolbarEnabled: false,
linkBlockingEnabled: true,
libraryButtonEnabled: false,
defaultUserLanguage: "en"
libraryButtonEnabled: false
}
)");
}
@ -2058,8 +2099,7 @@ R"(const viewerSettings = {
R"(const viewerSettings = {
toolbarEnabled: true,
linkBlockingEnabled: false,
libraryButtonEnabled: false,
defaultUserLanguage: "en"
libraryButtonEnabled: false
}
)");
}
@ -2070,47 +2110,7 @@ R"(const viewerSettings = {
R"(const viewerSettings = {
toolbarEnabled: true,
linkBlockingEnabled: false,
libraryButtonEnabled: true,
defaultUserLanguage: "en"
}
)");
}
{
resetServer(ZimFileServer::WITH_TASKBAR_AND_LIBRARY_BUTTON);
const Headers headers{ {"Accept-Language", "fr"} };
ASSERT_EQ(zfs1_->GET("/ROOT%23%3F/viewer_settings.js", headers)->body,
R"(const viewerSettings = {
toolbarEnabled: true,
linkBlockingEnabled: false,
libraryButtonEnabled: true,
defaultUserLanguage: "fr"
}
)");
}
{
resetServer(ZimFileServer::WITH_TASKBAR_AND_LIBRARY_BUTTON);
const Headers headers{ {"Accept-Language", "test;q=0.2, en;q=0.9"} };
ASSERT_EQ(zfs1_->GET("/ROOT%23%3F/viewer_settings.js", headers)->body,
R"(const viewerSettings = {
toolbarEnabled: true,
linkBlockingEnabled: false,
libraryButtonEnabled: true,
defaultUserLanguage: "en"
}
)");
}
{
resetServer(ZimFileServer::WITH_TASKBAR_AND_LIBRARY_BUTTON);
const Headers headers{ {"Accept-Language", "test;q=0.9, en;q=0.2"} };
ASSERT_EQ(zfs1_->GET("/ROOT%23%3F/viewer_settings.js", headers)->body,
R"(const viewerSettings = {
toolbarEnabled: true,
linkBlockingEnabled: false,
libraryButtonEnabled: true,
defaultUserLanguage: "test"
libraryButtonEnabled: true
}
)");
}