Merge pull request #871 from kiwix/zim_viewer_i18n

Internationalization of ZIM viewer

scripts/kiwix-compile-resources

@@ -202,14 +202,16 @@ if __name__ == "__main__":
     parser.add_argument('--source_dir',
                         help="Additional directory where to look for resources.",
                         action='append')
-    parser.add_argument('resource_file',
+    parser.add_argument('resource_files', nargs='+',
                         help='The list of resources to compile.')
     args = parser.parse_args()
 
-    base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
-    with open(args.resource_file, 'r') as f:
+    resources = []
-        resources = [Resource([base_dir]+source_dir, *line.strip().split())
+    for resfile in args.resource_files:
+        base_dir = os.path.dirname(os.path.realpath(resfile))
+        with open(resfile, 'r') as f:
+            resources += [Resource([base_dir]+source_dir, *line.strip().split())
                             for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))

scripts/kiwix-compile-resources.1

@@ -2,7 +2,7 @@
 .SH NAME
 kiwix-compile-resources \- helper to compile and generate some Kiwix resources
 .SH SYNOPSIS
-\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file ...\fR
 .SH DESCRIPTION
 .TP
 resource_file

static/generate_i18n_resources_list.py

@@ -17,15 +17,36 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from pathlib import Path
+import json
 
 script_path = Path(__file__)
 
 resource_file = script_path.parent / "i18n_resources_list.txt"
-translation_dir = script_path.parent / "i18n"
+translation_dir = script_path.parent / "skin/i18n"
+language_list_relpath = "skin/languages.js"
 
+def get_translation_info(filepath):
+    lang_code = Path(filepath).stem
+    with open(filepath, 'r', encoding="utf-8") as f:
+        content = json.load(f)
+        return lang_code, content["name"]
+
+language_list = []
 json_files = translation_dir.glob("*.json")
 with open(resource_file, 'w', encoding="utf-8") as f:
-    for json in sorted(translation_dir.glob("*.json")):
+    for i18n_file in sorted(translation_dir.glob("*.json")):
-        if json.name == "qqq.json":
+        if i18n_file.name == "qqq.json":
             continue
-        f.write(str(json.relative_to(script_path.parent)) + '\n')
+        print("Processing", i18n_file.name)
+        if i18n_file.name != "test.json":
+            language_list.append(get_translation_info(i18n_file))
+        f.write(str(i18n_file.relative_to(script_path.parent)) + '\n')
+
+language_list = [{name: code} for code, name in sorted(language_list)]
+language_list_jsobj_str = json.dumps(language_list,
+                                     indent=2,
+                                     ensure_ascii=False)
+print("Saving", language_list_relpath)
+fullpath = script_path.parent / language_list_relpath
+with open(fullpath, 'w', encoding="utf-8") as f:
+    f.write("const uiLanguages = " + language_list_jsobj_str)

static/i18n_resources_list.txt

@@ -1,24 +1,24 @@
-i18n/ar.json
+skin/i18n/ar.json
-i18n/bn.json
+skin/i18n/bn.json
-i18n/cs.json
+skin/i18n/cs.json
-i18n/de.json
+skin/i18n/de.json
-i18n/en.json
+skin/i18n/en.json
-i18n/fr.json
+skin/i18n/fr.json
-i18n/he.json
+skin/i18n/he.json
-i18n/hy.json
+skin/i18n/hy.json
-i18n/it.json
+skin/i18n/it.json
-i18n/ja.json
+skin/i18n/ja.json
-i18n/ko.json
+skin/i18n/ko.json
-i18n/ku-latn.json
+skin/i18n/ku-latn.json
-i18n/mk.json
+skin/i18n/mk.json
-i18n/nqo.json
+skin/i18n/nqo.json
-i18n/pl.json
+skin/i18n/pl.json
-i18n/ru.json
+skin/i18n/ru.json
-i18n/sc.json
+skin/i18n/sc.json
-i18n/sk.json
+skin/i18n/sk.json
-i18n/sl.json
+skin/i18n/sl.json
-i18n/sv.json
+skin/i18n/sv.json
-i18n/test.json
+skin/i18n/test.json
-i18n/tr.json
+skin/i18n/tr.json
-i18n/zh-hans.json
+skin/i18n/zh-hans.json
-i18n/zh-hant.json
+skin/i18n/zh-hant.json

static/meson.build

@@ -15,7 +15,7 @@ preprocessed_resources = custom_target('preprocessed_resource_files',
 )
 
 lib_resources = custom_target('resources',
-    input: preprocessed_resources,
+    input: [preprocessed_resources, 'i18n_resources_list.txt'],
     output: ['libkiwix-resources.cpp', 'libkiwix-resources.h'],
     command:[res_compiler,
              '--cxxfile', '@OUTPUT0@',

static/resources_list.txt

@@ -15,6 +15,9 @@ skin/fonts/Roboto.ttf
 skin/search_results.css
 skin/blank.html
 skin/viewer.js
+skin/i18n.js
+skin/languages.js
+skin/mustache.min.js
 viewer.html
 templates/search_result.html
 templates/search_result.xml

static/skin/i18n.js

@@ -0,0 +1,103 @@
+import mustache from '../skin/mustache.min.js?KIWIXCACHEID'
+
+const Translations = {
+  defaultLanguage: null,
+  currentLanguage: null,
+  promises: {},
+  data: {},
+
+  load: function(lang, asDefault=false) {
+    if ( asDefault ) {
+      this.defaultLanguage = lang;
+      this.loadTranslationsJSON(lang);
+    } else {
+      this.currentLanguage = lang;
+      if ( lang != this.defaultLanguage ) {
+        this.loadTranslationsJSON(lang);
+      }
+    }
+  },
+
+  loadTranslationsJSON: function(lang) {
+    if ( this.promises[lang] )
+      return;
+
+    const errorMsg = `Error loading translations for language '${lang}': `;
+    this.promises[lang] = fetch(`./skin/i18n/${lang}.json`).then(async (resp) => {
+      if ( resp.ok ) {
+        this.data[lang] = JSON.parse(await resp.text());
+      } else {
+        console.log(errorMsg + resp.statusText);
+      }
+    }).catch((err) => {
+      console.log(errorMsg + err);
+    });
+  },
+
+  whenReady: function(callback) {
+    const defaultLangPromise = this.promises[this.defaultLanguage];
+    const currentLangPromise = this.promises[this.currentLanguage];
+    Promise.all([defaultLangPromise, currentLangPromise]).then(callback);
+  },
+
+  get: function(msgId) {
+    const activeTranslation = this.data[this.currentLanguage];
+
+    const r = activeTranslation && activeTranslation[msgId];
+    if ( r )
+      return r;
+
+    const defaultMsgs = this.data[this.defaultLanguage];
+    if ( defaultMsgs )
+      return defaultMsgs[msgId];
+
+    throw "Translations are not loaded";
+  }
+}
+
+
+function $t(msgId, params={}) {
+  try {
+    const msgTemplate = Translations.get(msgId);
+    if ( ! msgTemplate ) {
+      return "Invalid message id: " + msgId;
+    }
+
+    return mustache.render(msgTemplate, params);
+  } catch (err) {
+    return "ERROR: " + err;
+  }
+}
+
+function getCookie(cookieName) {
+    const name = cookieName + "=";
+    let result;
+    decodeURIComponent(document.cookie).split('; ').forEach(val => {
+        if (val.indexOf(name) === 0) {
+            result = val.substring(name.length);
+        }
+    });
+    return result;
+}
+
+
+const DEFAULT_UI_LANGUAGE = 'en';
+
+Translations.load(DEFAULT_UI_LANGUAGE, /*asDefault=*/true);
+
+function getUserLanguage() {
+  return new URLSearchParams(window.location.search).get('userlang')
+      || getCookie('userlang')
+      || DEFAULT_UI_LANGUAGE;
+}
+
+function setUserLanguage(lang, callback) {
+  const rootPath = new URL('..', import.meta.url).pathname.replace(/\/$/, '');
+  document.cookie = `userlang=${lang};path=${rootPath};max-age=31536000`;
+  Translations.load(lang);
+  Translations.whenReady(callback);
+}
+
+window.$t = $t;
+window.getUserLanguage = getUserLanguage;
+window.setUserLanguage = setUserLanguage;

static/skin/i18n/ja.json

@@ -4,6 +4,7 @@
 			"MathXplore"
 		]
 	},
+	"name": "日本語",
 	"no-query": "クエリを指定していません。",
 	"400-page-title": "無効なリクエストです",
 	"400-page-heading": "無効なリクエストです",

static/skin/languages.js

@@ -0,0 +1,65 @@
+const uiLanguages = [
+  {
+    "বাংলা": "bn"
+  },
+  {
+    "Čeština": "cs"
+  },
+  {
+    "Deutsch": "de"
+  },
+  {
+    "English": "en"
+  },
+  {
+    "français": "fr"
+  },
+  {
+    "עברית": "he"
+  },
+  {
+    "Հայերեն": "hy"
+  },
+  {
+    "italiano": "it"
+  },
+  {
+    "日本語": "ja"
+  },
+  {
+    "한국어": "ko"
+  },
+  {
+    "kurdî": "ku-latn"
+  },
+  {
+    "македонски": "mk"
+  },
+  {
+    "ߒߞߏ": "nqo"
+  },
+  {
+    "Polski": "pl"
+  },
+  {
+    "русский": "ru"
+  },
+  {
+    "Sardu": "sc"
+  },
+  {
+    "slovenčina": "sk"
+  },
+  {
+    "Svenska": "sv"
+  },
+  {
+    "Türkçe": "tr"
+  },
+  {
+    "英语": "zh-hans"
+  },
+  {
+    "繁體中文": "zh-hant"
+  }
+]

static/skin/mustache.js

@@ -0,0 +1,764 @@
+/*!
+ * mustache.js - Logic-less {{mustache}} templates with JavaScript
+ * http://github.com/janl/mustache.js
+ */
+
+var objectToString = Object.prototype.toString;
+var isArray = Array.isArray || function isArrayPolyfill (object) {
+  return objectToString.call(object) === '[object Array]';
+};
+
+function isFunction (object) {
+  return typeof object === 'function';
+}
+
+/**
+ * More correct typeof string handling array
+ * which normally returns typeof 'object'
+ */
+function typeStr (obj) {
+  return isArray(obj) ? 'array' : typeof obj;
+}
+
+function escapeRegExp (string) {
+  return string.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g, '\\$&');
+}
+
+/**
+ * Null safe way of checking whether or not an object,
+ * including its prototype, has a given property
+ */
+function hasProperty (obj, propName) {
+  return obj != null && typeof obj === 'object' && (propName in obj);
+}
+
+/**
+ * Safe way of detecting whether or not the given thing is a primitive and
+ * whether it has the given property
+ */
+function primitiveHasOwnProperty (primitive, propName) {
+  return (
+    primitive != null
+    && typeof primitive !== 'object'
+    && primitive.hasOwnProperty
+    && primitive.hasOwnProperty(propName)
+  );
+}
+
+// Workaround for https://issues.apache.org/jira/browse/COUCHDB-577
+// See https://github.com/janl/mustache.js/issues/189
+var regExpTest = RegExp.prototype.test;
+function testRegExp (re, string) {
+  return regExpTest.call(re, string);
+}
+
+var nonSpaceRe = /\S/;
+function isWhitespace (string) {
+  return !testRegExp(nonSpaceRe, string);
+}
+
+var entityMap = {
+  '&': '&',
+  '<': '&lt;',
+  '>': '&gt;',
+  '"': '&quot;',
+  "'": '&#39;',
+  '/': '&#x2F;',
+  '`': '&#x60;',
+  '=': '&#x3D;'
+};
+
+function escapeHtml (string) {
+  return String(string).replace(/[&<>"'`=\/]/g, function fromEntityMap (s) {
+    return entityMap[s];
+  });
+}
+
+var whiteRe = /\s*/;
+var spaceRe = /\s+/;
+var equalsRe = /\s*=/;
+var curlyRe = /\s*\}/;
+var tagRe = /#|\^|\/|>|\{|&|=|!/;
+
+/**
+ * Breaks up the given `template` string into a tree of tokens. If the `tags`
+ * argument is given here it must be an array with two string values: the
+ * opening and closing tags used in the template (e.g. [ "<%", "%>" ]). Of
+ * course, the default is to use mustaches (i.e. mustache.tags).
+ *
+ * A token is an array with at least 4 elements. The first element is the
+ * mustache symbol that was used inside the tag, e.g. "#" or "&". If the tag
+ * did not contain a symbol (i.e. {{myValue}}) this element is "name". For
+ * all text that appears outside a symbol this element is "text".
+ *
+ * The second element of a token is its "value". For mustache tags this is
+ * whatever else was inside the tag besides the opening symbol. For text tokens
+ * this is the text itself.
+ *
+ * The third and fourth elements of the token are the start and end indices,
+ * respectively, of the token in the original template.
+ *
+ * Tokens that are the root node of a subtree contain two more elements: 1) an
+ * array of tokens in the subtree and 2) the index in the original template at
+ * which the closing tag for that section begins.
+ *
+ * Tokens for partials also contain two more elements: 1) a string value of
+ * indendation prior to that tag and 2) the index of that tag on that line -
+ * eg a value of 2 indicates the partial is the third tag on this line.
+ */
+function parseTemplate (template, tags) {
+  if (!template)
+    return [];
+  var lineHasNonSpace = false;
+  var sections = [];     // Stack to hold section tokens
+  var tokens = [];       // Buffer to hold the tokens
+  var spaces = [];       // Indices of whitespace tokens on the current line
+  var hasTag = false;    // Is there a {{tag}} on the current line?
+  var nonSpace = false;  // Is there a non-space char on the current line?
+  var indentation = '';  // Tracks indentation for tags that use it
+  var tagIndex = 0;      // Stores a count of number of tags encountered on a line
+
+  // Strips all whitespace tokens array for the current line
+  // if there was a {{#tag}} on it and otherwise only space.
+  function stripSpace () {
+    if (hasTag && !nonSpace) {
+      while (spaces.length)
+        delete tokens[spaces.pop()];
+    } else {
+      spaces = [];
+    }
+
+    hasTag = false;
+    nonSpace = false;
+  }
+
+  var openingTagRe, closingTagRe, closingCurlyRe;
+  function compileTags (tagsToCompile) {
+    if (typeof tagsToCompile === 'string')
+      tagsToCompile = tagsToCompile.split(spaceRe, 2);
+
+    if (!isArray(tagsToCompile) || tagsToCompile.length !== 2)
+      throw new Error('Invalid tags: ' + tagsToCompile);
+
+    openingTagRe = new RegExp(escapeRegExp(tagsToCompile[0]) + '\\s*');
+    closingTagRe = new RegExp('\\s*' + escapeRegExp(tagsToCompile[1]));
+    closingCurlyRe = new RegExp('\\s*' + escapeRegExp('}' + tagsToCompile[1]));
+  }
+
+  compileTags(tags || mustache.tags);
+
+  var scanner = new Scanner(template);
+
+  var start, type, value, chr, token, openSection;
+  while (!scanner.eos()) {
+    start = scanner.pos;
+
+    // Match any text between tags.
+    value = scanner.scanUntil(openingTagRe);
+
+    if (value) {
+      for (var i = 0, valueLength = value.length; i < valueLength; ++i) {
+        chr = value.charAt(i);
+
+        if (isWhitespace(chr)) {
+          spaces.push(tokens.length);
+          indentation += chr;
+        } else {
+          nonSpace = true;
+          lineHasNonSpace = true;
+          indentation += ' ';
+        }
+
+        tokens.push([ 'text', chr, start, start + 1 ]);
+        start += 1;
+
+        // Check for whitespace on the current line.
+        if (chr === '\n') {
+          stripSpace();
+          indentation = '';
+          tagIndex = 0;
+          lineHasNonSpace = false;
+        }
+      }
+    }
+
+    // Match the opening tag.
+    if (!scanner.scan(openingTagRe))
+      break;
+
+    hasTag = true;
+
+    // Get the tag type.
+    type = scanner.scan(tagRe) || 'name';
+    scanner.scan(whiteRe);
+
+    // Get the tag value.
+    if (type === '=') {
+      value = scanner.scanUntil(equalsRe);
+      scanner.scan(equalsRe);
+      scanner.scanUntil(closingTagRe);
+    } else if (type === '{') {
+      value = scanner.scanUntil(closingCurlyRe);
+      scanner.scan(curlyRe);
+      scanner.scanUntil(closingTagRe);
+      type = '&';
+    } else {
+      value = scanner.scanUntil(closingTagRe);
+    }
+
+    // Match the closing tag.
+    if (!scanner.scan(closingTagRe))
+      throw new Error('Unclosed tag at ' + scanner.pos);
+
+    if (type == '>') {
+      token = [ type, value, start, scanner.pos, indentation, tagIndex, lineHasNonSpace ];
+    } else {
+      token = [ type, value, start, scanner.pos ];
+    }
+    tagIndex++;
+    tokens.push(token);
+
+    if (type === '#' || type === '^') {
+      sections.push(token);
+    } else if (type === '/') {
+      // Check section nesting.
+      openSection = sections.pop();
+
+      if (!openSection)
+        throw new Error('Unopened section "' + value + '" at ' + start);
+
+      if (openSection[1] !== value)
+        throw new Error('Unclosed section "' + openSection[1] + '" at ' + start);
+    } else if (type === 'name' || type === '{' || type === '&') {
+      nonSpace = true;
+    } else if (type === '=') {
+      // Set the tags for the next time around.
+      compileTags(value);
+    }
+  }
+
+  stripSpace();
+
+  // Make sure there are no open sections when we're done.
+  openSection = sections.pop();
+
+  if (openSection)
+    throw new Error('Unclosed section "' + openSection[1] + '" at ' + scanner.pos);
+
+  return nestTokens(squashTokens(tokens));
+}
+
+/**
+ * Combines the values of consecutive text tokens in the given `tokens` array
+ * to a single token.
+ */
+function squashTokens (tokens) {
+  var squashedTokens = [];
+
+  var token, lastToken;
+  for (var i = 0, numTokens = tokens.length; i < numTokens; ++i) {
+    token = tokens[i];
+
+    if (token) {
+      if (token[0] === 'text' && lastToken && lastToken[0] === 'text') {
+        lastToken[1] += token[1];
+        lastToken[3] = token[3];
+      } else {
+        squashedTokens.push(token);
+        lastToken = token;
+      }
+    }
+  }
+
+  return squashedTokens;
+}
+
+/**
+ * Forms the given array of `tokens` into a nested tree structure where
+ * tokens that represent a section have two additional items: 1) an array of
+ * all tokens that appear in that section and 2) the index in the original
+ * template that represents the end of that section.
+ */
+function nestTokens (tokens) {
+  var nestedTokens = [];
+  var collector = nestedTokens;
+  var sections = [];
+
+  var token, section;
+  for (var i = 0, numTokens = tokens.length; i < numTokens; ++i) {
+    token = tokens[i];
+
+    switch (token[0]) {
+      case '#':
+      case '^':
+        collector.push(token);
+        sections.push(token);
+        collector = token[4] = [];
+        break;
+      case '/':
+        section = sections.pop();
+        section[5] = token[2];
+        collector = sections.length > 0 ? sections[sections.length - 1][4] : nestedTokens;
+        break;
+      default:
+        collector.push(token);
+    }
+  }
+
+  return nestedTokens;
+}
+
+/**
+ * A simple string scanner that is used by the template parser to find
+ * tokens in template strings.
+ */
+function Scanner (string) {
+  this.string = string;
+  this.tail = string;
+  this.pos = 0;
+}
+
+/**
+ * Returns `true` if the tail is empty (end of string).
+ */
+Scanner.prototype.eos = function eos () {
+  return this.tail === '';
+};
+
+/**
+ * Tries to match the given regular expression at the current position.
+ * Returns the matched text if it can match, the empty string otherwise.
+ */
+Scanner.prototype.scan = function scan (re) {
+  var match = this.tail.match(re);
+
+  if (!match || match.index !== 0)
+    return '';
+
+  var string = match[0];
+
+  this.tail = this.tail.substring(string.length);
+  this.pos += string.length;
+
+  return string;
+};
+
+/**
+ * Skips all text until the given regular expression can be matched. Returns
+ * the skipped string, which is the entire tail if no match can be made.
+ */
+Scanner.prototype.scanUntil = function scanUntil (re) {
+  var index = this.tail.search(re), match;
+
+  switch (index) {
+    case -1:
+      match = this.tail;
+      this.tail = '';
+      break;
+    case 0:
+      match = '';
+      break;
+    default:
+      match = this.tail.substring(0, index);
+      this.tail = this.tail.substring(index);
+  }
+
+  this.pos += match.length;
+
+  return match;
+};
+
+/**
+ * Represents a rendering context by wrapping a view object and
+ * maintaining a reference to the parent context.
+ */
+function Context (view, parentContext) {
+  this.view = view;
+  this.cache = { '.': this.view };
+  this.parent = parentContext;
+}
+
+/**
+ * Creates a new context using the given view with this context
+ * as the parent.
+ */
+Context.prototype.push = function push (view) {
+  return new Context(view, this);
+};
+
+/**
+ * Returns the value of the given name in this context, traversing
+ * up the context hierarchy if the value is absent in this context's view.
+ */
+Context.prototype.lookup = function lookup (name) {
+  var cache = this.cache;
+
+  var value;
+  if (cache.hasOwnProperty(name)) {
+    value = cache[name];
+  } else {
+    var context = this, intermediateValue, names, index, lookupHit = false;
+
+    while (context) {
+      if (name.indexOf('.') > 0) {
+        intermediateValue = context.view;
+        names = name.split('.');
+        index = 0;
+
+        /**
+         * Using the dot notion path in `name`, we descend through the
+         * nested objects.
+         *
+         * To be certain that the lookup has been successful, we have to
+         * check if the last object in the path actually has the property
+         * we are looking for. We store the result in `lookupHit`.
+         *
+         * This is specially necessary for when the value has been set to
+         * `undefined` and we want to avoid looking up parent contexts.
+         *
+         * In the case where dot notation is used, we consider the lookup
+         * to be successful even if the last "object" in the path is
+         * not actually an object but a primitive (e.g., a string, or an
+         * integer), because it is sometimes useful to access a property
+         * of an autoboxed primitive, such as the length of a string.
+         **/
+        while (intermediateValue != null && index < names.length) {
+          if (index === names.length - 1)
+            lookupHit = (
+              hasProperty(intermediateValue, names[index])
+              || primitiveHasOwnProperty(intermediateValue, names[index])
+            );
+
+          intermediateValue = intermediateValue[names[index++]];
+        }
+      } else {
+        intermediateValue = context.view[name];
+
+        /**
+         * Only checking against `hasProperty`, which always returns `false` if
+         * `context.view` is not an object. Deliberately omitting the check
+         * against `primitiveHasOwnProperty` if dot notation is not used.
+         *
+         * Consider this example:
+         * ```
+         * Mustache.render("The length of a football field is {{#length}}{{length}}{{/length}}.", {length: "100 yards"})
+         * ```
+         *
+         * If we were to check also against `primitiveHasOwnProperty`, as we do
+         * in the dot notation case, then render call would return:
+         *
+         * "The length of a football field is 9."
+         *
+         * rather than the expected:
+         *
+         * "The length of a football field is 100 yards."
+         **/
+        lookupHit = hasProperty(context.view, name);
+      }
+
+      if (lookupHit) {
+        value = intermediateValue;
+        break;
+      }
+
+      context = context.parent;
+    }
+
+    cache[name] = value;
+  }
+
+  if (isFunction(value))
+    value = value.call(this.view);
+
+  return value;
+};
+
+/**
+ * A Writer knows how to take a stream of tokens and render them to a
+ * string, given a context. It also maintains a cache of templates to
+ * avoid the need to parse the same template twice.
+ */
+function Writer () {
+  this.templateCache = {
+    _cache: {},
+    set: function set (key, value) {
+      this._cache[key] = value;
+    },
+    get: function get (key) {
+      return this._cache[key];
+    },
+    clear: function clear () {
+      this._cache = {};
+    }
+  };
+}
+
+/**
+ * Clears all cached templates in this writer.
+ */
+Writer.prototype.clearCache = function clearCache () {
+  if (typeof this.templateCache !== 'undefined') {
+    this.templateCache.clear();
+  }
+};
+
+/**
+ * Parses and caches the given `template` according to the given `tags` or
+ * `mustache.tags` if `tags` is omitted,  and returns the array of tokens
+ * that is generated from the parse.
+ */
+Writer.prototype.parse = function parse (template, tags) {
+  var cache = this.templateCache;
+  var cacheKey = template + ':' + (tags || mustache.tags).join(':');
+  var isCacheEnabled = typeof cache !== 'undefined';
+  var tokens = isCacheEnabled ? cache.get(cacheKey) : undefined;
+
+  if (tokens == undefined) {
+    tokens = parseTemplate(template, tags);
+    isCacheEnabled && cache.set(cacheKey, tokens);
+  }
+  return tokens;
+};
+
+/**
+ * High-level method that is used to render the given `template` with
+ * the given `view`.
+ *
+ * The optional `partials` argument may be an object that contains the
+ * names and templates of partials that are used in the template. It may
+ * also be a function that is used to load partial templates on the fly
+ * that takes a single argument: the name of the partial.
+ *
+ * If the optional `config` argument is given here, then it should be an
+ * object with a `tags` attribute or an `escape` attribute or both.
+ * If an array is passed, then it will be interpreted the same way as
+ * a `tags` attribute on a `config` object.
+ *
+ * The `tags` attribute of a `config` object must be an array with two
+ * string values: the opening and closing tags used in the template (e.g.
+ * [ "<%", "%>" ]). The default is to mustache.tags.
+ *
+ * The `escape` attribute of a `config` object must be a function which
+ * accepts a string as input and outputs a safely escaped string.
+ * If an `escape` function is not provided, then an HTML-safe string
+ * escaping function is used as the default.
+ */
+Writer.prototype.render = function render (template, view, partials, config) {
+  var tags = this.getConfigTags(config);
+  var tokens = this.parse(template, tags);
+  var context = (view instanceof Context) ? view : new Context(view, undefined);
+  return this.renderTokens(tokens, context, partials, template, config);
+};
+
+/**
+ * Low-level method that renders the given array of `tokens` using
+ * the given `context` and `partials`.
+ *
+ * Note: The `originalTemplate` is only ever used to extract the portion
+ * of the original template that was contained in a higher-order section.
+ * If the template doesn't use higher-order sections, this argument may
+ * be omitted.
+ */
+Writer.prototype.renderTokens = function renderTokens (tokens, context, partials, originalTemplate, config) {
+  var buffer = '';
+
+  var token, symbol, value;
+  for (var i = 0, numTokens = tokens.length; i < numTokens; ++i) {
+    value = undefined;
+    token = tokens[i];
+    symbol = token[0];
+
+    if (symbol === '#') value = this.renderSection(token, context, partials, originalTemplate, config);
+    else if (symbol === '^') value = this.renderInverted(token, context, partials, originalTemplate, config);
+    else if (symbol === '>') value = this.renderPartial(token, context, partials, config);
+    else if (symbol === '&') value = this.unescapedValue(token, context);
+    else if (symbol === 'name') value = this.escapedValue(token, context, config);
+    else if (symbol === 'text') value = this.rawValue(token);
+
+    if (value !== undefined)
+      buffer += value;
+  }
+
+  return buffer;
+};
+
+Writer.prototype.renderSection = function renderSection (token, context, partials, originalTemplate, config) {
+  var self = this;
+  var buffer = '';
+  var value = context.lookup(token[1]);
+
+  // This function is used to render an arbitrary template
+  // in the current context by higher-order sections.
+  function subRender (template) {
+    return self.render(template, context, partials, config);
+  }
+
+  if (!value) return;
+
+  if (isArray(value)) {
+    for (var j = 0, valueLength = value.length; j < valueLength; ++j) {
+      buffer += this.renderTokens(token[4], context.push(value[j]), partials, originalTemplate, config);
+    }
+  } else if (typeof value === 'object' || typeof value === 'string' || typeof value === 'number') {
+    buffer += this.renderTokens(token[4], context.push(value), partials, originalTemplate, config);
+  } else if (isFunction(value)) {
+    if (typeof originalTemplate !== 'string')
+      throw new Error('Cannot use higher-order sections without the original template');
+
+    // Extract the portion of the original template that the section contains.
+    value = value.call(context.view, originalTemplate.slice(token[3], token[5]), subRender);
+
+    if (value != null)
+      buffer += value;
+  } else {
+    buffer += this.renderTokens(token[4], context, partials, originalTemplate, config);
+  }
+  return buffer;
+};
+
+Writer.prototype.renderInverted = function renderInverted (token, context, partials, originalTemplate, config) {
+  var value = context.lookup(token[1]);
+
+  // Use JavaScript's definition of falsy. Include empty arrays.
+  // See https://github.com/janl/mustache.js/issues/186
+  if (!value || (isArray(value) && value.length === 0))
+    return this.renderTokens(token[4], context, partials, originalTemplate, config);
+};
+
+Writer.prototype.indentPartial = function indentPartial (partial, indentation, lineHasNonSpace) {
+  var filteredIndentation = indentation.replace(/[^ \t]/g, '');
+  var partialByNl = partial.split('\n');
+  for (var i = 0; i < partialByNl.length; i++) {
+    if (partialByNl[i].length && (i > 0 || !lineHasNonSpace)) {
+      partialByNl[i] = filteredIndentation + partialByNl[i];
+    }
+  }
+  return partialByNl.join('\n');
+};
+
+Writer.prototype.renderPartial = function renderPartial (token, context, partials, config) {
+  if (!partials) return;
+  var tags = this.getConfigTags(config);
+
+  var value = isFunction(partials) ? partials(token[1]) : partials[token[1]];
+  if (value != null) {
+    var lineHasNonSpace = token[6];
+    var tagIndex = token[5];
+    var indentation = token[4];
+    var indentedValue = value;
+    if (tagIndex == 0 && indentation) {
+      indentedValue = this.indentPartial(value, indentation, lineHasNonSpace);
+    }
+    var tokens = this.parse(indentedValue, tags);
+    return this.renderTokens(tokens, context, partials, indentedValue, config);
+  }
+};
+
+Writer.prototype.unescapedValue = function unescapedValue (token, context) {
+  var value = context.lookup(token[1]);
+  if (value != null)
+    return value;
+};
+
+Writer.prototype.escapedValue = function escapedValue (token, context, config) {
+  var escape = this.getConfigEscape(config) || mustache.escape;
+  var value = context.lookup(token[1]);
+  if (value != null)
+    return (typeof value === 'number' && escape === mustache.escape) ? String(value) : escape(value);
+};
+
+Writer.prototype.rawValue = function rawValue (token) {
+  return token[1];
+};
+
+Writer.prototype.getConfigTags = function getConfigTags (config) {
+  if (isArray(config)) {
+    return config;
+  }
+  else if (config && typeof config === 'object') {
+    return config.tags;
+  }
+  else {
+    return undefined;
+  }
+};
+
+Writer.prototype.getConfigEscape = function getConfigEscape (config) {
+  if (config && typeof config === 'object' && !isArray(config)) {
+    return config.escape;
+  }
+  else {
+    return undefined;
+  }
+};
+
+var mustache = {
+  name: 'mustache.js',
+  version: '4.2.0',
+  tags: [ '{{', '}}' ],
+  clearCache: undefined,
+  escape: undefined,
+  parse: undefined,
+  render: undefined,
+  Scanner: undefined,
+  Context: undefined,
+  Writer: undefined,
+  /**
+   * Allows a user to override the default caching strategy, by providing an
+   * object with set, get and clear methods. This can also be used to disable
+   * the cache by setting it to the literal `undefined`.
+   */
+  set templateCache (cache) {
+    defaultWriter.templateCache = cache;
+  },
+  /**
+   * Gets the default or overridden caching object from the default writer.
+   */
+  get templateCache () {
+    return defaultWriter.templateCache;
+  }
+};
+
+// All high-level mustache.* functions use this writer.
+var defaultWriter = new Writer();
+
+/**
+ * Clears all cached templates in the default writer.
+ */
+mustache.clearCache = function clearCache () {
+  return defaultWriter.clearCache();
+};
+
+/**
+ * Parses and caches the given template in the default writer and returns the
+ * array of tokens it contains. Doing this ahead of time avoids the need to
+ * parse templates on the fly as they are rendered.
+ */
+mustache.parse = function parse (template, tags) {
+  return defaultWriter.parse(template, tags);
+};
+
+/**
+ * Renders the `template` with the given `view`, `partials`, and `config`
+ * using the default writer.
+ */
+mustache.render = function render (template, view, partials, config) {
+  if (typeof template !== 'string') {
+    throw new TypeError('Invalid template! Template should be a "string" ' +
+                        'but "' + typeStr(template) + '" was given as the first ' +
+                        'argument for mustache#render(template, view, partials)');
+  }
+
+  return defaultWriter.render(template, view, partials, config);
+};
+
+// Export the escaping function so that the user may override it.
+// See https://github.com/janl/mustache.js/issues/244
+mustache.escape = escapeHtml;
+
+// Export these mainly for testing, but also for advanced usage.
+mustache.Scanner = Scanner;
+mustache.Context = Context;
+mustache.Writer = Writer;
+
+export default mustache;

static/skin/taskbar.css

@@ -44,6 +44,10 @@
     margin: 0 auto;
 }
 
+#ui_language {
+  float: left
+}
+
 #kiwix_button_show_toggle {
     display: none;
 }

static/skin/viewer.js

@@ -2,10 +2,14 @@
 //
 // user url: identifier of the page that has to be displayed in the viewer
 //           and that is used as the hash component of the viewer URL. For
-//           book resources the address url is {book}/{resource} .
+//           book resources the user url is {book}/{resource} .
 //
 // iframe url: the URL to be loaded in the viewer iframe.
 
+let viewerState = {
+  uiLanguage: 'en',
+};
+
 function userUrl2IframeUrl(url) {
   if ( url == '' ) {
     return blankPageUrl;
@@ -30,7 +34,7 @@ function getBookFromUserUrl(url) {
   return url.split('/')[0];
 }
 
-let currentBook = getBookFromUserUrl(location.hash.slice(1));
+let currentBook = null;
 let currentBookTitle = null;
 
 const bookUIGroup = document.getElementById('kiwix_serve_taskbar_book_ui_group');
@@ -68,14 +72,24 @@ function makeJSLink(jsCodeString, linkText, linkAttr="") {
 
 function suggestionsApiURL()
 {
-  return `${root}/suggest?content=${encodeURIComponent(currentBook)}`;
+  const uriEncodedBookName = encodeURIComponent(currentBook);
+  const userLang = viewerState.uiLanguage;
+  return `${root}/suggest?userlang=${userLang}&content=${uriEncodedBookName}`;
+}
+
+function setTitle(element, text) {
+  if ( element ) {
+    element.title = text;
+    if ( element.hasAttribute("aria-label") ) {
+      element.setAttribute("aria-label", text);
+    }
+  }
 }
 
 function setCurrentBook(book, title) {
   currentBook = book;
   currentBookTitle = title;
-  homeButton.title = `Go to the main page of '${title}'`;
+  setTitle(homeButton, $t("home-button-text", {BOOK_TITLE: title}));
-  homeButton.setAttribute("aria-label", homeButton.title);
   homeButton.innerHTML = `<button>${title}</button>`;
   bookUIGroup.style.display = 'inline';
   updateSearchBoxForBookChange();
@@ -153,7 +167,7 @@ function updateSearchBoxForBookChange() {
   const searchbox = document.getElementById('kiwixsearchbox');
   const kiwixSearchFormWrapper = document.querySelector('.kiwix_searchform');
   if ( currentBookTitle ) {
-    searchbox.title = `Search '${currentBookTitle}'`;
+    searchbox.title = $t("searchbox-tooltip", {BOOK_TITLE : currentBookTitle});
     searchbox.placeholder = searchbox.title;
     searchbox.setAttribute("aria-label", searchbox.title);
     kiwixSearchFormWrapper.style.display = 'inline';
@@ -197,6 +211,7 @@ function handle_location_hash_change() {
   }
   updateSearchBoxForLocationChange();
   previousScrollTop = Infinity;
+  history.replaceState(viewerState, null);
 }
 
 function handle_content_url_change() {
@@ -206,8 +221,7 @@ function handle_content_url_change() {
   const iframeContentUrl = iframeLocation.pathname;
   const iframeContentQuery = iframeLocation.search;
   const newHash = iframeUrl2UserUrl(iframeContentUrl, iframeContentQuery);
-  const viewerURL = location.origin + location.pathname + location.search;
+  history.replaceState(viewerState, null, makeURL(location.search, newHash));
-  window.location.replace(viewerURL + '#' + newHash);
   updateCurrentBookIfNeeded(newHash);
 };
 
@@ -291,17 +305,15 @@ function setup_external_link_blocker() {
 // End of external link blocking
 ////////////////////////////////////////////////////////////////////////////////
 
+let viewerSetupComplete = false;
+
 function on_content_load() {
+  if ( viewerSetupComplete ) {
     handle_content_url_change();
     setup_external_link_blocker();
+  }
 }
 
-window.onresize = handle_visual_viewport_change;
-window.onhashchange = handle_location_hash_change;
-
-updateCurrentBook(currentBook);
-handle_location_hash_change();
-
 function htmlDecode(input) {
     var doc = new DOMParser().parseFromString(input, "text/html");
     return doc.documentElement.textContent;
@@ -391,22 +403,82 @@ function setupSuggestions() {
   });
 }
 
+function makeURL(search, hash) {
+  let url = location.origin + location.pathname;
+  if (search != "") {
+    url += (search[0] == '?' ? search : '?' + search);
+  }
+
+  url += (hash[0] == '#' ? hash : '#' + hash);
+  return url;
+}
+
+function initUILanguageSelector() {
+  const languageSelector = document.getElementById("ui_language");
+  for (const lang of uiLanguages ) {
+    const lang_name = Object.getOwnPropertyNames(lang)[0];
+    const lang_code = lang[lang_name];
+    const is_selected = lang_code == viewerState.uiLanguage;
+    languageSelector.appendChild(new Option(lang_name, lang_code, is_selected, is_selected));
+  }
+}
+
+function updateUILanguageSelector(userLang) {
+  console.log(`updateUILanguageSelector(${userLang})`);
+  const languageSelector = document.getElementById("ui_language");
+  for (const opt of languageSelector.children ) {
+    if ( opt.value == userLang ) {
+      opt.selected = true;
+    }
+  }
+}
+
+function handle_history_state_change(event) {
+  console.log(`handle_history_state_change`);
+  if ( event.state ) {
+    viewerState = event.state;
+    updateUILanguageSelector(viewerState.uiLanguage);
+    setUserLanguage(viewerState.uiLanguage, updateUIText);
+  }
+}
+
+function changeUILanguage() {
+  const s = document.getElementById("ui_language");
+  const lang = s.options[s.selectedIndex].value;
+  viewerState.uiLanguage = lang;
+  setUserLanguage(lang, () => {
+    updateUIText();
+    history.pushState(viewerState, null);
+  });
+}
+
 function setupViewer() {
   // Defer the call of handle_visual_viewport_change() until after the
   // presence or absence of the taskbar as determined by this function
   // has been settled.
   setTimeout(handle_visual_viewport_change, 0);
 
+  window.onresize = handle_visual_viewport_change;
+
   const kiwixToolBarWrapper = document.getElementById('kiwixtoolbarwrapper');
   if ( ! viewerSettings.toolbarEnabled ) {
     return;
   }
 
+  const lang = getUserLanguage();
+  setUserLanguage(lang, finishViewerSetupOnceTranslationsAreLoaded);
+  viewerState.uiLanguage = lang;
+  const q = new URLSearchParams(window.location.search);
+  q.delete('userlang');
+  const rewrittenURL = makeURL(q.toString(), location.hash);
+  history.replaceState(viewerState, null, rewrittenURL);
+
   kiwixToolBarWrapper.style.display = 'block';
   if ( ! viewerSettings.libraryButtonEnabled ) {
     document.getElementById("kiwix_serve_taskbar_library_button").remove();
   }
 
+  initUILanguageSelector();
   setupSuggestions();
 
   // cybook hack
@@ -418,3 +490,25 @@ function setupViewer() {
     setupAutoHidingOfTheToolbar();
   }
 }
+
+function updateUIText() {
+  currentBook = getBookFromUserUrl(location.hash.slice(1));
+  updateCurrentBook(currentBook);
+
+  setTitle(document.getElementById("kiwix_serve_taskbar_library_button"),
+           $t("library-button-text"));
+
+  setTitle(document.getElementById("kiwix_serve_taskbar_random_button"),
+           $t("random-page-button-text"));
+}
+
+function finishViewerSetupOnceTranslationsAreLoaded()
+{
+  updateUIText();
+  handle_location_hash_change();
+
+  window.onhashchange = handle_location_hash_change;
+  window.onpopstate = handle_history_state_change;
+
+  viewerSetupComplete = true;
+}

static/viewer.html

@@ -7,6 +7,8 @@
     <link type="text/css" href="./skin/taskbar.css?KIWIXCACHEID" rel="Stylesheet" />
     <link type="text/css" href="./skin/css/autoComplete.css?KIWIXCACHEID" rel="Stylesheet" />
     <script type="text/javascript" src="./viewer_settings.js"></script>
+    <script type="module" src="./skin/i18n.js?KIWIXCACHEID" defer></script>
+    <script type="text/javascript" src="./skin/languages.js?KIWIXCACHEID" defer></script>
     <script type="text/javascript" src="./skin/viewer.js?KIWIXCACHEID" defer></script>
     <script type="text/javascript" src="./skin/autoComplete.min.js?KIWIXCACHEID"></script>
     <script>
@@ -27,6 +29,8 @@
   <body style="margin:0" onload="setupViewer()">
     <div class="kiwix" style="display:none" id="kiwixtoolbarwrapper">
       <div id="kiwixtoolbar" class="ui-widget-header">
+        <select id="ui_language" onchange="changeUILanguage()">
+        </select>
         <div class="kiwix_centered">
           <div class="kiwix_searchform">
             <form class="kiwixsearch" method="GET" action="javascript:performSearch()" id="kiwixsearchform">

test/server.cpp

@@ -58,6 +58,8 @@ const ResourceCollection resources200Compressible{
   { STATIC_CONTENT,  "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" },
   { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" },
   { STATIC_CONTENT,  "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/i18n.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/i18n.js?cacheid=dcf3d584" },
   { DYNAMIC_CONTENT, "/ROOT/skin/index.css" },
   { STATIC_CONTENT,  "/ROOT/skin/index.css?cacheid=0f9ba34e" },
   { DYNAMIC_CONTENT, "/ROOT/skin/index.js" },
@@ -66,10 +68,12 @@ const ResourceCollection resources200Compressible{
   { STATIC_CONTENT,  "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" },
   { DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" },
   { STATIC_CONTENT,  "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/mustache.min.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/mustache.min.js?cacheid=bd23c4fb" },
   { DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" },
-  { STATIC_CONTENT,  "/ROOT/skin/taskbar.css?cacheid=216d6b5d" },
+  { STATIC_CONTENT,  "/ROOT/skin/taskbar.css?cacheid=2cbac34b" },
   { DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" },
-  { STATIC_CONTENT,  "/ROOT/skin/viewer.js?cacheid=ab5374c5" },
+  { STATIC_CONTENT,  "/ROOT/skin/viewer.js?cacheid=b3c754ec" },
   { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" },
   { STATIC_CONTENT,  "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" },
   { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" },
@@ -135,6 +139,11 @@ const ResourceCollection resources200Uncompressible{
   { STATIC_CONTENT,  "/ROOT/skin/search-icon.svg?cacheid=b10ae7ed" },
   { DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" },
   { STATIC_CONTENT,  "/ROOT/skin/search_results.css?cacheid=76d39c84" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/i18n/test.json" },
+  // TODO: implement cache management of i18n resources
+  //{ STATIC_CONTENT, "/ROOT/skin/i18n/test.json?cacheid=unknown" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/languages.js" },
+  { STATIC_CONTENT, "/ROOT/skin/languages.js?cacheid=fe100348" },
 
   { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Title" },
   { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Description" },
@@ -289,9 +298,11 @@ R"EXPECTEDRESULT(                                <img src="../skin/download.png?
     },
     {
       /* url */ "/ROOT/viewer",
-R"EXPECTEDRESULT(    <link type="text/css" href="./skin/taskbar.css?cacheid=216d6b5d" rel="Stylesheet" />
+R"EXPECTEDRESULT(    <link type="text/css" href="./skin/taskbar.css?cacheid=2cbac34b" rel="Stylesheet" />
     <link type="text/css" href="./skin/css/autoComplete.css?cacheid=08951e06" rel="Stylesheet" />
-    <script type="text/javascript" src="./skin/viewer.js?cacheid=ab5374c5" defer></script>
+    <script type="module" src="./skin/i18n.js?cacheid=dcf3d584" defer></script>
+    <script type="text/javascript" src="./skin/languages.js?cacheid=fe100348" defer></script>
+    <script type="text/javascript" src="./skin/viewer.js?cacheid=b3c754ec" defer></script>
     <script type="text/javascript" src="./skin/autoComplete.min.js?cacheid=1191aaaf"></script>
       const blankPageUrl = root + "/skin/blank.html?cacheid=6b1fa032";
           <label for="kiwix_button_show_toggle"><img src="./skin/caret.png?cacheid=22b942b4" alt=""></label>
@@ -972,6 +983,77 @@ TEST_F(ServerTest, 500)
   }
 }
 
+TEST_F(ServerTest, UserLanguageList)
+{
+  const auto r = zfs1_->GET("/ROOT/skin/languages.js");
+  EXPECT_EQ(r->body,
+R"EXPECTEDRESPONSE(const uiLanguages = [
+  {
+    "বাংলা": "bn"
+  },
+  {
+    "Čeština": "cs"
+  },
+  {
+    "Deutsch": "de"
+  },
+  {
+    "English": "en"
+  },
+  {
+    "français": "fr"
+  },
+  {
+    "עברית": "he"
+  },
+  {
+    "Հայերեն": "hy"
+  },
+  {
+    "italiano": "it"
+  },
+  {
+    "日本語": "ja"
+  },
+  {
+    "한국어": "ko"
+  },
+  {
+    "kurdî": "ku-latn"
+  },
+  {
+    "македонски": "mk"
+  },
+  {
+    "ߒߞߏ": "nqo"
+  },
+  {
+    "Polski": "pl"
+  },
+  {
+    "русский": "ru"
+  },
+  {
+    "Sardu": "sc"
+  },
+  {
+    "slovenčina": "sk"
+  },
+  {
+    "Svenska": "sv"
+  },
+  {
+    "Türkçe": "tr"
+  },
+  {
+    "英语": "zh-hans"
+  },
+  {
+    "繁體中文": "zh-hant"
+  }
+])EXPECTEDRESPONSE");
+}
+
 TEST_F(ServerTest, UserLanguageControl)
 {
   struct TestData